1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_dma_buf.h" 7 8 #include <kunit/test.h> 9 #include <linux/dma-buf.h> 10 #include <linux/pci-p2pdma.h> 11 12 #include <drm/drm_device.h> 13 #include <drm/drm_prime.h> 14 #include <drm/ttm/ttm_tt.h> 15 16 #include "tests/xe_test.h" 17 #include "xe_bo.h" 18 #include "xe_device.h" 19 #include "xe_pm.h" 20 #include "xe_ttm_vram_mgr.h" 21 #include "xe_vm.h" 22 23 MODULE_IMPORT_NS("DMA_BUF"); 24 25 static int xe_dma_buf_attach(struct dma_buf *dmabuf, 26 struct dma_buf_attachment *attach) 27 { 28 struct drm_gem_object *obj = attach->dmabuf->priv; 29 30 if (attach->peer2peer && 31 pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0) 32 attach->peer2peer = false; 33 34 if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT)) 35 return -EOPNOTSUPP; 36 37 xe_pm_runtime_get(to_xe_device(obj->dev)); 38 return 0; 39 } 40 41 static void xe_dma_buf_detach(struct dma_buf *dmabuf, 42 struct dma_buf_attachment *attach) 43 { 44 struct drm_gem_object *obj = attach->dmabuf->priv; 45 46 xe_pm_runtime_put(to_xe_device(obj->dev)); 47 } 48 49 static int xe_dma_buf_pin(struct dma_buf_attachment *attach) 50 { 51 struct dma_buf *dmabuf = attach->dmabuf; 52 struct drm_gem_object *obj = dmabuf->priv; 53 struct xe_bo *bo = gem_to_xe_bo(obj); 54 struct xe_device *xe = xe_bo_device(bo); 55 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 56 bool allow_vram = true; 57 int ret; 58 59 list_for_each_entry(attach, &dmabuf->attachments, node) { 60 if (!attach->peer2peer) { 61 allow_vram = false; 62 break; 63 } 64 } 65 66 if (xe_bo_is_pinned(bo) && !xe_bo_is_mem_type(bo, XE_PL_TT) && 67 !(xe_bo_is_vram(bo) && allow_vram)) { 68 drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n"); 69 return -EINVAL; 70 } 71 72 if (!allow_vram) { 73 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); 74 if (ret) { 75 if (ret != -EINTR && ret != -ERESTARTSYS) 76 drm_dbg(&xe->drm, 77 "Failed migrating dma-buf to TT memory: %pe\n", 78 ERR_PTR(ret)); 79 return ret; 80 } 81 } 82 83 ret = xe_bo_pin_external(bo, !allow_vram, exec); 84 xe_assert(xe, !ret); 85 86 return 0; 87 } 88 89 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach) 90 { 91 struct drm_gem_object *obj = attach->dmabuf->priv; 92 struct xe_bo *bo = gem_to_xe_bo(obj); 93 94 xe_bo_unpin_external(bo); 95 } 96 97 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, 98 enum dma_data_direction dir) 99 { 100 struct dma_buf *dma_buf = attach->dmabuf; 101 struct drm_gem_object *obj = dma_buf->priv; 102 struct xe_bo *bo = gem_to_xe_bo(obj); 103 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 104 struct sg_table *sgt; 105 int r = 0; 106 107 if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT)) 108 return ERR_PTR(-EOPNOTSUPP); 109 110 if (!xe_bo_is_pinned(bo)) { 111 if (!attach->peer2peer) 112 r = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); 113 else 114 r = xe_bo_validate(bo, NULL, false, exec); 115 if (r) 116 return ERR_PTR(r); 117 } 118 119 switch (bo->ttm.resource->mem_type) { 120 case XE_PL_TT: 121 sgt = drm_prime_pages_to_sg(obj->dev, 122 bo->ttm.ttm->pages, 123 obj->size >> PAGE_SHIFT); 124 if (IS_ERR(sgt)) 125 return sgt; 126 127 if (dma_map_sgtable(attach->dev, sgt, dir, 128 DMA_ATTR_SKIP_CPU_SYNC)) 129 goto error_free; 130 break; 131 132 case XE_PL_VRAM0: 133 case XE_PL_VRAM1: 134 r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo), 135 bo->ttm.resource, 0, 136 bo->ttm.base.size, attach->dev, 137 dir, &sgt); 138 if (r) 139 return ERR_PTR(r); 140 break; 141 default: 142 return ERR_PTR(-EINVAL); 143 } 144 145 return sgt; 146 147 error_free: 148 sg_free_table(sgt); 149 kfree(sgt); 150 return ERR_PTR(-EBUSY); 151 } 152 153 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach, 154 struct sg_table *sgt, 155 enum dma_data_direction dir) 156 { 157 if (sg_page(sgt->sgl)) { 158 dma_unmap_sgtable(attach->dev, sgt, dir, 0); 159 sg_free_table(sgt); 160 kfree(sgt); 161 } else { 162 xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt); 163 } 164 } 165 166 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, 167 enum dma_data_direction direction) 168 { 169 struct drm_gem_object *obj = dma_buf->priv; 170 struct xe_bo *bo = gem_to_xe_bo(obj); 171 bool reads = (direction == DMA_BIDIRECTIONAL || 172 direction == DMA_FROM_DEVICE); 173 struct xe_validation_ctx ctx; 174 struct drm_exec exec; 175 int ret = 0; 176 177 if (!reads) 178 return 0; 179 180 /* Can we do interruptible lock here? */ 181 xe_validation_guard(&ctx, &xe_bo_device(bo)->val, &exec, (struct xe_val_flags) {}, ret) { 182 ret = drm_exec_lock_obj(&exec, &bo->ttm.base); 183 drm_exec_retry_on_contention(&exec); 184 if (ret) 185 break; 186 187 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, &exec); 188 drm_exec_retry_on_contention(&exec); 189 xe_validation_retry_on_oom(&ctx, &ret); 190 } 191 192 /* If we failed, cpu-access takes place in current placement. */ 193 return 0; 194 } 195 196 static void xe_dma_buf_release(struct dma_buf *dmabuf) 197 { 198 struct drm_gem_object *obj = dmabuf->priv; 199 struct xe_bo *bo = gem_to_xe_bo(obj); 200 201 xe_bo_lock(bo, false); 202 xe_bo_willneed_put_locked(bo); 203 xe_bo_unlock(bo); 204 205 drm_gem_dmabuf_release(dmabuf); 206 } 207 208 static const struct dma_buf_ops xe_dmabuf_ops = { 209 .attach = xe_dma_buf_attach, 210 .detach = xe_dma_buf_detach, 211 .pin = xe_dma_buf_pin, 212 .unpin = xe_dma_buf_unpin, 213 .map_dma_buf = xe_dma_buf_map, 214 .unmap_dma_buf = xe_dma_buf_unmap, 215 .release = xe_dma_buf_release, 216 .begin_cpu_access = xe_dma_buf_begin_cpu_access, 217 .mmap = drm_gem_dmabuf_mmap, 218 .vmap = drm_gem_dmabuf_vmap, 219 .vunmap = drm_gem_dmabuf_vunmap, 220 }; 221 222 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) 223 { 224 struct xe_bo *bo = gem_to_xe_bo(obj); 225 struct dma_buf *buf; 226 struct ttm_operation_ctx ctx = { 227 .interruptible = true, 228 .no_wait_gpu = true, 229 /* We opt to avoid OOM on system pages allocations */ 230 .gfp_retry_mayfail = true, 231 .allow_res_evict = false, 232 }; 233 int ret; 234 235 if (bo->vm) 236 return ERR_PTR(-EPERM); 237 238 /* 239 * Reject exporting purgeable BOs. DONTNEED BOs can be purged 240 * at any time, making the exported dma-buf unusable. Purged BOs 241 * have no backing store and are permanently invalid. 242 */ 243 ret = xe_bo_lock(bo, true); 244 if (ret) 245 return ERR_PTR(ret); 246 247 if (xe_bo_madv_is_dontneed(bo)) { 248 ret = -EBUSY; 249 goto out_unlock; 250 } 251 252 if (xe_bo_is_purged(bo)) { 253 ret = -EINVAL; 254 goto out_unlock; 255 } 256 257 xe_bo_willneed_get_locked(bo); 258 xe_bo_unlock(bo); 259 260 ret = ttm_bo_setup_export(&bo->ttm, &ctx); 261 if (ret) 262 goto out_put; 263 264 buf = drm_gem_prime_export(obj, flags); 265 if (IS_ERR(buf)) { 266 ret = PTR_ERR(buf); 267 goto out_put; 268 } 269 270 buf->ops = &xe_dmabuf_ops; 271 return buf; 272 273 out_put: 274 xe_bo_lock(bo, false); 275 xe_bo_willneed_put_locked(bo); 276 out_unlock: 277 xe_bo_unlock(bo); 278 return ERR_PTR(ret); 279 } 280 281 static struct drm_gem_object * 282 xe_dma_buf_create_obj(struct drm_device *dev, struct dma_buf *dma_buf) 283 { 284 struct dma_resv *resv = dma_buf->resv; 285 struct xe_device *xe = to_xe_device(dev); 286 struct xe_validation_ctx ctx; 287 struct drm_gem_object *dummy_obj; 288 struct drm_exec exec; 289 struct xe_bo *bo; 290 int ret = 0; 291 292 dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm); 293 if (!dummy_obj) 294 return ERR_PTR(-ENOMEM); 295 296 dummy_obj->resv = resv; 297 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) { 298 ret = drm_exec_lock_obj(&exec, dummy_obj); 299 drm_exec_retry_on_contention(&exec); 300 if (ret) 301 break; 302 303 bo = xe_bo_init_locked(xe, NULL, NULL, resv, NULL, dma_buf->size, 304 0, /* Will require 1way or 2way for vm_bind */ 305 ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); 306 drm_exec_retry_on_contention(&exec); 307 if (IS_ERR(bo)) { 308 ret = PTR_ERR(bo); 309 xe_validation_retry_on_oom(&ctx, &ret); 310 break; 311 } 312 } 313 drm_gem_object_put(dummy_obj); 314 315 return ret ? ERR_PTR(ret) : &bo->ttm.base; 316 } 317 318 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach) 319 { 320 struct drm_gem_object *obj = attach->importer_priv; 321 struct xe_bo *bo = gem_to_xe_bo(obj); 322 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 323 324 XE_WARN_ON(xe_bo_evict(bo, exec)); 325 } 326 327 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = { 328 .allow_peer2peer = true, 329 .invalidate_mappings = xe_dma_buf_move_notify 330 }; 331 332 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 333 334 struct dma_buf_test_params { 335 struct xe_test_priv base; 336 const struct dma_buf_attach_ops *attach_ops; 337 bool force_different_devices; 338 u32 mem_mask; 339 }; 340 341 #define to_dma_buf_test_params(_priv) \ 342 container_of(_priv, struct dma_buf_test_params, base) 343 #endif 344 345 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, 346 struct dma_buf *dma_buf) 347 { 348 XE_TEST_DECLARE(struct dma_buf_test_params *test = 349 to_dma_buf_test_params 350 (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));) 351 const struct dma_buf_attach_ops *attach_ops; 352 struct dma_buf_attachment *attach; 353 struct drm_gem_object *obj; 354 355 if (dma_buf->ops == &xe_dmabuf_ops) { 356 obj = dma_buf->priv; 357 if (obj->dev == dev && 358 !XE_TEST_ONLY(test && test->force_different_devices)) { 359 /* 360 * Importing dmabuf exported from out own gem increases 361 * refcount on gem itself instead of f_count of dmabuf. 362 */ 363 drm_gem_object_get(obj); 364 return obj; 365 } 366 } 367 368 /* 369 * This needs to happen before the attach, since it will create a new 370 * attachment for this, and add it to the list of attachments, at which 371 * point it is globally visible, and at any point the export side can 372 * call into on invalidate_mappings callback, which require a working 373 * object. 374 */ 375 obj = xe_dma_buf_create_obj(dev, dma_buf); 376 if (IS_ERR(obj)) 377 return obj; 378 379 attach_ops = &xe_dma_buf_attach_ops; 380 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 381 if (test) 382 attach_ops = test->attach_ops; 383 #endif 384 385 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, obj); 386 if (IS_ERR(attach)) { 387 xe_bo_put(gem_to_xe_bo(obj)); 388 return ERR_CAST(attach); 389 } 390 391 get_dma_buf(dma_buf); 392 obj->import_attach = attach; 393 return obj; 394 } 395 396 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 397 #include "tests/xe_dma_buf.c" 398 #endif 399