1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_dma_buf.h" 7 8 #include <kunit/test.h> 9 #include <linux/dma-buf.h> 10 #include <linux/pci-p2pdma.h> 11 12 #include <drm/drm_device.h> 13 #include <drm/drm_prime.h> 14 #include <drm/ttm/ttm_tt.h> 15 16 #include "tests/xe_test.h" 17 #include "xe_bo.h" 18 #include "xe_device.h" 19 #include "xe_pm.h" 20 #include "xe_ttm_vram_mgr.h" 21 #include "xe_vm.h" 22 23 MODULE_IMPORT_NS("DMA_BUF"); 24 25 static int xe_dma_buf_attach(struct dma_buf *dmabuf, 26 struct dma_buf_attachment *attach) 27 { 28 struct drm_gem_object *obj = attach->dmabuf->priv; 29 30 if (attach->peer2peer && 31 pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0) 32 attach->peer2peer = false; 33 34 if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT)) 35 return -EOPNOTSUPP; 36 37 xe_pm_runtime_get(to_xe_device(obj->dev)); 38 return 0; 39 } 40 41 static void xe_dma_buf_detach(struct dma_buf *dmabuf, 42 struct dma_buf_attachment *attach) 43 { 44 struct drm_gem_object *obj = attach->dmabuf->priv; 45 46 xe_pm_runtime_put(to_xe_device(obj->dev)); 47 } 48 49 static int xe_dma_buf_pin(struct dma_buf_attachment *attach) 50 { 51 struct drm_gem_object *obj = attach->dmabuf->priv; 52 struct xe_bo *bo = gem_to_xe_bo(obj); 53 struct xe_device *xe = xe_bo_device(bo); 54 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 55 int ret; 56 57 /* 58 * For now only support pinning in TT memory, for two reasons: 59 * 1) Avoid pinning in a placement not accessible to some importers. 60 * 2) Pinning in VRAM requires PIN accounting which is a to-do. 61 */ 62 if (xe_bo_is_pinned(bo) && !xe_bo_is_mem_type(bo, XE_PL_TT)) { 63 drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n"); 64 return -EINVAL; 65 } 66 67 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); 68 if (ret) { 69 if (ret != -EINTR && ret != -ERESTARTSYS) 70 drm_dbg(&xe->drm, 71 "Failed migrating dma-buf to TT memory: %pe\n", 72 ERR_PTR(ret)); 73 return ret; 74 } 75 76 ret = xe_bo_pin_external(bo, true, exec); 77 xe_assert(xe, !ret); 78 79 return 0; 80 } 81 82 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach) 83 { 84 struct drm_gem_object *obj = attach->dmabuf->priv; 85 struct xe_bo *bo = gem_to_xe_bo(obj); 86 87 xe_bo_unpin_external(bo); 88 } 89 90 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, 91 enum dma_data_direction dir) 92 { 93 struct dma_buf *dma_buf = attach->dmabuf; 94 struct drm_gem_object *obj = dma_buf->priv; 95 struct xe_bo *bo = gem_to_xe_bo(obj); 96 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 97 struct sg_table *sgt; 98 int r = 0; 99 100 if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT)) 101 return ERR_PTR(-EOPNOTSUPP); 102 103 if (!xe_bo_is_pinned(bo)) { 104 if (!attach->peer2peer) 105 r = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); 106 else 107 r = xe_bo_validate(bo, NULL, false, exec); 108 if (r) 109 return ERR_PTR(r); 110 } 111 112 switch (bo->ttm.resource->mem_type) { 113 case XE_PL_TT: 114 sgt = drm_prime_pages_to_sg(obj->dev, 115 bo->ttm.ttm->pages, 116 bo->ttm.ttm->num_pages); 117 if (IS_ERR(sgt)) 118 return sgt; 119 120 if (dma_map_sgtable(attach->dev, sgt, dir, 121 DMA_ATTR_SKIP_CPU_SYNC)) 122 goto error_free; 123 break; 124 125 case XE_PL_VRAM0: 126 case XE_PL_VRAM1: 127 r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo), 128 bo->ttm.resource, 0, 129 bo->ttm.base.size, attach->dev, 130 dir, &sgt); 131 if (r) 132 return ERR_PTR(r); 133 break; 134 default: 135 return ERR_PTR(-EINVAL); 136 } 137 138 return sgt; 139 140 error_free: 141 sg_free_table(sgt); 142 kfree(sgt); 143 return ERR_PTR(-EBUSY); 144 } 145 146 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach, 147 struct sg_table *sgt, 148 enum dma_data_direction dir) 149 { 150 if (sg_page(sgt->sgl)) { 151 dma_unmap_sgtable(attach->dev, sgt, dir, 0); 152 sg_free_table(sgt); 153 kfree(sgt); 154 } else { 155 xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt); 156 } 157 } 158 159 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, 160 enum dma_data_direction direction) 161 { 162 struct drm_gem_object *obj = dma_buf->priv; 163 struct xe_bo *bo = gem_to_xe_bo(obj); 164 bool reads = (direction == DMA_BIDIRECTIONAL || 165 direction == DMA_FROM_DEVICE); 166 struct xe_validation_ctx ctx; 167 struct drm_exec exec; 168 int ret = 0; 169 170 if (!reads) 171 return 0; 172 173 /* Can we do interruptible lock here? */ 174 xe_validation_guard(&ctx, &xe_bo_device(bo)->val, &exec, (struct xe_val_flags) {}, ret) { 175 ret = drm_exec_lock_obj(&exec, &bo->ttm.base); 176 drm_exec_retry_on_contention(&exec); 177 if (ret) 178 break; 179 180 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, &exec); 181 drm_exec_retry_on_contention(&exec); 182 xe_validation_retry_on_oom(&ctx, &ret); 183 } 184 185 /* If we failed, cpu-access takes place in current placement. */ 186 return 0; 187 } 188 189 static const struct dma_buf_ops xe_dmabuf_ops = { 190 .attach = xe_dma_buf_attach, 191 .detach = xe_dma_buf_detach, 192 .pin = xe_dma_buf_pin, 193 .unpin = xe_dma_buf_unpin, 194 .map_dma_buf = xe_dma_buf_map, 195 .unmap_dma_buf = xe_dma_buf_unmap, 196 .release = drm_gem_dmabuf_release, 197 .begin_cpu_access = xe_dma_buf_begin_cpu_access, 198 .mmap = drm_gem_dmabuf_mmap, 199 .vmap = drm_gem_dmabuf_vmap, 200 .vunmap = drm_gem_dmabuf_vunmap, 201 }; 202 203 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) 204 { 205 struct xe_bo *bo = gem_to_xe_bo(obj); 206 struct dma_buf *buf; 207 struct ttm_operation_ctx ctx = { 208 .interruptible = true, 209 .no_wait_gpu = true, 210 /* We opt to avoid OOM on system pages allocations */ 211 .gfp_retry_mayfail = true, 212 .allow_res_evict = false, 213 }; 214 int ret; 215 216 if (bo->vm) 217 return ERR_PTR(-EPERM); 218 219 ret = ttm_bo_setup_export(&bo->ttm, &ctx); 220 if (ret) 221 return ERR_PTR(ret); 222 223 buf = drm_gem_prime_export(obj, flags); 224 if (!IS_ERR(buf)) 225 buf->ops = &xe_dmabuf_ops; 226 227 return buf; 228 } 229 230 static struct drm_gem_object * 231 xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, 232 struct dma_buf *dma_buf) 233 { 234 struct dma_resv *resv = dma_buf->resv; 235 struct xe_device *xe = to_xe_device(dev); 236 struct xe_validation_ctx ctx; 237 struct drm_gem_object *dummy_obj; 238 struct drm_exec exec; 239 struct xe_bo *bo; 240 int ret = 0; 241 242 dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm); 243 if (!dummy_obj) 244 return ERR_PTR(-ENOMEM); 245 246 dummy_obj->resv = resv; 247 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) { 248 ret = drm_exec_lock_obj(&exec, dummy_obj); 249 drm_exec_retry_on_contention(&exec); 250 if (ret) 251 break; 252 253 bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size, 254 0, /* Will require 1way or 2way for vm_bind */ 255 ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); 256 drm_exec_retry_on_contention(&exec); 257 if (IS_ERR(bo)) { 258 ret = PTR_ERR(bo); 259 xe_validation_retry_on_oom(&ctx, &ret); 260 break; 261 } 262 } 263 drm_gem_object_put(dummy_obj); 264 265 return ret ? ERR_PTR(ret) : &bo->ttm.base; 266 } 267 268 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach) 269 { 270 struct drm_gem_object *obj = attach->importer_priv; 271 struct xe_bo *bo = gem_to_xe_bo(obj); 272 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 273 274 XE_WARN_ON(xe_bo_evict(bo, exec)); 275 } 276 277 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = { 278 .allow_peer2peer = true, 279 .move_notify = xe_dma_buf_move_notify 280 }; 281 282 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 283 284 struct dma_buf_test_params { 285 struct xe_test_priv base; 286 const struct dma_buf_attach_ops *attach_ops; 287 bool force_different_devices; 288 u32 mem_mask; 289 }; 290 291 #define to_dma_buf_test_params(_priv) \ 292 container_of(_priv, struct dma_buf_test_params, base) 293 #endif 294 295 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, 296 struct dma_buf *dma_buf) 297 { 298 XE_TEST_DECLARE(struct dma_buf_test_params *test = 299 to_dma_buf_test_params 300 (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));) 301 const struct dma_buf_attach_ops *attach_ops; 302 struct dma_buf_attachment *attach; 303 struct drm_gem_object *obj; 304 struct xe_bo *bo; 305 306 if (dma_buf->ops == &xe_dmabuf_ops) { 307 obj = dma_buf->priv; 308 if (obj->dev == dev && 309 !XE_TEST_ONLY(test && test->force_different_devices)) { 310 /* 311 * Importing dmabuf exported from out own gem increases 312 * refcount on gem itself instead of f_count of dmabuf. 313 */ 314 drm_gem_object_get(obj); 315 return obj; 316 } 317 } 318 319 /* 320 * Don't publish the bo until we have a valid attachment, and a 321 * valid attachment needs the bo address. So pre-create a bo before 322 * creating the attachment and publish. 323 */ 324 bo = xe_bo_alloc(); 325 if (IS_ERR(bo)) 326 return ERR_CAST(bo); 327 328 attach_ops = &xe_dma_buf_attach_ops; 329 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 330 if (test) 331 attach_ops = test->attach_ops; 332 #endif 333 334 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base); 335 if (IS_ERR(attach)) { 336 obj = ERR_CAST(attach); 337 goto out_err; 338 } 339 340 /* Errors here will take care of freeing the bo. */ 341 obj = xe_dma_buf_init_obj(dev, bo, dma_buf); 342 if (IS_ERR(obj)) 343 return obj; 344 345 346 get_dma_buf(dma_buf); 347 obj->import_attach = attach; 348 return obj; 349 350 out_err: 351 xe_bo_free(bo); 352 353 return obj; 354 } 355 356 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 357 #include "tests/xe_dma_buf.c" 358 #endif 359