1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_dma_buf.h" 7 8 #include <kunit/test.h> 9 #include <linux/dma-buf.h> 10 #include <linux/pci-p2pdma.h> 11 12 #include <drm/drm_device.h> 13 #include <drm/drm_prime.h> 14 #include <drm/ttm/ttm_tt.h> 15 16 #include "tests/xe_test.h" 17 #include "xe_bo.h" 18 #include "xe_device.h" 19 #include "xe_pm.h" 20 #include "xe_ttm_vram_mgr.h" 21 #include "xe_vm.h" 22 23 MODULE_IMPORT_NS("DMA_BUF"); 24 25 static int xe_dma_buf_attach(struct dma_buf *dmabuf, 26 struct dma_buf_attachment *attach) 27 { 28 struct drm_gem_object *obj = attach->dmabuf->priv; 29 30 if (attach->peer2peer && 31 pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0) 32 attach->peer2peer = false; 33 34 if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT)) 35 return -EOPNOTSUPP; 36 37 xe_pm_runtime_get(to_xe_device(obj->dev)); 38 return 0; 39 } 40 41 static void xe_dma_buf_detach(struct dma_buf *dmabuf, 42 struct dma_buf_attachment *attach) 43 { 44 struct drm_gem_object *obj = attach->dmabuf->priv; 45 46 xe_pm_runtime_put(to_xe_device(obj->dev)); 47 } 48 49 static int xe_dma_buf_pin(struct dma_buf_attachment *attach) 50 { 51 struct dma_buf *dmabuf = attach->dmabuf; 52 struct drm_gem_object *obj = dmabuf->priv; 53 struct xe_bo *bo = gem_to_xe_bo(obj); 54 struct xe_device *xe = xe_bo_device(bo); 55 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 56 bool allow_vram = true; 57 int ret; 58 59 if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) { 60 allow_vram = false; 61 } else { 62 list_for_each_entry(attach, &dmabuf->attachments, node) { 63 if (!attach->peer2peer) { 64 allow_vram = false; 65 break; 66 } 67 } 68 } 69 70 if (xe_bo_is_pinned(bo) && !xe_bo_is_mem_type(bo, XE_PL_TT) && 71 !(xe_bo_is_vram(bo) && allow_vram)) { 72 drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n"); 73 return -EINVAL; 74 } 75 76 if (!allow_vram) { 77 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); 78 if (ret) { 79 if (ret != -EINTR && ret != -ERESTARTSYS) 80 drm_dbg(&xe->drm, 81 "Failed migrating dma-buf to TT memory: %pe\n", 82 ERR_PTR(ret)); 83 return ret; 84 } 85 } 86 87 ret = xe_bo_pin_external(bo, !allow_vram, exec); 88 xe_assert(xe, !ret); 89 90 return 0; 91 } 92 93 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach) 94 { 95 struct drm_gem_object *obj = attach->dmabuf->priv; 96 struct xe_bo *bo = gem_to_xe_bo(obj); 97 98 xe_bo_unpin_external(bo); 99 } 100 101 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, 102 enum dma_data_direction dir) 103 { 104 struct dma_buf *dma_buf = attach->dmabuf; 105 struct drm_gem_object *obj = dma_buf->priv; 106 struct xe_bo *bo = gem_to_xe_bo(obj); 107 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 108 struct sg_table *sgt; 109 int r = 0; 110 111 if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT)) 112 return ERR_PTR(-EOPNOTSUPP); 113 114 if (!xe_bo_is_pinned(bo)) { 115 if (!attach->peer2peer) 116 r = xe_bo_migrate(bo, XE_PL_TT, NULL, exec); 117 else 118 r = xe_bo_validate(bo, NULL, false, exec); 119 if (r) 120 return ERR_PTR(r); 121 } 122 123 switch (bo->ttm.resource->mem_type) { 124 case XE_PL_TT: 125 sgt = drm_prime_pages_to_sg(obj->dev, 126 bo->ttm.ttm->pages, 127 bo->ttm.ttm->num_pages); 128 if (IS_ERR(sgt)) 129 return sgt; 130 131 if (dma_map_sgtable(attach->dev, sgt, dir, 132 DMA_ATTR_SKIP_CPU_SYNC)) 133 goto error_free; 134 break; 135 136 case XE_PL_VRAM0: 137 case XE_PL_VRAM1: 138 r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo), 139 bo->ttm.resource, 0, 140 bo->ttm.base.size, attach->dev, 141 dir, &sgt); 142 if (r) 143 return ERR_PTR(r); 144 break; 145 default: 146 return ERR_PTR(-EINVAL); 147 } 148 149 return sgt; 150 151 error_free: 152 sg_free_table(sgt); 153 kfree(sgt); 154 return ERR_PTR(-EBUSY); 155 } 156 157 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach, 158 struct sg_table *sgt, 159 enum dma_data_direction dir) 160 { 161 if (sg_page(sgt->sgl)) { 162 dma_unmap_sgtable(attach->dev, sgt, dir, 0); 163 sg_free_table(sgt); 164 kfree(sgt); 165 } else { 166 xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt); 167 } 168 } 169 170 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, 171 enum dma_data_direction direction) 172 { 173 struct drm_gem_object *obj = dma_buf->priv; 174 struct xe_bo *bo = gem_to_xe_bo(obj); 175 bool reads = (direction == DMA_BIDIRECTIONAL || 176 direction == DMA_FROM_DEVICE); 177 struct xe_validation_ctx ctx; 178 struct drm_exec exec; 179 int ret = 0; 180 181 if (!reads) 182 return 0; 183 184 /* Can we do interruptible lock here? */ 185 xe_validation_guard(&ctx, &xe_bo_device(bo)->val, &exec, (struct xe_val_flags) {}, ret) { 186 ret = drm_exec_lock_obj(&exec, &bo->ttm.base); 187 drm_exec_retry_on_contention(&exec); 188 if (ret) 189 break; 190 191 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, &exec); 192 drm_exec_retry_on_contention(&exec); 193 xe_validation_retry_on_oom(&ctx, &ret); 194 } 195 196 /* If we failed, cpu-access takes place in current placement. */ 197 return 0; 198 } 199 200 static const struct dma_buf_ops xe_dmabuf_ops = { 201 .attach = xe_dma_buf_attach, 202 .detach = xe_dma_buf_detach, 203 .pin = xe_dma_buf_pin, 204 .unpin = xe_dma_buf_unpin, 205 .map_dma_buf = xe_dma_buf_map, 206 .unmap_dma_buf = xe_dma_buf_unmap, 207 .release = drm_gem_dmabuf_release, 208 .begin_cpu_access = xe_dma_buf_begin_cpu_access, 209 .mmap = drm_gem_dmabuf_mmap, 210 .vmap = drm_gem_dmabuf_vmap, 211 .vunmap = drm_gem_dmabuf_vunmap, 212 }; 213 214 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) 215 { 216 struct xe_bo *bo = gem_to_xe_bo(obj); 217 struct dma_buf *buf; 218 struct ttm_operation_ctx ctx = { 219 .interruptible = true, 220 .no_wait_gpu = true, 221 /* We opt to avoid OOM on system pages allocations */ 222 .gfp_retry_mayfail = true, 223 .allow_res_evict = false, 224 }; 225 int ret; 226 227 if (bo->vm) 228 return ERR_PTR(-EPERM); 229 230 ret = ttm_bo_setup_export(&bo->ttm, &ctx); 231 if (ret) 232 return ERR_PTR(ret); 233 234 buf = drm_gem_prime_export(obj, flags); 235 if (!IS_ERR(buf)) 236 buf->ops = &xe_dmabuf_ops; 237 238 return buf; 239 } 240 241 static struct drm_gem_object * 242 xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, 243 struct dma_buf *dma_buf) 244 { 245 struct dma_resv *resv = dma_buf->resv; 246 struct xe_device *xe = to_xe_device(dev); 247 struct xe_validation_ctx ctx; 248 struct drm_gem_object *dummy_obj; 249 struct drm_exec exec; 250 struct xe_bo *bo; 251 int ret = 0; 252 253 dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm); 254 if (!dummy_obj) 255 return ERR_PTR(-ENOMEM); 256 257 dummy_obj->resv = resv; 258 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) { 259 ret = drm_exec_lock_obj(&exec, dummy_obj); 260 drm_exec_retry_on_contention(&exec); 261 if (ret) 262 break; 263 264 bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size, 265 0, /* Will require 1way or 2way for vm_bind */ 266 ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec); 267 drm_exec_retry_on_contention(&exec); 268 if (IS_ERR(bo)) { 269 ret = PTR_ERR(bo); 270 xe_validation_retry_on_oom(&ctx, &ret); 271 break; 272 } 273 } 274 drm_gem_object_put(dummy_obj); 275 276 return ret ? ERR_PTR(ret) : &bo->ttm.base; 277 } 278 279 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach) 280 { 281 struct drm_gem_object *obj = attach->importer_priv; 282 struct xe_bo *bo = gem_to_xe_bo(obj); 283 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED; 284 285 XE_WARN_ON(xe_bo_evict(bo, exec)); 286 } 287 288 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = { 289 .allow_peer2peer = true, 290 .move_notify = xe_dma_buf_move_notify 291 }; 292 293 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 294 295 struct dma_buf_test_params { 296 struct xe_test_priv base; 297 const struct dma_buf_attach_ops *attach_ops; 298 bool force_different_devices; 299 u32 mem_mask; 300 }; 301 302 #define to_dma_buf_test_params(_priv) \ 303 container_of(_priv, struct dma_buf_test_params, base) 304 #endif 305 306 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, 307 struct dma_buf *dma_buf) 308 { 309 XE_TEST_DECLARE(struct dma_buf_test_params *test = 310 to_dma_buf_test_params 311 (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));) 312 const struct dma_buf_attach_ops *attach_ops; 313 struct dma_buf_attachment *attach; 314 struct drm_gem_object *obj; 315 struct xe_bo *bo; 316 317 if (dma_buf->ops == &xe_dmabuf_ops) { 318 obj = dma_buf->priv; 319 if (obj->dev == dev && 320 !XE_TEST_ONLY(test && test->force_different_devices)) { 321 /* 322 * Importing dmabuf exported from out own gem increases 323 * refcount on gem itself instead of f_count of dmabuf. 324 */ 325 drm_gem_object_get(obj); 326 return obj; 327 } 328 } 329 330 /* 331 * Don't publish the bo until we have a valid attachment, and a 332 * valid attachment needs the bo address. So pre-create a bo before 333 * creating the attachment and publish. 334 */ 335 bo = xe_bo_alloc(); 336 if (IS_ERR(bo)) 337 return ERR_CAST(bo); 338 339 attach_ops = &xe_dma_buf_attach_ops; 340 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 341 if (test) 342 attach_ops = test->attach_ops; 343 #endif 344 345 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base); 346 if (IS_ERR(attach)) { 347 obj = ERR_CAST(attach); 348 goto out_err; 349 } 350 351 /* Errors here will take care of freeing the bo. */ 352 obj = xe_dma_buf_init_obj(dev, bo, dma_buf); 353 if (IS_ERR(obj)) 354 return obj; 355 356 357 get_dma_buf(dma_buf); 358 obj->import_attach = attach; 359 return obj; 360 361 out_err: 362 xe_bo_free(bo); 363 364 return obj; 365 } 366 367 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 368 #include "tests/xe_dma_buf.c" 369 #endif 370