1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_dma_buf.h"
7
8 #include <kunit/test.h>
9 #include <linux/dma-buf.h>
10 #include <linux/pci-p2pdma.h>
11
12 #include <drm/drm_device.h>
13 #include <drm/drm_prime.h>
14 #include <drm/ttm/ttm_tt.h>
15
16 #include "tests/xe_test.h"
17 #include "xe_bo.h"
18 #include "xe_device.h"
19 #include "xe_pm.h"
20 #include "xe_ttm_vram_mgr.h"
21 #include "xe_vm.h"
22
23 MODULE_IMPORT_NS("DMA_BUF");
24
xe_dma_buf_attach(struct dma_buf * dmabuf,struct dma_buf_attachment * attach)25 static int xe_dma_buf_attach(struct dma_buf *dmabuf,
26 struct dma_buf_attachment *attach)
27 {
28 struct drm_gem_object *obj = attach->dmabuf->priv;
29
30 if (attach->peer2peer &&
31 pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
32 attach->peer2peer = false;
33
34 if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
35 return -EOPNOTSUPP;
36
37 xe_pm_runtime_get(to_xe_device(obj->dev));
38 return 0;
39 }
40
xe_dma_buf_detach(struct dma_buf * dmabuf,struct dma_buf_attachment * attach)41 static void xe_dma_buf_detach(struct dma_buf *dmabuf,
42 struct dma_buf_attachment *attach)
43 {
44 struct drm_gem_object *obj = attach->dmabuf->priv;
45
46 xe_pm_runtime_put(to_xe_device(obj->dev));
47 }
48
xe_dma_buf_pin(struct dma_buf_attachment * attach)49 static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
50 {
51 struct dma_buf *dmabuf = attach->dmabuf;
52 struct drm_gem_object *obj = dmabuf->priv;
53 struct xe_bo *bo = gem_to_xe_bo(obj);
54 struct xe_device *xe = xe_bo_device(bo);
55 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
56 bool allow_vram = true;
57 int ret;
58
59 list_for_each_entry(attach, &dmabuf->attachments, node) {
60 if (!attach->peer2peer) {
61 allow_vram = false;
62 break;
63 }
64 }
65
66 if (xe_bo_is_pinned(bo) && !xe_bo_is_mem_type(bo, XE_PL_TT) &&
67 !(xe_bo_is_vram(bo) && allow_vram)) {
68 drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n");
69 return -EINVAL;
70 }
71
72 if (!allow_vram) {
73 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, exec);
74 if (ret) {
75 if (ret != -EINTR && ret != -ERESTARTSYS)
76 drm_dbg(&xe->drm,
77 "Failed migrating dma-buf to TT memory: %pe\n",
78 ERR_PTR(ret));
79 return ret;
80 }
81 }
82
83 ret = xe_bo_pin_external(bo, !allow_vram, exec);
84 xe_assert(xe, !ret);
85
86 return 0;
87 }
88
xe_dma_buf_unpin(struct dma_buf_attachment * attach)89 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
90 {
91 struct drm_gem_object *obj = attach->dmabuf->priv;
92 struct xe_bo *bo = gem_to_xe_bo(obj);
93
94 xe_bo_unpin_external(bo);
95 }
96
xe_dma_buf_map(struct dma_buf_attachment * attach,enum dma_data_direction dir)97 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
98 enum dma_data_direction dir)
99 {
100 struct dma_buf *dma_buf = attach->dmabuf;
101 struct drm_gem_object *obj = dma_buf->priv;
102 struct xe_bo *bo = gem_to_xe_bo(obj);
103 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
104 struct sg_table *sgt;
105 int r = 0;
106
107 if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
108 return ERR_PTR(-EOPNOTSUPP);
109
110 if (!xe_bo_is_pinned(bo)) {
111 if (!attach->peer2peer)
112 r = xe_bo_migrate(bo, XE_PL_TT, NULL, exec);
113 else
114 r = xe_bo_validate(bo, NULL, false, exec);
115 if (r)
116 return ERR_PTR(r);
117 }
118
119 switch (bo->ttm.resource->mem_type) {
120 case XE_PL_TT:
121 sgt = drm_prime_pages_to_sg(obj->dev,
122 bo->ttm.ttm->pages,
123 obj->size >> PAGE_SHIFT);
124 if (IS_ERR(sgt))
125 return sgt;
126
127 if (dma_map_sgtable(attach->dev, sgt, dir,
128 DMA_ATTR_SKIP_CPU_SYNC))
129 goto error_free;
130 break;
131
132 case XE_PL_VRAM0:
133 case XE_PL_VRAM1:
134 r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo),
135 bo->ttm.resource, 0,
136 bo->ttm.base.size, attach->dev,
137 dir, &sgt);
138 if (r)
139 return ERR_PTR(r);
140 break;
141 default:
142 return ERR_PTR(-EINVAL);
143 }
144
145 return sgt;
146
147 error_free:
148 sg_free_table(sgt);
149 kfree(sgt);
150 return ERR_PTR(-EBUSY);
151 }
152
xe_dma_buf_unmap(struct dma_buf_attachment * attach,struct sg_table * sgt,enum dma_data_direction dir)153 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
154 struct sg_table *sgt,
155 enum dma_data_direction dir)
156 {
157 if (sg_page(sgt->sgl)) {
158 dma_unmap_sgtable(attach->dev, sgt, dir, 0);
159 sg_free_table(sgt);
160 kfree(sgt);
161 } else {
162 xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt);
163 }
164 }
165
xe_dma_buf_begin_cpu_access(struct dma_buf * dma_buf,enum dma_data_direction direction)166 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
167 enum dma_data_direction direction)
168 {
169 struct drm_gem_object *obj = dma_buf->priv;
170 struct xe_bo *bo = gem_to_xe_bo(obj);
171 bool reads = (direction == DMA_BIDIRECTIONAL ||
172 direction == DMA_FROM_DEVICE);
173 struct xe_validation_ctx ctx;
174 struct drm_exec exec;
175 int ret = 0;
176
177 if (!reads)
178 return 0;
179
180 /* Can we do interruptible lock here? */
181 xe_validation_guard(&ctx, &xe_bo_device(bo)->val, &exec, (struct xe_val_flags) {}, ret) {
182 ret = drm_exec_lock_obj(&exec, &bo->ttm.base);
183 drm_exec_retry_on_contention(&exec);
184 if (ret)
185 break;
186
187 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, &exec);
188 drm_exec_retry_on_contention(&exec);
189 xe_validation_retry_on_oom(&ctx, &ret);
190 }
191
192 /* If we failed, cpu-access takes place in current placement. */
193 return 0;
194 }
195
196 static const struct dma_buf_ops xe_dmabuf_ops = {
197 .attach = xe_dma_buf_attach,
198 .detach = xe_dma_buf_detach,
199 .pin = xe_dma_buf_pin,
200 .unpin = xe_dma_buf_unpin,
201 .map_dma_buf = xe_dma_buf_map,
202 .unmap_dma_buf = xe_dma_buf_unmap,
203 .release = drm_gem_dmabuf_release,
204 .begin_cpu_access = xe_dma_buf_begin_cpu_access,
205 .mmap = drm_gem_dmabuf_mmap,
206 .vmap = drm_gem_dmabuf_vmap,
207 .vunmap = drm_gem_dmabuf_vunmap,
208 };
209
xe_gem_prime_export(struct drm_gem_object * obj,int flags)210 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
211 {
212 struct xe_bo *bo = gem_to_xe_bo(obj);
213 struct dma_buf *buf;
214 struct ttm_operation_ctx ctx = {
215 .interruptible = true,
216 .no_wait_gpu = true,
217 /* We opt to avoid OOM on system pages allocations */
218 .gfp_retry_mayfail = true,
219 .allow_res_evict = false,
220 };
221 int ret;
222
223 if (bo->vm)
224 return ERR_PTR(-EPERM);
225
226 /*
227 * Reject exporting purgeable BOs. DONTNEED BOs can be purged
228 * at any time, making the exported dma-buf unusable. Purged BOs
229 * have no backing store and are permanently invalid.
230 */
231 ret = xe_bo_lock(bo, true);
232 if (ret)
233 return ERR_PTR(ret);
234
235 if (xe_bo_madv_is_dontneed(bo)) {
236 ret = -EBUSY;
237 goto out_unlock;
238 }
239
240 if (xe_bo_is_purged(bo)) {
241 ret = -EINVAL;
242 goto out_unlock;
243 }
244 xe_bo_unlock(bo);
245
246 ret = ttm_bo_setup_export(&bo->ttm, &ctx);
247 if (ret)
248 return ERR_PTR(ret);
249
250 buf = drm_gem_prime_export(obj, flags);
251 if (!IS_ERR(buf))
252 buf->ops = &xe_dmabuf_ops;
253
254 return buf;
255
256 out_unlock:
257 xe_bo_unlock(bo);
258 return ERR_PTR(ret);
259 }
260
261 /*
262 * Takes ownership of @storage: on success it is transferred to the returned
263 * drm_gem_object; on failure it is freed before returning the error.
264 * This matches the contract of xe_bo_init_locked() which frees @storage on
265 * its error paths, so callers need not (and must not) free @storage after
266 * this call.
267 */
268 static struct drm_gem_object *
xe_dma_buf_init_obj(struct drm_device * dev,struct xe_bo * storage,struct dma_buf * dma_buf)269 xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
270 struct dma_buf *dma_buf)
271 {
272 struct dma_resv *resv = dma_buf->resv;
273 struct xe_device *xe = to_xe_device(dev);
274 struct xe_validation_ctx ctx;
275 struct drm_gem_object *dummy_obj;
276 struct drm_exec exec;
277 struct xe_bo *bo;
278 int ret = 0;
279
280 dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
281 if (!dummy_obj) {
282 xe_bo_free(storage);
283 return ERR_PTR(-ENOMEM);
284 }
285
286 dummy_obj->resv = resv;
287 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) {
288 ret = drm_exec_lock_obj(&exec, dummy_obj);
289 drm_exec_retry_on_contention(&exec);
290 if (ret)
291 break;
292
293 /* xe_bo_init_locked() frees storage on error */
294 bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
295 0, /* Will require 1way or 2way for vm_bind */
296 ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec);
297 drm_exec_retry_on_contention(&exec);
298 if (IS_ERR(bo)) {
299 ret = PTR_ERR(bo);
300 xe_validation_retry_on_oom(&ctx, &ret);
301 break;
302 }
303 }
304 drm_gem_object_put(dummy_obj);
305
306 return ret ? ERR_PTR(ret) : &bo->ttm.base;
307 }
308
xe_dma_buf_move_notify(struct dma_buf_attachment * attach)309 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach)
310 {
311 struct drm_gem_object *obj = attach->importer_priv;
312 struct xe_bo *bo = gem_to_xe_bo(obj);
313 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
314
315 XE_WARN_ON(xe_bo_evict(bo, exec));
316 }
317
318 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = {
319 .allow_peer2peer = true,
320 .invalidate_mappings = xe_dma_buf_move_notify
321 };
322
323 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
324
325 struct dma_buf_test_params {
326 struct xe_test_priv base;
327 const struct dma_buf_attach_ops *attach_ops;
328 bool force_different_devices;
329 u32 mem_mask;
330 };
331
332 #define to_dma_buf_test_params(_priv) \
333 container_of(_priv, struct dma_buf_test_params, base)
334 #endif
335
xe_gem_prime_import(struct drm_device * dev,struct dma_buf * dma_buf)336 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
337 struct dma_buf *dma_buf)
338 {
339 XE_TEST_DECLARE(struct dma_buf_test_params *test =
340 to_dma_buf_test_params
341 (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));)
342 const struct dma_buf_attach_ops *attach_ops;
343 struct dma_buf_attachment *attach;
344 struct drm_gem_object *obj;
345 struct xe_bo *bo;
346
347 if (dma_buf->ops == &xe_dmabuf_ops) {
348 obj = dma_buf->priv;
349 if (obj->dev == dev &&
350 !XE_TEST_ONLY(test && test->force_different_devices)) {
351 /*
352 * Importing dmabuf exported from out own gem increases
353 * refcount on gem itself instead of f_count of dmabuf.
354 */
355 drm_gem_object_get(obj);
356 return obj;
357 }
358 }
359
360 /*
361 * Don't publish the bo until we have a valid attachment, and a
362 * valid attachment needs the bo address. So pre-create a bo before
363 * creating the attachment and publish.
364 */
365 bo = xe_bo_alloc();
366 if (IS_ERR(bo))
367 return ERR_CAST(bo);
368
369 attach_ops = &xe_dma_buf_attach_ops;
370 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
371 if (test)
372 attach_ops = test->attach_ops;
373 #endif
374
375 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
376 if (IS_ERR(attach)) {
377 obj = ERR_CAST(attach);
378 goto out_err;
379 }
380
381 /*
382 * xe_dma_buf_init_obj() takes ownership of bo on both success
383 * and failure, so we must not touch bo after this call.
384 */
385 obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
386 if (IS_ERR(obj)) {
387 dma_buf_detach(dma_buf, attach);
388 return obj;
389 }
390 get_dma_buf(dma_buf);
391 obj->import_attach = attach;
392 return obj;
393
394 out_err:
395 xe_bo_free(bo);
396
397 return obj;
398 }
399
400 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
401 #include "tests/xe_dma_buf.c"
402 #endif
403