1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_dma_buf.h"
7
8 #include <kunit/test.h>
9 #include <linux/dma-buf.h>
10 #include <linux/pci-p2pdma.h>
11
12 #include <drm/drm_device.h>
13 #include <drm/drm_prime.h>
14 #include <drm/ttm/ttm_tt.h>
15
16 #include "tests/xe_test.h"
17 #include "xe_bo.h"
18 #include "xe_device.h"
19 #include "xe_pm.h"
20 #include "xe_ttm_vram_mgr.h"
21 #include "xe_vm.h"
22
23 MODULE_IMPORT_NS("DMA_BUF");
24
xe_dma_buf_attach(struct dma_buf * dmabuf,struct dma_buf_attachment * attach)25 static int xe_dma_buf_attach(struct dma_buf *dmabuf,
26 struct dma_buf_attachment *attach)
27 {
28 struct drm_gem_object *obj = attach->dmabuf->priv;
29
30 if (attach->peer2peer &&
31 pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
32 attach->peer2peer = false;
33
34 if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
35 return -EOPNOTSUPP;
36
37 xe_pm_runtime_get(to_xe_device(obj->dev));
38 return 0;
39 }
40
xe_dma_buf_detach(struct dma_buf * dmabuf,struct dma_buf_attachment * attach)41 static void xe_dma_buf_detach(struct dma_buf *dmabuf,
42 struct dma_buf_attachment *attach)
43 {
44 struct drm_gem_object *obj = attach->dmabuf->priv;
45
46 xe_pm_runtime_put(to_xe_device(obj->dev));
47 }
48
xe_dma_buf_pin(struct dma_buf_attachment * attach)49 static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
50 {
51 struct drm_gem_object *obj = attach->dmabuf->priv;
52 struct xe_bo *bo = gem_to_xe_bo(obj);
53 struct xe_device *xe = xe_bo_device(bo);
54 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
55 int ret;
56
57 /*
58 * For now only support pinning in TT memory, for two reasons:
59 * 1) Avoid pinning in a placement not accessible to some importers.
60 * 2) Pinning in VRAM requires PIN accounting which is a to-do.
61 */
62 if (xe_bo_is_pinned(bo) && !xe_bo_is_mem_type(bo, XE_PL_TT)) {
63 drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n");
64 return -EINVAL;
65 }
66
67 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, exec);
68 if (ret) {
69 if (ret != -EINTR && ret != -ERESTARTSYS)
70 drm_dbg(&xe->drm,
71 "Failed migrating dma-buf to TT memory: %pe\n",
72 ERR_PTR(ret));
73 return ret;
74 }
75
76 ret = xe_bo_pin_external(bo, true, exec);
77 xe_assert(xe, !ret);
78
79 return 0;
80 }
81
xe_dma_buf_unpin(struct dma_buf_attachment * attach)82 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
83 {
84 struct drm_gem_object *obj = attach->dmabuf->priv;
85 struct xe_bo *bo = gem_to_xe_bo(obj);
86
87 xe_bo_unpin_external(bo);
88 }
89
xe_dma_buf_map(struct dma_buf_attachment * attach,enum dma_data_direction dir)90 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
91 enum dma_data_direction dir)
92 {
93 struct dma_buf *dma_buf = attach->dmabuf;
94 struct drm_gem_object *obj = dma_buf->priv;
95 struct xe_bo *bo = gem_to_xe_bo(obj);
96 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
97 struct sg_table *sgt;
98 int r = 0;
99
100 if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
101 return ERR_PTR(-EOPNOTSUPP);
102
103 if (!xe_bo_is_pinned(bo)) {
104 if (!attach->peer2peer)
105 r = xe_bo_migrate(bo, XE_PL_TT, NULL, exec);
106 else
107 r = xe_bo_validate(bo, NULL, false, exec);
108 if (r)
109 return ERR_PTR(r);
110 }
111
112 switch (bo->ttm.resource->mem_type) {
113 case XE_PL_TT:
114 sgt = drm_prime_pages_to_sg(obj->dev,
115 bo->ttm.ttm->pages,
116 bo->ttm.ttm->num_pages);
117 if (IS_ERR(sgt))
118 return sgt;
119
120 if (dma_map_sgtable(attach->dev, sgt, dir,
121 DMA_ATTR_SKIP_CPU_SYNC))
122 goto error_free;
123 break;
124
125 case XE_PL_VRAM0:
126 case XE_PL_VRAM1:
127 r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo),
128 bo->ttm.resource, 0,
129 bo->ttm.base.size, attach->dev,
130 dir, &sgt);
131 if (r)
132 return ERR_PTR(r);
133 break;
134 default:
135 return ERR_PTR(-EINVAL);
136 }
137
138 return sgt;
139
140 error_free:
141 sg_free_table(sgt);
142 kfree(sgt);
143 return ERR_PTR(-EBUSY);
144 }
145
xe_dma_buf_unmap(struct dma_buf_attachment * attach,struct sg_table * sgt,enum dma_data_direction dir)146 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
147 struct sg_table *sgt,
148 enum dma_data_direction dir)
149 {
150 if (sg_page(sgt->sgl)) {
151 dma_unmap_sgtable(attach->dev, sgt, dir, 0);
152 sg_free_table(sgt);
153 kfree(sgt);
154 } else {
155 xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt);
156 }
157 }
158
xe_dma_buf_begin_cpu_access(struct dma_buf * dma_buf,enum dma_data_direction direction)159 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
160 enum dma_data_direction direction)
161 {
162 struct drm_gem_object *obj = dma_buf->priv;
163 struct xe_bo *bo = gem_to_xe_bo(obj);
164 bool reads = (direction == DMA_BIDIRECTIONAL ||
165 direction == DMA_FROM_DEVICE);
166 struct xe_validation_ctx ctx;
167 struct drm_exec exec;
168 int ret = 0;
169
170 if (!reads)
171 return 0;
172
173 /* Can we do interruptible lock here? */
174 xe_validation_guard(&ctx, &xe_bo_device(bo)->val, &exec, (struct xe_val_flags) {}, ret) {
175 ret = drm_exec_lock_obj(&exec, &bo->ttm.base);
176 drm_exec_retry_on_contention(&exec);
177 if (ret)
178 break;
179
180 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, &exec);
181 drm_exec_retry_on_contention(&exec);
182 xe_validation_retry_on_oom(&ctx, &ret);
183 }
184
185 /* If we failed, cpu-access takes place in current placement. */
186 return 0;
187 }
188
189 static const struct dma_buf_ops xe_dmabuf_ops = {
190 .attach = xe_dma_buf_attach,
191 .detach = xe_dma_buf_detach,
192 .pin = xe_dma_buf_pin,
193 .unpin = xe_dma_buf_unpin,
194 .map_dma_buf = xe_dma_buf_map,
195 .unmap_dma_buf = xe_dma_buf_unmap,
196 .release = drm_gem_dmabuf_release,
197 .begin_cpu_access = xe_dma_buf_begin_cpu_access,
198 .mmap = drm_gem_dmabuf_mmap,
199 .vmap = drm_gem_dmabuf_vmap,
200 .vunmap = drm_gem_dmabuf_vunmap,
201 };
202
xe_gem_prime_export(struct drm_gem_object * obj,int flags)203 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
204 {
205 struct xe_bo *bo = gem_to_xe_bo(obj);
206 struct dma_buf *buf;
207 struct ttm_operation_ctx ctx = {
208 .interruptible = true,
209 .no_wait_gpu = true,
210 /* We opt to avoid OOM on system pages allocations */
211 .gfp_retry_mayfail = true,
212 .allow_res_evict = false,
213 };
214 int ret;
215
216 if (bo->vm)
217 return ERR_PTR(-EPERM);
218
219 ret = ttm_bo_setup_export(&bo->ttm, &ctx);
220 if (ret)
221 return ERR_PTR(ret);
222
223 buf = drm_gem_prime_export(obj, flags);
224 if (!IS_ERR(buf))
225 buf->ops = &xe_dmabuf_ops;
226
227 return buf;
228 }
229
230 static struct drm_gem_object *
xe_dma_buf_init_obj(struct drm_device * dev,struct xe_bo * storage,struct dma_buf * dma_buf)231 xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
232 struct dma_buf *dma_buf)
233 {
234 struct dma_resv *resv = dma_buf->resv;
235 struct xe_device *xe = to_xe_device(dev);
236 struct xe_validation_ctx ctx;
237 struct drm_gem_object *dummy_obj;
238 struct drm_exec exec;
239 struct xe_bo *bo;
240 int ret = 0;
241
242 dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
243 if (!dummy_obj)
244 return ERR_PTR(-ENOMEM);
245
246 dummy_obj->resv = resv;
247 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) {
248 ret = drm_exec_lock_obj(&exec, dummy_obj);
249 drm_exec_retry_on_contention(&exec);
250 if (ret)
251 break;
252
253 bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
254 0, /* Will require 1way or 2way for vm_bind */
255 ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec);
256 drm_exec_retry_on_contention(&exec);
257 if (IS_ERR(bo)) {
258 ret = PTR_ERR(bo);
259 xe_validation_retry_on_oom(&ctx, &ret);
260 break;
261 }
262 }
263 drm_gem_object_put(dummy_obj);
264
265 return ret ? ERR_PTR(ret) : &bo->ttm.base;
266 }
267
xe_dma_buf_move_notify(struct dma_buf_attachment * attach)268 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach)
269 {
270 struct drm_gem_object *obj = attach->importer_priv;
271 struct xe_bo *bo = gem_to_xe_bo(obj);
272 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
273
274 XE_WARN_ON(xe_bo_evict(bo, exec));
275 }
276
277 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = {
278 .allow_peer2peer = true,
279 .move_notify = xe_dma_buf_move_notify
280 };
281
282 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
283
284 struct dma_buf_test_params {
285 struct xe_test_priv base;
286 const struct dma_buf_attach_ops *attach_ops;
287 bool force_different_devices;
288 u32 mem_mask;
289 };
290
291 #define to_dma_buf_test_params(_priv) \
292 container_of(_priv, struct dma_buf_test_params, base)
293 #endif
294
xe_gem_prime_import(struct drm_device * dev,struct dma_buf * dma_buf)295 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
296 struct dma_buf *dma_buf)
297 {
298 XE_TEST_DECLARE(struct dma_buf_test_params *test =
299 to_dma_buf_test_params
300 (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));)
301 const struct dma_buf_attach_ops *attach_ops;
302 struct dma_buf_attachment *attach;
303 struct drm_gem_object *obj;
304 struct xe_bo *bo;
305
306 if (dma_buf->ops == &xe_dmabuf_ops) {
307 obj = dma_buf->priv;
308 if (obj->dev == dev &&
309 !XE_TEST_ONLY(test && test->force_different_devices)) {
310 /*
311 * Importing dmabuf exported from out own gem increases
312 * refcount on gem itself instead of f_count of dmabuf.
313 */
314 drm_gem_object_get(obj);
315 return obj;
316 }
317 }
318
319 /*
320 * Don't publish the bo until we have a valid attachment, and a
321 * valid attachment needs the bo address. So pre-create a bo before
322 * creating the attachment and publish.
323 */
324 bo = xe_bo_alloc();
325 if (IS_ERR(bo))
326 return ERR_CAST(bo);
327
328 attach_ops = &xe_dma_buf_attach_ops;
329 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
330 if (test)
331 attach_ops = test->attach_ops;
332 #endif
333
334 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
335 if (IS_ERR(attach)) {
336 obj = ERR_CAST(attach);
337 goto out_err;
338 }
339
340 /* Errors here will take care of freeing the bo. */
341 obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
342 if (IS_ERR(obj))
343 return obj;
344
345
346 get_dma_buf(dma_buf);
347 obj->import_attach = attach;
348 return obj;
349
350 out_err:
351 xe_bo_free(bo);
352
353 return obj;
354 }
355
356 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
357 #include "tests/xe_dma_buf.c"
358 #endif
359