1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include "xe_dma_buf.h"
7
8 #include <kunit/test.h>
9 #include <linux/dma-buf.h>
10 #include <linux/pci-p2pdma.h>
11
12 #include <drm/drm_device.h>
13 #include <drm/drm_prime.h>
14 #include <drm/ttm/ttm_tt.h>
15
16 #include "tests/xe_test.h"
17 #include "xe_bo.h"
18 #include "xe_device.h"
19 #include "xe_pm.h"
20 #include "xe_ttm_vram_mgr.h"
21 #include "xe_vm.h"
22
23 MODULE_IMPORT_NS("DMA_BUF");
24
xe_dma_buf_attach(struct dma_buf * dmabuf,struct dma_buf_attachment * attach)25 static int xe_dma_buf_attach(struct dma_buf *dmabuf,
26 struct dma_buf_attachment *attach)
27 {
28 struct drm_gem_object *obj = attach->dmabuf->priv;
29
30 if (attach->peer2peer &&
31 pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
32 attach->peer2peer = false;
33
34 if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
35 return -EOPNOTSUPP;
36
37 xe_pm_runtime_get(to_xe_device(obj->dev));
38 return 0;
39 }
40
xe_dma_buf_detach(struct dma_buf * dmabuf,struct dma_buf_attachment * attach)41 static void xe_dma_buf_detach(struct dma_buf *dmabuf,
42 struct dma_buf_attachment *attach)
43 {
44 struct drm_gem_object *obj = attach->dmabuf->priv;
45
46 xe_pm_runtime_put(to_xe_device(obj->dev));
47 }
48
xe_dma_buf_pin(struct dma_buf_attachment * attach)49 static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
50 {
51 struct dma_buf *dmabuf = attach->dmabuf;
52 struct drm_gem_object *obj = dmabuf->priv;
53 struct xe_bo *bo = gem_to_xe_bo(obj);
54 struct xe_device *xe = xe_bo_device(bo);
55 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
56 bool allow_vram = true;
57 int ret;
58
59 if (!IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY)) {
60 allow_vram = false;
61 } else {
62 list_for_each_entry(attach, &dmabuf->attachments, node) {
63 if (!attach->peer2peer) {
64 allow_vram = false;
65 break;
66 }
67 }
68 }
69
70 if (xe_bo_is_pinned(bo) && !xe_bo_is_mem_type(bo, XE_PL_TT) &&
71 !(xe_bo_is_vram(bo) && allow_vram)) {
72 drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n");
73 return -EINVAL;
74 }
75
76 if (!allow_vram) {
77 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, exec);
78 if (ret) {
79 if (ret != -EINTR && ret != -ERESTARTSYS)
80 drm_dbg(&xe->drm,
81 "Failed migrating dma-buf to TT memory: %pe\n",
82 ERR_PTR(ret));
83 return ret;
84 }
85 }
86
87 ret = xe_bo_pin_external(bo, !allow_vram, exec);
88 xe_assert(xe, !ret);
89
90 return 0;
91 }
92
xe_dma_buf_unpin(struct dma_buf_attachment * attach)93 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
94 {
95 struct drm_gem_object *obj = attach->dmabuf->priv;
96 struct xe_bo *bo = gem_to_xe_bo(obj);
97
98 xe_bo_unpin_external(bo);
99 }
100
xe_dma_buf_map(struct dma_buf_attachment * attach,enum dma_data_direction dir)101 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
102 enum dma_data_direction dir)
103 {
104 struct dma_buf *dma_buf = attach->dmabuf;
105 struct drm_gem_object *obj = dma_buf->priv;
106 struct xe_bo *bo = gem_to_xe_bo(obj);
107 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
108 struct sg_table *sgt;
109 int r = 0;
110
111 if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
112 return ERR_PTR(-EOPNOTSUPP);
113
114 if (!xe_bo_is_pinned(bo)) {
115 if (!attach->peer2peer)
116 r = xe_bo_migrate(bo, XE_PL_TT, NULL, exec);
117 else
118 r = xe_bo_validate(bo, NULL, false, exec);
119 if (r)
120 return ERR_PTR(r);
121 }
122
123 switch (bo->ttm.resource->mem_type) {
124 case XE_PL_TT:
125 sgt = drm_prime_pages_to_sg(obj->dev,
126 bo->ttm.ttm->pages,
127 obj->size >> PAGE_SHIFT);
128 if (IS_ERR(sgt))
129 return sgt;
130
131 if (dma_map_sgtable(attach->dev, sgt, dir,
132 DMA_ATTR_SKIP_CPU_SYNC))
133 goto error_free;
134 break;
135
136 case XE_PL_VRAM0:
137 case XE_PL_VRAM1:
138 r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo),
139 bo->ttm.resource, 0,
140 bo->ttm.base.size, attach->dev,
141 dir, &sgt);
142 if (r)
143 return ERR_PTR(r);
144 break;
145 default:
146 return ERR_PTR(-EINVAL);
147 }
148
149 return sgt;
150
151 error_free:
152 sg_free_table(sgt);
153 kfree(sgt);
154 return ERR_PTR(-EBUSY);
155 }
156
xe_dma_buf_unmap(struct dma_buf_attachment * attach,struct sg_table * sgt,enum dma_data_direction dir)157 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
158 struct sg_table *sgt,
159 enum dma_data_direction dir)
160 {
161 if (sg_page(sgt->sgl)) {
162 dma_unmap_sgtable(attach->dev, sgt, dir, 0);
163 sg_free_table(sgt);
164 kfree(sgt);
165 } else {
166 xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt);
167 }
168 }
169
xe_dma_buf_begin_cpu_access(struct dma_buf * dma_buf,enum dma_data_direction direction)170 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
171 enum dma_data_direction direction)
172 {
173 struct drm_gem_object *obj = dma_buf->priv;
174 struct xe_bo *bo = gem_to_xe_bo(obj);
175 bool reads = (direction == DMA_BIDIRECTIONAL ||
176 direction == DMA_FROM_DEVICE);
177 struct xe_validation_ctx ctx;
178 struct drm_exec exec;
179 int ret = 0;
180
181 if (!reads)
182 return 0;
183
184 /* Can we do interruptible lock here? */
185 xe_validation_guard(&ctx, &xe_bo_device(bo)->val, &exec, (struct xe_val_flags) {}, ret) {
186 ret = drm_exec_lock_obj(&exec, &bo->ttm.base);
187 drm_exec_retry_on_contention(&exec);
188 if (ret)
189 break;
190
191 ret = xe_bo_migrate(bo, XE_PL_TT, NULL, &exec);
192 drm_exec_retry_on_contention(&exec);
193 xe_validation_retry_on_oom(&ctx, &ret);
194 }
195
196 /* If we failed, cpu-access takes place in current placement. */
197 return 0;
198 }
199
200 static const struct dma_buf_ops xe_dmabuf_ops = {
201 .attach = xe_dma_buf_attach,
202 .detach = xe_dma_buf_detach,
203 .pin = xe_dma_buf_pin,
204 .unpin = xe_dma_buf_unpin,
205 .map_dma_buf = xe_dma_buf_map,
206 .unmap_dma_buf = xe_dma_buf_unmap,
207 .release = drm_gem_dmabuf_release,
208 .begin_cpu_access = xe_dma_buf_begin_cpu_access,
209 .mmap = drm_gem_dmabuf_mmap,
210 .vmap = drm_gem_dmabuf_vmap,
211 .vunmap = drm_gem_dmabuf_vunmap,
212 };
213
xe_gem_prime_export(struct drm_gem_object * obj,int flags)214 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
215 {
216 struct xe_bo *bo = gem_to_xe_bo(obj);
217 struct dma_buf *buf;
218 struct ttm_operation_ctx ctx = {
219 .interruptible = true,
220 .no_wait_gpu = true,
221 /* We opt to avoid OOM on system pages allocations */
222 .gfp_retry_mayfail = true,
223 .allow_res_evict = false,
224 };
225 int ret;
226
227 if (bo->vm)
228 return ERR_PTR(-EPERM);
229
230 ret = ttm_bo_setup_export(&bo->ttm, &ctx);
231 if (ret)
232 return ERR_PTR(ret);
233
234 buf = drm_gem_prime_export(obj, flags);
235 if (!IS_ERR(buf))
236 buf->ops = &xe_dmabuf_ops;
237
238 return buf;
239 }
240
241 static struct drm_gem_object *
xe_dma_buf_init_obj(struct drm_device * dev,struct xe_bo * storage,struct dma_buf * dma_buf)242 xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
243 struct dma_buf *dma_buf)
244 {
245 struct dma_resv *resv = dma_buf->resv;
246 struct xe_device *xe = to_xe_device(dev);
247 struct xe_validation_ctx ctx;
248 struct drm_gem_object *dummy_obj;
249 struct drm_exec exec;
250 struct xe_bo *bo;
251 int ret = 0;
252
253 dummy_obj = drm_gpuvm_resv_object_alloc(&xe->drm);
254 if (!dummy_obj)
255 return ERR_PTR(-ENOMEM);
256
257 dummy_obj->resv = resv;
258 xe_validation_guard(&ctx, &xe->val, &exec, (struct xe_val_flags) {}, ret) {
259 ret = drm_exec_lock_obj(&exec, dummy_obj);
260 drm_exec_retry_on_contention(&exec);
261 if (ret)
262 break;
263
264 bo = xe_bo_init_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
265 0, /* Will require 1way or 2way for vm_bind */
266 ttm_bo_type_sg, XE_BO_FLAG_SYSTEM, &exec);
267 drm_exec_retry_on_contention(&exec);
268 if (IS_ERR(bo)) {
269 ret = PTR_ERR(bo);
270 xe_validation_retry_on_oom(&ctx, &ret);
271 break;
272 }
273 }
274 drm_gem_object_put(dummy_obj);
275
276 return ret ? ERR_PTR(ret) : &bo->ttm.base;
277 }
278
xe_dma_buf_move_notify(struct dma_buf_attachment * attach)279 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach)
280 {
281 struct drm_gem_object *obj = attach->importer_priv;
282 struct xe_bo *bo = gem_to_xe_bo(obj);
283 struct drm_exec *exec = XE_VALIDATION_UNSUPPORTED;
284
285 XE_WARN_ON(xe_bo_evict(bo, exec));
286 }
287
288 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = {
289 .allow_peer2peer = true,
290 .move_notify = xe_dma_buf_move_notify
291 };
292
293 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
294
295 struct dma_buf_test_params {
296 struct xe_test_priv base;
297 const struct dma_buf_attach_ops *attach_ops;
298 bool force_different_devices;
299 u32 mem_mask;
300 };
301
302 #define to_dma_buf_test_params(_priv) \
303 container_of(_priv, struct dma_buf_test_params, base)
304 #endif
305
xe_gem_prime_import(struct drm_device * dev,struct dma_buf * dma_buf)306 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
307 struct dma_buf *dma_buf)
308 {
309 XE_TEST_DECLARE(struct dma_buf_test_params *test =
310 to_dma_buf_test_params
311 (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));)
312 const struct dma_buf_attach_ops *attach_ops;
313 struct dma_buf_attachment *attach;
314 struct drm_gem_object *obj;
315 struct xe_bo *bo;
316
317 if (dma_buf->ops == &xe_dmabuf_ops) {
318 obj = dma_buf->priv;
319 if (obj->dev == dev &&
320 !XE_TEST_ONLY(test && test->force_different_devices)) {
321 /*
322 * Importing dmabuf exported from out own gem increases
323 * refcount on gem itself instead of f_count of dmabuf.
324 */
325 drm_gem_object_get(obj);
326 return obj;
327 }
328 }
329
330 /*
331 * Don't publish the bo until we have a valid attachment, and a
332 * valid attachment needs the bo address. So pre-create a bo before
333 * creating the attachment and publish.
334 */
335 bo = xe_bo_alloc();
336 if (IS_ERR(bo))
337 return ERR_CAST(bo);
338
339 attach_ops = &xe_dma_buf_attach_ops;
340 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
341 if (test)
342 attach_ops = test->attach_ops;
343 #endif
344
345 attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
346 if (IS_ERR(attach)) {
347 obj = ERR_CAST(attach);
348 goto out_err;
349 }
350
351 /* Errors here will take care of freeing the bo. */
352 obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
353 if (IS_ERR(obj))
354 return obj;
355
356
357 get_dma_buf(dma_buf);
358 obj->import_attach = attach;
359 return obj;
360
361 out_err:
362 xe_bo_free(bo);
363
364 return obj;
365 }
366
367 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
368 #include "tests/xe_dma_buf.c"
369 #endif
370