1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6 #include <linux/dma-buf.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/set_memory.h>
10 #include <linux/xarray.h>
11
12 #include <drm/drm_cache.h>
13 #include <drm/drm_debugfs.h>
14 #include <drm/drm_file.h>
15 #include <drm/drm_utils.h>
16
17 #include "ivpu_drv.h"
18 #include "ivpu_gem.h"
19 #include "ivpu_hw.h"
20 #include "ivpu_mmu.h"
21 #include "ivpu_mmu_context.h"
22
23 MODULE_IMPORT_NS("DMA_BUF");
24
25 static const struct drm_gem_object_funcs ivpu_gem_funcs;
26
ivpu_dbg_bo(struct ivpu_device * vdev,struct ivpu_bo * bo,const char * action)27 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
28 {
29 ivpu_dbg(vdev, BO,
30 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",
31 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx_id,
32 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
33 (bool)drm_gem_is_imported(&bo->base.base));
34 }
35
ivpu_bo_lock(struct ivpu_bo * bo)36 static inline int ivpu_bo_lock(struct ivpu_bo *bo)
37 {
38 return dma_resv_lock(bo->base.base.resv, NULL);
39 }
40
ivpu_bo_unlock(struct ivpu_bo * bo)41 static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
42 {
43 dma_resv_unlock(bo->base.base.resv);
44 }
45
46 /*
47 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
48 *
49 * This function pins physical memory pages, then maps the physical pages
50 * to IOMMU address space and finally updates the VPU MMU page tables
51 * to allow the VPU to translate VPU address to IOMMU address.
52 */
ivpu_bo_pin(struct ivpu_bo * bo)53 int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
54 {
55 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
56 struct sg_table *sgt;
57 int ret = 0;
58
59 ivpu_dbg_bo(vdev, bo, "pin");
60
61 sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
62 if (IS_ERR(sgt)) {
63 ret = PTR_ERR(sgt);
64 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
65 return ret;
66 }
67
68 ivpu_bo_lock(bo);
69
70 if (!bo->mmu_mapped) {
71 drm_WARN_ON(&vdev->drm, !bo->ctx);
72 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
73 ivpu_bo_is_snooped(bo));
74 if (ret) {
75 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
76 goto unlock;
77 }
78 bo->mmu_mapped = true;
79 }
80
81 unlock:
82 ivpu_bo_unlock(bo);
83
84 return ret;
85 }
86
87 static int
ivpu_bo_alloc_vpu_addr(struct ivpu_bo * bo,struct ivpu_mmu_context * ctx,const struct ivpu_addr_range * range)88 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
89 const struct ivpu_addr_range *range)
90 {
91 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
92 int idx, ret;
93
94 if (!drm_dev_enter(&vdev->drm, &idx))
95 return -ENODEV;
96
97 ivpu_bo_lock(bo);
98
99 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
100 if (!ret) {
101 bo->ctx = ctx;
102 bo->vpu_addr = bo->mm_node.start;
103 } else {
104 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
105 }
106
107 ivpu_bo_unlock(bo);
108
109 drm_dev_exit(idx);
110
111 return ret;
112 }
113
ivpu_bo_unbind_locked(struct ivpu_bo * bo)114 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
115 {
116 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
117
118 lockdep_assert(dma_resv_held(bo->base.base.resv) || !kref_read(&bo->base.base.refcount));
119
120 if (bo->mmu_mapped) {
121 drm_WARN_ON(&vdev->drm, !bo->ctx);
122 drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
123 drm_WARN_ON(&vdev->drm, !bo->base.sgt);
124 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
125 bo->mmu_mapped = false;
126 }
127
128 if (bo->ctx) {
129 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
130 bo->ctx = NULL;
131 }
132
133 if (drm_gem_is_imported(&bo->base.base))
134 return;
135
136 if (bo->base.sgt) {
137 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
138 sg_free_table(bo->base.sgt);
139 kfree(bo->base.sgt);
140 bo->base.sgt = NULL;
141 }
142 }
143
ivpu_bo_unbind_all_bos_from_context(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx)144 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
145 {
146 struct ivpu_bo *bo;
147
148 if (drm_WARN_ON(&vdev->drm, !ctx))
149 return;
150
151 mutex_lock(&vdev->bo_list_lock);
152 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
153 ivpu_bo_lock(bo);
154 if (bo->ctx == ctx) {
155 ivpu_dbg_bo(vdev, bo, "unbind");
156 ivpu_bo_unbind_locked(bo);
157 }
158 ivpu_bo_unlock(bo);
159 }
160 mutex_unlock(&vdev->bo_list_lock);
161 }
162
ivpu_gem_create_object(struct drm_device * dev,size_t size)163 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
164 {
165 struct ivpu_bo *bo;
166
167 if (size == 0 || !PAGE_ALIGNED(size))
168 return ERR_PTR(-EINVAL);
169
170 bo = kzalloc(sizeof(*bo), GFP_KERNEL);
171 if (!bo)
172 return ERR_PTR(-ENOMEM);
173
174 bo->base.base.funcs = &ivpu_gem_funcs;
175 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
176
177 INIT_LIST_HEAD(&bo->bo_list_node);
178
179 return &bo->base.base;
180 }
181
ivpu_gem_prime_import(struct drm_device * dev,struct dma_buf * dma_buf)182 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev,
183 struct dma_buf *dma_buf)
184 {
185 struct device *attach_dev = dev->dev;
186 struct dma_buf_attachment *attach;
187 struct sg_table *sgt;
188 struct drm_gem_object *obj;
189 int ret;
190
191 attach = dma_buf_attach(dma_buf, attach_dev);
192 if (IS_ERR(attach))
193 return ERR_CAST(attach);
194
195 get_dma_buf(dma_buf);
196
197 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
198 if (IS_ERR(sgt)) {
199 ret = PTR_ERR(sgt);
200 goto fail_detach;
201 }
202
203 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
204 if (IS_ERR(obj)) {
205 ret = PTR_ERR(obj);
206 goto fail_unmap;
207 }
208
209 obj->import_attach = attach;
210 obj->resv = dma_buf->resv;
211
212 return obj;
213
214 fail_unmap:
215 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
216 fail_detach:
217 dma_buf_detach(dma_buf, attach);
218 dma_buf_put(dma_buf);
219
220 return ERR_PTR(ret);
221 }
222
ivpu_bo_alloc(struct ivpu_device * vdev,u64 size,u32 flags,u32 ctx_id)223 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags, u32 ctx_id)
224 {
225 struct drm_gem_shmem_object *shmem;
226 struct ivpu_bo *bo;
227
228 switch (flags & DRM_IVPU_BO_CACHE_MASK) {
229 case DRM_IVPU_BO_CACHED:
230 case DRM_IVPU_BO_WC:
231 break;
232 default:
233 return ERR_PTR(-EINVAL);
234 }
235
236 shmem = drm_gem_shmem_create(&vdev->drm, size);
237 if (IS_ERR(shmem))
238 return ERR_CAST(shmem);
239
240 bo = to_ivpu_bo(&shmem->base);
241 bo->ctx_id = ctx_id;
242 bo->base.map_wc = flags & DRM_IVPU_BO_WC;
243 bo->flags = flags;
244
245 mutex_lock(&vdev->bo_list_lock);
246 list_add_tail(&bo->bo_list_node, &vdev->bo_list);
247 mutex_unlock(&vdev->bo_list_lock);
248
249 ivpu_dbg_bo(vdev, bo, "alloc");
250
251 return bo;
252 }
253
ivpu_gem_bo_open(struct drm_gem_object * obj,struct drm_file * file)254 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
255 {
256 struct ivpu_file_priv *file_priv = file->driver_priv;
257 struct ivpu_device *vdev = file_priv->vdev;
258 struct ivpu_bo *bo = to_ivpu_bo(obj);
259 struct ivpu_addr_range *range;
260
261 if (bo->ctx) {
262 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
263 file_priv->ctx.id, bo->ctx->id);
264 return -EALREADY;
265 }
266
267 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
268 range = &vdev->hw->ranges.shave;
269 else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
270 range = &vdev->hw->ranges.dma;
271 else
272 range = &vdev->hw->ranges.user;
273
274 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
275 }
276
ivpu_gem_bo_free(struct drm_gem_object * obj)277 static void ivpu_gem_bo_free(struct drm_gem_object *obj)
278 {
279 struct ivpu_device *vdev = to_ivpu_device(obj->dev);
280 struct ivpu_bo *bo = to_ivpu_bo(obj);
281
282 ivpu_dbg_bo(vdev, bo, "free");
283
284 mutex_lock(&vdev->bo_list_lock);
285 list_del(&bo->bo_list_node);
286 mutex_unlock(&vdev->bo_list_lock);
287
288 drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
289 !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
290 drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
291 drm_WARN_ON(&vdev->drm, bo->base.vaddr);
292
293 ivpu_bo_unbind_locked(bo);
294 drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
295 drm_WARN_ON(&vdev->drm, bo->ctx);
296
297 drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1);
298 drm_gem_shmem_free(&bo->base);
299 }
300
301 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
302 .free = ivpu_gem_bo_free,
303 .open = ivpu_gem_bo_open,
304 .print_info = drm_gem_shmem_object_print_info,
305 .pin = drm_gem_shmem_object_pin,
306 .unpin = drm_gem_shmem_object_unpin,
307 .get_sg_table = drm_gem_shmem_object_get_sg_table,
308 .vmap = drm_gem_shmem_object_vmap,
309 .vunmap = drm_gem_shmem_object_vunmap,
310 .mmap = drm_gem_shmem_object_mmap,
311 .vm_ops = &drm_gem_shmem_vm_ops,
312 };
313
ivpu_bo_create_ioctl(struct drm_device * dev,void * data,struct drm_file * file)314 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
315 {
316 struct ivpu_file_priv *file_priv = file->driver_priv;
317 struct ivpu_device *vdev = file_priv->vdev;
318 struct drm_ivpu_bo_create *args = data;
319 u64 size = PAGE_ALIGN(args->size);
320 struct ivpu_bo *bo;
321 int ret;
322
323 if (args->flags & ~DRM_IVPU_BO_FLAGS)
324 return -EINVAL;
325
326 if (size == 0)
327 return -EINVAL;
328
329 bo = ivpu_bo_alloc(vdev, size, args->flags, file_priv->ctx.id);
330 if (IS_ERR(bo)) {
331 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
332 bo, file_priv->ctx.id, args->size, args->flags);
333 return PTR_ERR(bo);
334 }
335
336 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
337 if (ret)
338 ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)",
339 bo, file_priv->ctx.id, args->size, args->flags);
340 else
341 args->vpu_addr = bo->vpu_addr;
342
343 drm_gem_object_put(&bo->base.base);
344
345 return ret;
346 }
347
348 struct ivpu_bo *
ivpu_bo_create(struct ivpu_device * vdev,struct ivpu_mmu_context * ctx,struct ivpu_addr_range * range,u64 size,u32 flags)349 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
350 struct ivpu_addr_range *range, u64 size, u32 flags)
351 {
352 struct iosys_map map;
353 struct ivpu_bo *bo;
354 int ret;
355
356 if (drm_WARN_ON(&vdev->drm, !range))
357 return NULL;
358
359 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
360 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
361 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
362
363 bo = ivpu_bo_alloc(vdev, size, flags, IVPU_GLOBAL_CONTEXT_MMU_SSID);
364 if (IS_ERR(bo)) {
365 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
366 bo, range->start, size, flags);
367 return NULL;
368 }
369
370 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
371 if (ret)
372 goto err_put;
373
374 ret = ivpu_bo_pin(bo);
375 if (ret)
376 goto err_put;
377
378 if (flags & DRM_IVPU_BO_MAPPABLE) {
379 ivpu_bo_lock(bo);
380 ret = drm_gem_shmem_vmap_locked(&bo->base, &map);
381 ivpu_bo_unlock(bo);
382
383 if (ret)
384 goto err_put;
385 }
386
387 return bo;
388
389 err_put:
390 drm_gem_object_put(&bo->base.base);
391 return NULL;
392 }
393
ivpu_bo_create_global(struct ivpu_device * vdev,u64 size,u32 flags)394 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
395 {
396 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
397 }
398
ivpu_bo_free(struct ivpu_bo * bo)399 void ivpu_bo_free(struct ivpu_bo *bo)
400 {
401 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
402
403 if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
404 ivpu_bo_lock(bo);
405 drm_gem_shmem_vunmap_locked(&bo->base, &map);
406 ivpu_bo_unlock(bo);
407 }
408
409 drm_gem_object_put(&bo->base.base);
410 }
411
ivpu_bo_info_ioctl(struct drm_device * dev,void * data,struct drm_file * file)412 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
413 {
414 struct drm_ivpu_bo_info *args = data;
415 struct drm_gem_object *obj;
416 struct ivpu_bo *bo;
417 int ret = 0;
418
419 obj = drm_gem_object_lookup(file, args->handle);
420 if (!obj)
421 return -ENOENT;
422
423 bo = to_ivpu_bo(obj);
424
425 ivpu_bo_lock(bo);
426 args->flags = bo->flags;
427 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
428 args->vpu_addr = bo->vpu_addr;
429 args->size = obj->size;
430 ivpu_bo_unlock(bo);
431
432 drm_gem_object_put(obj);
433 return ret;
434 }
435
ivpu_bo_wait_ioctl(struct drm_device * dev,void * data,struct drm_file * file)436 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
437 {
438 struct drm_ivpu_bo_wait *args = data;
439 struct drm_gem_object *obj;
440 unsigned long timeout;
441 long ret;
442
443 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
444
445 /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
446 timeout += 1;
447
448 obj = drm_gem_object_lookup(file, args->handle);
449 if (!obj)
450 return -EINVAL;
451
452 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
453 if (ret == 0) {
454 ret = -ETIMEDOUT;
455 } else if (ret > 0) {
456 ret = 0;
457 args->job_status = to_ivpu_bo(obj)->job_status;
458 }
459
460 drm_gem_object_put(obj);
461
462 return ret;
463 }
464
ivpu_bo_print_info(struct ivpu_bo * bo,struct drm_printer * p)465 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
466 {
467 ivpu_bo_lock(bo);
468
469 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
470 bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
471 bo->flags, kref_read(&bo->base.base.refcount));
472
473 if (bo->base.pages)
474 drm_printf(p, " has_pages");
475
476 if (bo->mmu_mapped)
477 drm_printf(p, " mmu_mapped");
478
479 if (drm_gem_is_imported(&bo->base.base))
480 drm_printf(p, " imported");
481
482 drm_printf(p, "\n");
483
484 ivpu_bo_unlock(bo);
485 }
486
ivpu_bo_list(struct drm_device * dev,struct drm_printer * p)487 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
488 {
489 struct ivpu_device *vdev = to_ivpu_device(dev);
490 struct ivpu_bo *bo;
491
492 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
493 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
494
495 mutex_lock(&vdev->bo_list_lock);
496 list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
497 ivpu_bo_print_info(bo, p);
498 mutex_unlock(&vdev->bo_list_lock);
499 }
500
ivpu_bo_list_print(struct drm_device * dev)501 void ivpu_bo_list_print(struct drm_device *dev)
502 {
503 struct drm_printer p = drm_info_printer(dev->dev);
504
505 ivpu_bo_list(dev, &p);
506 }
507