xref: /linux/drivers/accel/ivpu/ivpu_gem.c (revision 9c815230630cb4e80e78ed3ef40909a71fc92403) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/dma-buf.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/set_memory.h>
10 #include <linux/xarray.h>
11 
12 #include <drm/drm_cache.h>
13 #include <drm/drm_debugfs.h>
14 #include <drm/drm_file.h>
15 #include <drm/drm_utils.h>
16 
17 #include "ivpu_drv.h"
18 #include "ivpu_fw.h"
19 #include "ivpu_gem.h"
20 #include "ivpu_hw.h"
21 #include "ivpu_mmu.h"
22 #include "ivpu_mmu_context.h"
23 
24 MODULE_IMPORT_NS("DMA_BUF");
25 
26 static const struct drm_gem_object_funcs ivpu_gem_funcs;
27 
28 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
29 {
30 	ivpu_dbg(vdev, BO,
31 		 "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n",
32 		 action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr,
33 		 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
34 		 (bool)drm_gem_is_imported(&bo->base.base));
35 }
36 
37 static inline int ivpu_bo_lock(struct ivpu_bo *bo)
38 {
39 	return dma_resv_lock(bo->base.base.resv, NULL);
40 }
41 
42 static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
43 {
44 	dma_resv_unlock(bo->base.base.resv);
45 }
46 
47 static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo)
48 {
49 	struct sg_table *sgt = bo->base.sgt;
50 
51 	drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach);
52 
53 	ivpu_bo_lock(bo);
54 
55 	if (!sgt) {
56 		sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL);
57 		if (IS_ERR(sgt))
58 			ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt));
59 		else
60 			bo->base.sgt = sgt;
61 	}
62 
63 	ivpu_bo_unlock(bo);
64 
65 	return sgt;
66 }
67 
68 /*
69  * ivpu_bo_bind() - pin the backing physical pages and map them to VPU.
70  *
71  * This function pins physical memory pages, then maps the physical pages
72  * to IOMMU address space and finally updates the VPU MMU page tables
73  * to allow the VPU to translate VPU address to IOMMU address.
74  */
75 int __must_check ivpu_bo_bind(struct ivpu_bo *bo)
76 {
77 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
78 	struct sg_table *sgt;
79 	int ret = 0;
80 
81 	ivpu_dbg_bo(vdev, bo, "bind");
82 
83 	if (bo->base.base.import_attach)
84 		sgt = ivpu_bo_map_attachment(vdev, bo);
85 	else
86 		sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
87 	if (IS_ERR(sgt)) {
88 		ret = PTR_ERR(sgt);
89 		ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
90 		return ret;
91 	}
92 
93 	ivpu_bo_lock(bo);
94 
95 	if (!bo->mmu_mapped) {
96 		drm_WARN_ON(&vdev->drm, !bo->ctx);
97 		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
98 					       ivpu_bo_is_snooped(bo));
99 		if (ret) {
100 			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
101 			goto unlock;
102 		}
103 		bo->mmu_mapped = true;
104 	}
105 
106 unlock:
107 	ivpu_bo_unlock(bo);
108 
109 	return ret;
110 }
111 
112 static int
113 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
114 		       const struct ivpu_addr_range *range)
115 {
116 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
117 	int idx, ret;
118 
119 	if (!drm_dev_enter(&vdev->drm, &idx))
120 		return -ENODEV;
121 
122 	ivpu_bo_lock(bo);
123 
124 	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
125 	if (!ret) {
126 		bo->ctx = ctx;
127 		bo->ctx_id = ctx->id;
128 		bo->vpu_addr = bo->mm_node.start;
129 		ivpu_dbg_bo(vdev, bo, "vaddr");
130 	} else {
131 		ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
132 	}
133 
134 	ivpu_bo_unlock(bo);
135 
136 	drm_dev_exit(idx);
137 
138 	return ret;
139 }
140 
141 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
142 {
143 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
144 
145 	dma_resv_assert_held(bo->base.base.resv);
146 
147 	if (bo->mmu_mapped) {
148 		drm_WARN_ON(&vdev->drm, !bo->ctx);
149 		drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
150 		drm_WARN_ON(&vdev->drm, !bo->base.sgt);
151 		ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
152 		bo->mmu_mapped = false;
153 	}
154 
155 	if (bo->ctx) {
156 		ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
157 		bo->ctx = NULL;
158 	}
159 
160 	if (drm_gem_is_imported(&bo->base.base))
161 		return;
162 
163 	if (bo->base.sgt) {
164 		if (bo->base.base.import_attach) {
165 			dma_buf_unmap_attachment(bo->base.base.import_attach,
166 						 bo->base.sgt, DMA_BIDIRECTIONAL);
167 		} else {
168 			dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
169 			sg_free_table(bo->base.sgt);
170 			kfree(bo->base.sgt);
171 		}
172 		bo->base.sgt = NULL;
173 	}
174 }
175 
176 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
177 {
178 	struct ivpu_bo *bo;
179 
180 	if (drm_WARN_ON(&vdev->drm, !ctx))
181 		return;
182 
183 	mutex_lock(&vdev->bo_list_lock);
184 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
185 		ivpu_bo_lock(bo);
186 		if (bo->ctx == ctx) {
187 			ivpu_dbg_bo(vdev, bo, "unbind");
188 			ivpu_bo_unbind_locked(bo);
189 		}
190 		ivpu_bo_unlock(bo);
191 	}
192 	mutex_unlock(&vdev->bo_list_lock);
193 }
194 
195 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
196 {
197 	struct ivpu_bo *bo;
198 
199 	if (size == 0 || !PAGE_ALIGNED(size))
200 		return ERR_PTR(-EINVAL);
201 
202 	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
203 	if (!bo)
204 		return ERR_PTR(-ENOMEM);
205 
206 	bo->base.base.funcs = &ivpu_gem_funcs;
207 	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
208 
209 	INIT_LIST_HEAD(&bo->bo_list_node);
210 
211 	return &bo->base.base;
212 }
213 
214 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev,
215 					     struct dma_buf *dma_buf)
216 {
217 	struct ivpu_device *vdev = to_ivpu_device(dev);
218 	struct device *attach_dev = dev->dev;
219 	struct dma_buf_attachment *attach;
220 	struct drm_gem_object *obj;
221 	struct ivpu_bo *bo;
222 	int ret;
223 
224 	attach = dma_buf_attach(dma_buf, attach_dev);
225 	if (IS_ERR(attach))
226 		return ERR_CAST(attach);
227 
228 	get_dma_buf(dma_buf);
229 
230 	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL);
231 	if (IS_ERR(obj)) {
232 		ret = PTR_ERR(obj);
233 		goto fail_detach;
234 	}
235 
236 	obj->import_attach = attach;
237 	obj->resv = dma_buf->resv;
238 
239 	bo = to_ivpu_bo(obj);
240 
241 	mutex_lock(&vdev->bo_list_lock);
242 	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
243 	mutex_unlock(&vdev->bo_list_lock);
244 
245 	ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo));
246 
247 	return obj;
248 
249 fail_detach:
250 	dma_buf_detach(dma_buf, attach);
251 	dma_buf_put(dma_buf);
252 
253 	return ERR_PTR(ret);
254 }
255 
256 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)
257 {
258 	struct drm_gem_shmem_object *shmem;
259 	struct ivpu_bo *bo;
260 
261 	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
262 	case DRM_IVPU_BO_CACHED:
263 	case DRM_IVPU_BO_WC:
264 		break;
265 	default:
266 		return ERR_PTR(-EINVAL);
267 	}
268 
269 	shmem = drm_gem_shmem_create(&vdev->drm, size);
270 	if (IS_ERR(shmem))
271 		return ERR_CAST(shmem);
272 
273 	bo = to_ivpu_bo(&shmem->base);
274 	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
275 	bo->flags = flags;
276 
277 	mutex_lock(&vdev->bo_list_lock);
278 	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
279 	mutex_unlock(&vdev->bo_list_lock);
280 
281 	ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size);
282 
283 	return bo;
284 }
285 
286 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
287 {
288 	struct ivpu_file_priv *file_priv = file->driver_priv;
289 	struct ivpu_device *vdev = file_priv->vdev;
290 	struct ivpu_bo *bo = to_ivpu_bo(obj);
291 	struct ivpu_addr_range *range;
292 
293 	if (bo->ctx) {
294 		ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
295 			  file_priv->ctx.id, bo->ctx->id);
296 		return -EALREADY;
297 	}
298 
299 	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
300 		range = &vdev->hw->ranges.shave;
301 	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
302 		range = &vdev->hw->ranges.dma;
303 	else
304 		range = &vdev->hw->ranges.user;
305 
306 	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
307 }
308 
309 static void ivpu_gem_bo_free(struct drm_gem_object *obj)
310 {
311 	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
312 	struct ivpu_bo *bo = to_ivpu_bo(obj);
313 
314 	ivpu_dbg_bo(vdev, bo, "free");
315 
316 	drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node));
317 
318 	mutex_lock(&vdev->bo_list_lock);
319 	list_del(&bo->bo_list_node);
320 	mutex_unlock(&vdev->bo_list_lock);
321 
322 	drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
323 		    !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
324 	drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
325 	drm_WARN_ON(&vdev->drm, bo->base.vaddr);
326 
327 	ivpu_bo_lock(bo);
328 	ivpu_bo_unbind_locked(bo);
329 	ivpu_bo_unlock(bo);
330 
331 	drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
332 	drm_WARN_ON(&vdev->drm, bo->ctx);
333 
334 	drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1);
335 	drm_WARN_ON(obj->dev, bo->base.base.vma_node.vm_files.rb_node);
336 	drm_gem_shmem_free(&bo->base);
337 }
338 
339 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
340 	.free = ivpu_gem_bo_free,
341 	.open = ivpu_gem_bo_open,
342 	.print_info = drm_gem_shmem_object_print_info,
343 	.pin = drm_gem_shmem_object_pin,
344 	.unpin = drm_gem_shmem_object_unpin,
345 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
346 	.vmap = drm_gem_shmem_object_vmap,
347 	.vunmap = drm_gem_shmem_object_vunmap,
348 	.mmap = drm_gem_shmem_object_mmap,
349 	.vm_ops = &drm_gem_shmem_vm_ops,
350 };
351 
352 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
353 {
354 	struct ivpu_file_priv *file_priv = file->driver_priv;
355 	struct ivpu_device *vdev = file_priv->vdev;
356 	struct drm_ivpu_bo_create *args = data;
357 	u64 size = PAGE_ALIGN(args->size);
358 	struct ivpu_bo *bo;
359 	int ret;
360 
361 	if (args->flags & ~DRM_IVPU_BO_FLAGS)
362 		return -EINVAL;
363 
364 	if (size == 0)
365 		return -EINVAL;
366 
367 	bo = ivpu_bo_alloc(vdev, size, args->flags);
368 	if (IS_ERR(bo)) {
369 		ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
370 			 bo, file_priv->ctx.id, args->size, args->flags);
371 		return PTR_ERR(bo);
372 	}
373 
374 	drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 0);
375 
376 	ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
377 	if (ret) {
378 		ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)",
379 			 bo, file_priv->ctx.id, args->size, args->flags);
380 	} else {
381 		args->vpu_addr = bo->vpu_addr;
382 		drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 1);
383 	}
384 
385 	drm_gem_object_put(&bo->base.base);
386 
387 	return ret;
388 }
389 
390 struct ivpu_bo *
391 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
392 	       struct ivpu_addr_range *range, u64 size, u32 flags)
393 {
394 	struct iosys_map map;
395 	struct ivpu_bo *bo;
396 	int ret;
397 
398 	if (drm_WARN_ON(&vdev->drm, !range))
399 		return NULL;
400 
401 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
402 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
403 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
404 
405 	bo = ivpu_bo_alloc(vdev, size, flags);
406 	if (IS_ERR(bo)) {
407 		ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
408 			 bo, range->start, size, flags);
409 		return NULL;
410 	}
411 
412 	ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
413 	if (ret)
414 		goto err_put;
415 
416 	ret = ivpu_bo_bind(bo);
417 	if (ret)
418 		goto err_put;
419 
420 	if (flags & DRM_IVPU_BO_MAPPABLE) {
421 		ivpu_bo_lock(bo);
422 		ret = drm_gem_shmem_vmap_locked(&bo->base, &map);
423 		ivpu_bo_unlock(bo);
424 
425 		if (ret)
426 			goto err_put;
427 	}
428 
429 	return bo;
430 
431 err_put:
432 	drm_gem_object_put(&bo->base.base);
433 	return NULL;
434 }
435 
436 struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags)
437 {
438 	struct ivpu_addr_range range;
439 
440 	if (!ivpu_is_within_range(addr, size, &vdev->hw->ranges.runtime)) {
441 		ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size);
442 		return NULL;
443 	}
444 
445 	if (ivpu_hw_range_init(vdev, &range, addr, size))
446 		return NULL;
447 
448 	return ivpu_bo_create(vdev, &vdev->gctx, &range, size, flags);
449 }
450 
451 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
452 {
453 	return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
454 }
455 
456 void ivpu_bo_free(struct ivpu_bo *bo)
457 {
458 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
459 
460 	if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
461 		ivpu_bo_lock(bo);
462 		drm_gem_shmem_vunmap_locked(&bo->base, &map);
463 		ivpu_bo_unlock(bo);
464 	}
465 
466 	drm_gem_object_put(&bo->base.base);
467 }
468 
469 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
470 {
471 	struct drm_ivpu_bo_info *args = data;
472 	struct drm_gem_object *obj;
473 	struct ivpu_bo *bo;
474 	int ret = 0;
475 
476 	obj = drm_gem_object_lookup(file, args->handle);
477 	if (!obj)
478 		return -ENOENT;
479 
480 	bo = to_ivpu_bo(obj);
481 
482 	ivpu_bo_lock(bo);
483 	args->flags = bo->flags;
484 	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
485 	args->vpu_addr = bo->vpu_addr;
486 	args->size = obj->size;
487 	ivpu_bo_unlock(bo);
488 
489 	drm_gem_object_put(obj);
490 	return ret;
491 }
492 
493 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
494 {
495 	struct drm_ivpu_bo_wait *args = data;
496 	struct drm_gem_object *obj;
497 	unsigned long timeout;
498 	long ret;
499 
500 	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
501 
502 	/* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
503 	timeout += 1;
504 
505 	obj = drm_gem_object_lookup(file, args->handle);
506 	if (!obj)
507 		return -EINVAL;
508 
509 	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
510 	if (ret == 0) {
511 		ret = -ETIMEDOUT;
512 	} else if (ret > 0) {
513 		ret = 0;
514 		args->job_status = to_ivpu_bo(obj)->job_status;
515 	}
516 
517 	drm_gem_object_put(obj);
518 
519 	return ret;
520 }
521 
522 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
523 {
524 	ivpu_bo_lock(bo);
525 
526 	drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
527 		   bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
528 		   bo->flags, kref_read(&bo->base.base.refcount));
529 
530 	if (bo->base.pages)
531 		drm_printf(p, " has_pages");
532 
533 	if (bo->mmu_mapped)
534 		drm_printf(p, " mmu_mapped");
535 
536 	if (drm_gem_is_imported(&bo->base.base))
537 		drm_printf(p, " imported");
538 
539 	drm_printf(p, "\n");
540 
541 	ivpu_bo_unlock(bo);
542 }
543 
544 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
545 {
546 	struct ivpu_device *vdev = to_ivpu_device(dev);
547 	struct ivpu_bo *bo;
548 
549 	drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
550 		   "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
551 
552 	mutex_lock(&vdev->bo_list_lock);
553 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
554 		ivpu_bo_print_info(bo, p);
555 	mutex_unlock(&vdev->bo_list_lock);
556 }
557 
558 void ivpu_bo_list_print(struct drm_device *dev)
559 {
560 	struct drm_printer p = drm_info_printer(dev->dev);
561 
562 	ivpu_bo_list(dev, &p);
563 }
564