xref: /linux/drivers/accel/ivpu/ivpu_gem.c (revision face6a3615a649456eb4549f6d474221d877d604) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/dma-buf.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/set_memory.h>
10 #include <linux/xarray.h>
11 
12 #include <drm/drm_cache.h>
13 #include <drm/drm_debugfs.h>
14 #include <drm/drm_file.h>
15 #include <drm/drm_utils.h>
16 
17 #include "ivpu_drv.h"
18 #include "ivpu_fw.h"
19 #include "ivpu_gem.h"
20 #include "ivpu_hw.h"
21 #include "ivpu_mmu.h"
22 #include "ivpu_mmu_context.h"
23 
24 MODULE_IMPORT_NS("DMA_BUF");
25 
26 static const struct drm_gem_object_funcs ivpu_gem_funcs;
27 
28 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
29 {
30 	ivpu_dbg(vdev, BO,
31 		 "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n",
32 		 action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr,
33 		 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
34 		 (bool)drm_gem_is_imported(&bo->base.base));
35 }
36 
37 static inline int ivpu_bo_lock(struct ivpu_bo *bo)
38 {
39 	return dma_resv_lock(bo->base.base.resv, NULL);
40 }
41 
42 static inline void ivpu_bo_unlock(struct ivpu_bo *bo)
43 {
44 	dma_resv_unlock(bo->base.base.resv);
45 }
46 
47 static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo)
48 {
49 	struct sg_table *sgt;
50 
51 	drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach);
52 
53 	ivpu_bo_lock(bo);
54 
55 	sgt = bo->base.sgt;
56 	if (!sgt) {
57 		sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL);
58 		if (IS_ERR(sgt))
59 			ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt));
60 		else
61 			bo->base.sgt = sgt;
62 	}
63 
64 	ivpu_bo_unlock(bo);
65 
66 	return sgt;
67 }
68 
69 /*
70  * ivpu_bo_bind() - pin the backing physical pages and map them to VPU.
71  *
72  * This function pins physical memory pages, then maps the physical pages
73  * to IOMMU address space and finally updates the VPU MMU page tables
74  * to allow the VPU to translate VPU address to IOMMU address.
75  */
76 int __must_check ivpu_bo_bind(struct ivpu_bo *bo)
77 {
78 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
79 	struct sg_table *sgt;
80 	int ret = 0;
81 
82 	ivpu_dbg_bo(vdev, bo, "bind");
83 
84 	if (bo->base.base.import_attach)
85 		sgt = ivpu_bo_map_attachment(vdev, bo);
86 	else
87 		sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
88 	if (IS_ERR(sgt)) {
89 		ret = PTR_ERR(sgt);
90 		ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
91 		return ret;
92 	}
93 
94 	ivpu_bo_lock(bo);
95 
96 	if (!bo->mmu_mapped) {
97 		drm_WARN_ON(&vdev->drm, !bo->ctx);
98 		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
99 					       ivpu_bo_is_snooped(bo));
100 		if (ret) {
101 			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
102 			goto unlock;
103 		}
104 		bo->mmu_mapped = true;
105 	}
106 
107 unlock:
108 	ivpu_bo_unlock(bo);
109 
110 	return ret;
111 }
112 
113 static int
114 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
115 		       const struct ivpu_addr_range *range)
116 {
117 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
118 	int idx, ret;
119 
120 	if (!drm_dev_enter(&vdev->drm, &idx))
121 		return -ENODEV;
122 
123 	ivpu_bo_lock(bo);
124 
125 	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
126 	if (!ret) {
127 		bo->ctx = ctx;
128 		bo->ctx_id = ctx->id;
129 		bo->vpu_addr = bo->mm_node.start;
130 		ivpu_dbg_bo(vdev, bo, "vaddr");
131 	} else {
132 		ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
133 	}
134 
135 	ivpu_bo_unlock(bo);
136 
137 	drm_dev_exit(idx);
138 
139 	return ret;
140 }
141 
142 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
143 {
144 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
145 
146 	dma_resv_assert_held(bo->base.base.resv);
147 
148 	if (bo->mmu_mapped) {
149 		drm_WARN_ON(&vdev->drm, !bo->ctx);
150 		drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
151 		drm_WARN_ON(&vdev->drm, !bo->base.sgt);
152 		ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
153 		bo->mmu_mapped = false;
154 	}
155 
156 	if (bo->ctx) {
157 		ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
158 		bo->ctx = NULL;
159 	}
160 
161 	if (drm_gem_is_imported(&bo->base.base))
162 		return;
163 
164 	if (bo->base.sgt) {
165 		if (bo->base.base.import_attach) {
166 			dma_buf_unmap_attachment(bo->base.base.import_attach,
167 						 bo->base.sgt, DMA_BIDIRECTIONAL);
168 		} else {
169 			dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
170 			sg_free_table(bo->base.sgt);
171 			kfree(bo->base.sgt);
172 		}
173 		bo->base.sgt = NULL;
174 	}
175 }
176 
177 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
178 {
179 	struct ivpu_bo *bo;
180 
181 	if (drm_WARN_ON(&vdev->drm, !ctx))
182 		return;
183 
184 	mutex_lock(&vdev->bo_list_lock);
185 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
186 		ivpu_bo_lock(bo);
187 		if (bo->ctx == ctx) {
188 			ivpu_dbg_bo(vdev, bo, "unbind");
189 			ivpu_bo_unbind_locked(bo);
190 		}
191 		ivpu_bo_unlock(bo);
192 	}
193 	mutex_unlock(&vdev->bo_list_lock);
194 }
195 
196 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
197 {
198 	struct ivpu_bo *bo;
199 
200 	if (size == 0 || !PAGE_ALIGNED(size))
201 		return ERR_PTR(-EINVAL);
202 
203 	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
204 	if (!bo)
205 		return ERR_PTR(-ENOMEM);
206 
207 	bo->base.base.funcs = &ivpu_gem_funcs;
208 	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
209 
210 	INIT_LIST_HEAD(&bo->bo_list_node);
211 
212 	return &bo->base.base;
213 }
214 
215 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev,
216 					     struct dma_buf *dma_buf)
217 {
218 	struct ivpu_device *vdev = to_ivpu_device(dev);
219 	struct device *attach_dev = dev->dev;
220 	struct dma_buf_attachment *attach;
221 	struct drm_gem_object *obj;
222 	struct ivpu_bo *bo;
223 	int ret;
224 
225 	attach = dma_buf_attach(dma_buf, attach_dev);
226 	if (IS_ERR(attach))
227 		return ERR_CAST(attach);
228 
229 	get_dma_buf(dma_buf);
230 
231 	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL);
232 	if (IS_ERR(obj)) {
233 		ret = PTR_ERR(obj);
234 		goto fail_detach;
235 	}
236 
237 	obj->import_attach = attach;
238 	obj->resv = dma_buf->resv;
239 
240 	bo = to_ivpu_bo(obj);
241 
242 	mutex_lock(&vdev->bo_list_lock);
243 	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
244 	mutex_unlock(&vdev->bo_list_lock);
245 
246 	ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo));
247 
248 	return obj;
249 
250 fail_detach:
251 	dma_buf_detach(dma_buf, attach);
252 	dma_buf_put(dma_buf);
253 
254 	return ERR_PTR(ret);
255 }
256 
257 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)
258 {
259 	struct drm_gem_shmem_object *shmem;
260 	struct ivpu_bo *bo;
261 
262 	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
263 	case DRM_IVPU_BO_CACHED:
264 	case DRM_IVPU_BO_WC:
265 		break;
266 	default:
267 		return ERR_PTR(-EINVAL);
268 	}
269 
270 	shmem = drm_gem_shmem_create(&vdev->drm, size);
271 	if (IS_ERR(shmem))
272 		return ERR_CAST(shmem);
273 
274 	bo = to_ivpu_bo(&shmem->base);
275 	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
276 	bo->flags = flags;
277 
278 	mutex_lock(&vdev->bo_list_lock);
279 	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
280 	mutex_unlock(&vdev->bo_list_lock);
281 
282 	ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size);
283 
284 	return bo;
285 }
286 
287 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
288 {
289 	struct ivpu_file_priv *file_priv = file->driver_priv;
290 	struct ivpu_device *vdev = file_priv->vdev;
291 	struct ivpu_bo *bo = to_ivpu_bo(obj);
292 	struct ivpu_addr_range *range;
293 
294 	if (bo->ctx) {
295 		ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
296 			  file_priv->ctx.id, bo->ctx->id);
297 		return -EALREADY;
298 	}
299 
300 	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
301 		range = &vdev->hw->ranges.shave;
302 	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
303 		range = &vdev->hw->ranges.dma;
304 	else
305 		range = &vdev->hw->ranges.user;
306 
307 	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
308 }
309 
310 static void ivpu_gem_bo_free(struct drm_gem_object *obj)
311 {
312 	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
313 	struct ivpu_bo *bo = to_ivpu_bo(obj);
314 
315 	ivpu_dbg_bo(vdev, bo, "free");
316 
317 	drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node));
318 
319 	mutex_lock(&vdev->bo_list_lock);
320 	list_del(&bo->bo_list_node);
321 	mutex_unlock(&vdev->bo_list_lock);
322 
323 	drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) &&
324 		    !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
325 	drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0);
326 	drm_WARN_ON(&vdev->drm, bo->base.vaddr);
327 
328 	ivpu_bo_lock(bo);
329 	ivpu_bo_unbind_locked(bo);
330 	ivpu_bo_unlock(bo);
331 
332 	drm_WARN_ON(&vdev->drm, bo->mmu_mapped);
333 	drm_WARN_ON(&vdev->drm, bo->ctx);
334 
335 	drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1);
336 	drm_WARN_ON(obj->dev, bo->base.base.vma_node.vm_files.rb_node);
337 	drm_gem_shmem_free(&bo->base);
338 }
339 
340 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
341 	.free = ivpu_gem_bo_free,
342 	.open = ivpu_gem_bo_open,
343 	.print_info = drm_gem_shmem_object_print_info,
344 	.pin = drm_gem_shmem_object_pin,
345 	.unpin = drm_gem_shmem_object_unpin,
346 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
347 	.vmap = drm_gem_shmem_object_vmap,
348 	.vunmap = drm_gem_shmem_object_vunmap,
349 	.mmap = drm_gem_shmem_object_mmap,
350 	.vm_ops = &drm_gem_shmem_vm_ops,
351 };
352 
353 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
354 {
355 	struct ivpu_file_priv *file_priv = file->driver_priv;
356 	struct ivpu_device *vdev = file_priv->vdev;
357 	struct drm_ivpu_bo_create *args = data;
358 	u64 size = PAGE_ALIGN(args->size);
359 	struct ivpu_bo *bo;
360 	int ret;
361 
362 	if (args->flags & ~DRM_IVPU_BO_FLAGS)
363 		return -EINVAL;
364 
365 	if (size == 0)
366 		return -EINVAL;
367 
368 	bo = ivpu_bo_alloc(vdev, size, args->flags);
369 	if (IS_ERR(bo)) {
370 		ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
371 			 bo, file_priv->ctx.id, args->size, args->flags);
372 		return PTR_ERR(bo);
373 	}
374 
375 	drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 0);
376 
377 	ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
378 	if (ret) {
379 		ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)",
380 			 bo, file_priv->ctx.id, args->size, args->flags);
381 	} else {
382 		args->vpu_addr = bo->vpu_addr;
383 		drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 1);
384 	}
385 
386 	drm_gem_object_put(&bo->base.base);
387 
388 	return ret;
389 }
390 
391 struct ivpu_bo *
392 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
393 	       struct ivpu_addr_range *range, u64 size, u32 flags)
394 {
395 	struct iosys_map map;
396 	struct ivpu_bo *bo;
397 	int ret;
398 
399 	if (drm_WARN_ON(&vdev->drm, !range))
400 		return NULL;
401 
402 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
403 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
404 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
405 
406 	bo = ivpu_bo_alloc(vdev, size, flags);
407 	if (IS_ERR(bo)) {
408 		ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
409 			 bo, range->start, size, flags);
410 		return NULL;
411 	}
412 
413 	ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
414 	if (ret)
415 		goto err_put;
416 
417 	ret = ivpu_bo_bind(bo);
418 	if (ret)
419 		goto err_put;
420 
421 	if (flags & DRM_IVPU_BO_MAPPABLE) {
422 		ivpu_bo_lock(bo);
423 		ret = drm_gem_shmem_vmap_locked(&bo->base, &map);
424 		ivpu_bo_unlock(bo);
425 
426 		if (ret)
427 			goto err_put;
428 	}
429 
430 	return bo;
431 
432 err_put:
433 	drm_gem_object_put(&bo->base.base);
434 	return NULL;
435 }
436 
437 struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags)
438 {
439 	struct ivpu_addr_range range;
440 
441 	if (!ivpu_is_within_range(addr, size, &vdev->hw->ranges.runtime)) {
442 		ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size);
443 		return NULL;
444 	}
445 
446 	if (ivpu_hw_range_init(vdev, &range, addr, size))
447 		return NULL;
448 
449 	return ivpu_bo_create(vdev, &vdev->gctx, &range, size, flags);
450 }
451 
452 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
453 {
454 	return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
455 }
456 
457 void ivpu_bo_free(struct ivpu_bo *bo)
458 {
459 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
460 
461 	if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
462 		ivpu_bo_lock(bo);
463 		drm_gem_shmem_vunmap_locked(&bo->base, &map);
464 		ivpu_bo_unlock(bo);
465 	}
466 
467 	drm_gem_object_put(&bo->base.base);
468 }
469 
470 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
471 {
472 	struct drm_ivpu_bo_info *args = data;
473 	struct drm_gem_object *obj;
474 	struct ivpu_bo *bo;
475 	int ret = 0;
476 
477 	obj = drm_gem_object_lookup(file, args->handle);
478 	if (!obj)
479 		return -ENOENT;
480 
481 	bo = to_ivpu_bo(obj);
482 
483 	ivpu_bo_lock(bo);
484 	args->flags = bo->flags;
485 	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
486 	args->vpu_addr = bo->vpu_addr;
487 	args->size = obj->size;
488 	ivpu_bo_unlock(bo);
489 
490 	drm_gem_object_put(obj);
491 	return ret;
492 }
493 
494 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
495 {
496 	struct drm_ivpu_bo_wait *args = data;
497 	struct drm_gem_object *obj;
498 	unsigned long timeout;
499 	long ret;
500 
501 	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
502 
503 	/* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
504 	timeout += 1;
505 
506 	obj = drm_gem_object_lookup(file, args->handle);
507 	if (!obj)
508 		return -EINVAL;
509 
510 	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
511 	if (ret == 0) {
512 		ret = -ETIMEDOUT;
513 	} else if (ret > 0) {
514 		ret = 0;
515 		args->job_status = to_ivpu_bo(obj)->job_status;
516 	}
517 
518 	drm_gem_object_put(obj);
519 
520 	return ret;
521 }
522 
523 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
524 {
525 	ivpu_bo_lock(bo);
526 
527 	drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
528 		   bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size,
529 		   bo->flags, kref_read(&bo->base.base.refcount));
530 
531 	if (bo->base.pages)
532 		drm_printf(p, " has_pages");
533 
534 	if (bo->mmu_mapped)
535 		drm_printf(p, " mmu_mapped");
536 
537 	if (drm_gem_is_imported(&bo->base.base))
538 		drm_printf(p, " imported");
539 
540 	drm_printf(p, "\n");
541 
542 	ivpu_bo_unlock(bo);
543 }
544 
545 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
546 {
547 	struct ivpu_device *vdev = to_ivpu_device(dev);
548 	struct ivpu_bo *bo;
549 
550 	drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
551 		   "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
552 
553 	mutex_lock(&vdev->bo_list_lock);
554 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
555 		ivpu_bo_print_info(bo, p);
556 	mutex_unlock(&vdev->bo_list_lock);
557 }
558 
559 void ivpu_bo_list_print(struct drm_device *dev)
560 {
561 	struct drm_printer p = drm_info_printer(dev->dev);
562 
563 	ivpu_bo_list(dev, &p);
564 }
565