xref: /linux/drivers/accel/ivpu/ivpu_gem.c (revision 1947b92464c3268381604bbe2ac977a3fd78192f) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/dma-buf.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/set_memory.h>
10 #include <linux/xarray.h>
11 
12 #include <drm/drm_cache.h>
13 #include <drm/drm_debugfs.h>
14 #include <drm/drm_file.h>
15 #include <drm/drm_utils.h>
16 
17 #include "ivpu_drv.h"
18 #include "ivpu_gem.h"
19 #include "ivpu_hw.h"
20 #include "ivpu_mmu.h"
21 #include "ivpu_mmu_context.h"
22 
23 static const struct drm_gem_object_funcs ivpu_gem_funcs;
24 
25 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
26 {
27 	if (bo->ctx)
28 		ivpu_dbg(vdev, BO, "%6s: size %zu has_pages %d dma_mapped %d handle %u ctx %d vpu_addr 0x%llx mmu_mapped %d\n",
29 			 action, ivpu_bo_size(bo), (bool)bo->base.pages, (bool)bo->base.sgt,
30 			 bo->handle, bo->ctx->id, bo->vpu_addr, bo->mmu_mapped);
31 	else
32 		ivpu_dbg(vdev, BO, "%6s: size %zu has_pages %d dma_mapped %d handle %u (not added to context)\n",
33 			 action, ivpu_bo_size(bo), (bool)bo->base.pages, (bool)bo->base.sgt,
34 			 bo->handle);
35 }
36 
37 /*
38  * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
39  *
40  * This function pins physical memory pages, then maps the physical pages
41  * to IOMMU address space and finally updates the VPU MMU page tables
42  * to allow the VPU to translate VPU address to IOMMU address.
43  */
44 int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
45 {
46 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
47 	int ret = 0;
48 
49 	mutex_lock(&bo->lock);
50 
51 	ivpu_dbg_bo(vdev, bo, "pin");
52 
53 	if (!bo->ctx) {
54 		ivpu_err(vdev, "vpu_addr not allocated for BO %d\n", bo->handle);
55 		ret = -EINVAL;
56 		goto unlock;
57 	}
58 
59 	if (!bo->mmu_mapped) {
60 		struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
61 
62 		if (IS_ERR(sgt)) {
63 			ret = PTR_ERR(sgt);
64 			ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
65 			goto unlock;
66 		}
67 
68 		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
69 					       ivpu_bo_is_snooped(bo));
70 		if (ret) {
71 			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
72 			goto unlock;
73 		}
74 		bo->mmu_mapped = true;
75 	}
76 
77 unlock:
78 	mutex_unlock(&bo->lock);
79 
80 	return ret;
81 }
82 
83 static int
84 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
85 		       const struct ivpu_addr_range *range)
86 {
87 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
88 	int ret;
89 
90 	mutex_lock(&bo->lock);
91 
92 	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
93 	if (!ret) {
94 		bo->ctx = ctx;
95 		bo->vpu_addr = bo->mm_node.start;
96 	} else {
97 		ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
98 	}
99 
100 	ivpu_dbg_bo(vdev, bo, "alloc");
101 
102 	mutex_unlock(&bo->lock);
103 
104 	return ret;
105 }
106 
107 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
108 {
109 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
110 
111 	lockdep_assert_held(&bo->lock);
112 
113 	ivpu_dbg_bo(vdev, bo, "unbind");
114 
115 	/* TODO: dma_unmap */
116 
117 	if (bo->mmu_mapped) {
118 		drm_WARN_ON(&vdev->drm, !bo->ctx);
119 		drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
120 		drm_WARN_ON(&vdev->drm, !bo->base.sgt);
121 		ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
122 		bo->mmu_mapped = false;
123 	}
124 
125 	if (bo->ctx) {
126 		ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
127 		bo->vpu_addr = 0;
128 		bo->ctx = NULL;
129 	}
130 }
131 
132 static void ivpu_bo_unbind(struct ivpu_bo *bo)
133 {
134 	mutex_lock(&bo->lock);
135 	ivpu_bo_unbind_locked(bo);
136 	mutex_unlock(&bo->lock);
137 }
138 
139 void ivpu_bo_remove_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
140 {
141 	struct ivpu_bo *bo;
142 
143 	if (drm_WARN_ON(&vdev->drm, !ctx))
144 		return;
145 
146 	mutex_lock(&vdev->bo_list_lock);
147 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
148 		mutex_lock(&bo->lock);
149 		if (bo->ctx == ctx)
150 			ivpu_bo_unbind_locked(bo);
151 		mutex_unlock(&bo->lock);
152 	}
153 	mutex_unlock(&vdev->bo_list_lock);
154 }
155 
156 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
157 {
158 	struct ivpu_bo *bo;
159 
160 	if (size == 0 || !PAGE_ALIGNED(size))
161 		return ERR_PTR(-EINVAL);
162 
163 	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
164 	if (!bo)
165 		return ERR_PTR(-ENOMEM);
166 
167 	bo->base.base.funcs = &ivpu_gem_funcs;
168 	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
169 
170 	INIT_LIST_HEAD(&bo->bo_list_node);
171 	mutex_init(&bo->lock);
172 
173 	return &bo->base.base;
174 }
175 
176 static struct ivpu_bo *
177 ivpu_bo_create(struct ivpu_device *vdev, u64 size, u32 flags)
178 {
179 	struct drm_gem_shmem_object *shmem;
180 	struct ivpu_bo *bo;
181 
182 	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
183 	case DRM_IVPU_BO_CACHED:
184 	case DRM_IVPU_BO_WC:
185 		break;
186 	default:
187 		return ERR_PTR(-EINVAL);
188 	}
189 
190 	shmem = drm_gem_shmem_create(&vdev->drm, size);
191 	if (IS_ERR(shmem))
192 		return ERR_CAST(shmem);
193 
194 	bo = to_ivpu_bo(&shmem->base);
195 	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
196 	bo->flags = flags;
197 
198 	mutex_lock(&vdev->bo_list_lock);
199 	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
200 	mutex_unlock(&vdev->bo_list_lock);
201 
202 	ivpu_dbg(vdev, BO, "create: vpu_addr 0x%llx size %zu flags 0x%x\n",
203 		 bo->vpu_addr, bo->base.base.size, flags);
204 
205 	return bo;
206 }
207 
208 static int ivpu_bo_open(struct drm_gem_object *obj, struct drm_file *file)
209 {
210 	struct ivpu_file_priv *file_priv = file->driver_priv;
211 	struct ivpu_device *vdev = file_priv->vdev;
212 	struct ivpu_bo *bo = to_ivpu_bo(obj);
213 	struct ivpu_addr_range *range;
214 
215 	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
216 		range = &vdev->hw->ranges.shave;
217 	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
218 		range = &vdev->hw->ranges.dma;
219 	else
220 		range = &vdev->hw->ranges.user;
221 
222 	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
223 }
224 
225 static void ivpu_bo_free(struct drm_gem_object *obj)
226 {
227 	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
228 	struct ivpu_bo *bo = to_ivpu_bo(obj);
229 
230 	mutex_lock(&vdev->bo_list_lock);
231 	list_del(&bo->bo_list_node);
232 	mutex_unlock(&vdev->bo_list_lock);
233 
234 	drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
235 
236 	ivpu_dbg_bo(vdev, bo, "free");
237 
238 	ivpu_bo_unbind(bo);
239 	mutex_destroy(&bo->lock);
240 
241 	drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);
242 	drm_gem_shmem_free(&bo->base);
243 }
244 
245 static const struct dma_buf_ops ivpu_bo_dmabuf_ops =  {
246 	.cache_sgt_mapping = true,
247 	.attach = drm_gem_map_attach,
248 	.detach = drm_gem_map_detach,
249 	.map_dma_buf = drm_gem_map_dma_buf,
250 	.unmap_dma_buf = drm_gem_unmap_dma_buf,
251 	.release = drm_gem_dmabuf_release,
252 	.mmap = drm_gem_dmabuf_mmap,
253 	.vmap = drm_gem_dmabuf_vmap,
254 	.vunmap = drm_gem_dmabuf_vunmap,
255 };
256 
257 static struct dma_buf *ivpu_bo_export(struct drm_gem_object *obj, int flags)
258 {
259 	struct drm_device *dev = obj->dev;
260 	struct dma_buf_export_info exp_info = {
261 		.exp_name = KBUILD_MODNAME,
262 		.owner = dev->driver->fops->owner,
263 		.ops = &ivpu_bo_dmabuf_ops,
264 		.size = obj->size,
265 		.flags = flags,
266 		.priv = obj,
267 		.resv = obj->resv,
268 	};
269 	void *sgt;
270 
271 	/*
272 	 * Make sure that pages are allocated and dma-mapped before exporting the bo.
273 	 * DMA-mapping is required if the bo will be imported to the same device.
274 	 */
275 	sgt = drm_gem_shmem_get_pages_sgt(to_drm_gem_shmem_obj(obj));
276 	if (IS_ERR(sgt))
277 		return sgt;
278 
279 	return drm_gem_dmabuf_export(dev, &exp_info);
280 }
281 
282 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
283 	.free = ivpu_bo_free,
284 	.open = ivpu_bo_open,
285 	.export = ivpu_bo_export,
286 	.print_info = drm_gem_shmem_object_print_info,
287 	.pin = drm_gem_shmem_object_pin,
288 	.unpin = drm_gem_shmem_object_unpin,
289 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
290 	.vmap = drm_gem_shmem_object_vmap,
291 	.vunmap = drm_gem_shmem_object_vunmap,
292 	.mmap = drm_gem_shmem_object_mmap,
293 	.vm_ops = &drm_gem_shmem_vm_ops,
294 };
295 
296 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
297 {
298 	struct ivpu_file_priv *file_priv = file->driver_priv;
299 	struct ivpu_device *vdev = file_priv->vdev;
300 	struct drm_ivpu_bo_create *args = data;
301 	u64 size = PAGE_ALIGN(args->size);
302 	struct ivpu_bo *bo;
303 	int ret;
304 
305 	if (args->flags & ~DRM_IVPU_BO_FLAGS)
306 		return -EINVAL;
307 
308 	if (size == 0)
309 		return -EINVAL;
310 
311 	bo = ivpu_bo_create(vdev, size, args->flags);
312 	if (IS_ERR(bo)) {
313 		ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)",
314 			 bo, file_priv->ctx.id, args->size, args->flags);
315 		return PTR_ERR(bo);
316 	}
317 
318 	ret = drm_gem_handle_create(file, &bo->base.base, &bo->handle);
319 	if (!ret) {
320 		args->vpu_addr = bo->vpu_addr;
321 		args->handle = bo->handle;
322 	}
323 
324 	drm_gem_object_put(&bo->base.base);
325 
326 	return ret;
327 }
328 
329 struct ivpu_bo *
330 ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags)
331 {
332 	const struct ivpu_addr_range *range;
333 	struct ivpu_addr_range fixed_range;
334 	struct iosys_map map;
335 	struct ivpu_bo *bo;
336 	int ret;
337 
338 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr));
339 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
340 
341 	if (vpu_addr) {
342 		fixed_range.start = vpu_addr;
343 		fixed_range.end = vpu_addr + size;
344 		range = &fixed_range;
345 	} else {
346 		range = &vdev->hw->ranges.global;
347 	}
348 
349 	bo = ivpu_bo_create(vdev, size, flags);
350 	if (IS_ERR(bo)) {
351 		ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
352 			 bo, vpu_addr, size, flags);
353 		return NULL;
354 	}
355 
356 	ret = ivpu_bo_alloc_vpu_addr(bo, &vdev->gctx, range);
357 	if (ret)
358 		goto err_put;
359 
360 	ret = ivpu_bo_pin(bo);
361 	if (ret)
362 		goto err_put;
363 
364 	ret = drm_gem_shmem_vmap(&bo->base, &map);
365 	if (ret)
366 		goto err_put;
367 
368 	return bo;
369 
370 err_put:
371 	drm_gem_object_put(&bo->base.base);
372 	return NULL;
373 }
374 
375 void ivpu_bo_free_internal(struct ivpu_bo *bo)
376 {
377 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
378 
379 	drm_gem_shmem_vunmap(&bo->base, &map);
380 	drm_gem_object_put(&bo->base.base);
381 }
382 
383 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
384 {
385 	struct drm_ivpu_bo_info *args = data;
386 	struct drm_gem_object *obj;
387 	struct ivpu_bo *bo;
388 	int ret = 0;
389 
390 	obj = drm_gem_object_lookup(file, args->handle);
391 	if (!obj)
392 		return -ENOENT;
393 
394 	bo = to_ivpu_bo(obj);
395 
396 	mutex_lock(&bo->lock);
397 	args->flags = bo->flags;
398 	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
399 	args->vpu_addr = bo->vpu_addr;
400 	args->size = obj->size;
401 	mutex_unlock(&bo->lock);
402 
403 	drm_gem_object_put(obj);
404 	return ret;
405 }
406 
407 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
408 {
409 	struct drm_ivpu_bo_wait *args = data;
410 	struct drm_gem_object *obj;
411 	unsigned long timeout;
412 	long ret;
413 
414 	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
415 
416 	obj = drm_gem_object_lookup(file, args->handle);
417 	if (!obj)
418 		return -EINVAL;
419 
420 	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
421 	if (ret == 0) {
422 		ret = -ETIMEDOUT;
423 	} else if (ret > 0) {
424 		ret = 0;
425 		args->job_status = to_ivpu_bo(obj)->job_status;
426 	}
427 
428 	drm_gem_object_put(obj);
429 
430 	return ret;
431 }
432 
433 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
434 {
435 	unsigned long dma_refcount = 0;
436 
437 	mutex_lock(&bo->lock);
438 
439 	if (bo->base.base.dma_buf && bo->base.base.dma_buf->file)
440 		dma_refcount = atomic_long_read(&bo->base.base.dma_buf->file->f_count);
441 
442 	drm_printf(p, "%-3u %-6d 0x%-12llx %-10lu 0x%-8x %-4u %-8lu",
443 		   bo->ctx->id, bo->handle, bo->vpu_addr, bo->base.base.size,
444 		   bo->flags, kref_read(&bo->base.base.refcount), dma_refcount);
445 
446 	if (bo->base.base.import_attach)
447 		drm_printf(p, " imported");
448 
449 	if (bo->base.pages)
450 		drm_printf(p, " has_pages");
451 
452 	if (bo->mmu_mapped)
453 		drm_printf(p, " mmu_mapped");
454 
455 	drm_printf(p, "\n");
456 
457 	mutex_unlock(&bo->lock);
458 }
459 
460 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
461 {
462 	struct ivpu_device *vdev = to_ivpu_device(dev);
463 	struct ivpu_bo *bo;
464 
465 	drm_printf(p, "%-3s %-6s %-14s %-10s %-10s %-4s %-8s %s\n",
466 		   "ctx", "handle", "vpu_addr", "size", "flags", "refs", "dma_refs", "attribs");
467 
468 	mutex_lock(&vdev->bo_list_lock);
469 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
470 		ivpu_bo_print_info(bo, p);
471 	mutex_unlock(&vdev->bo_list_lock);
472 }
473 
474 void ivpu_bo_list_print(struct drm_device *dev)
475 {
476 	struct drm_printer p = drm_info_printer(dev->dev);
477 
478 	ivpu_bo_list(dev, &p);
479 }
480