xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c (revision cb8fef4b07f71da01843457b66e579ad248a625e)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/module.h>
29 #include <linux/overflow.h>
30 #include <linux/pagemap.h>
31 #include <linux/pci.h>
32 #include <linux/dma-buf.h>
33 #include <linux/dma-fence-unwrap.h>
34 #include <linux/uaccess.h>
35 #include <linux/pm_runtime.h>
36 
37 #include <drm/amdgpu_drm.h>
38 #include <drm/drm_drv.h>
39 #include <drm/drm_exec.h>
40 #include <drm/drm_gem_ttm_helper.h>
41 #include <drm/ttm/ttm_tt.h>
42 #include <drm/drm_syncobj.h>
43 #include <drm/drm_timeout.h>
44 
45 #include "amdgpu.h"
46 #include "amdgpu_display.h"
47 #include "amdgpu_dma_buf.h"
48 #include "amdgpu_hmm.h"
49 #include "amdgpu_xgmi.h"
50 #include "amdgpu_vm.h"
51 
52 static int
53 amdgpu_gem_add_input_fence(struct drm_file *filp,
54 			   uint64_t syncobj_handles_array,
55 			   uint32_t num_syncobj_handles)
56 {
57 	struct dma_fence *fence;
58 	uint32_t *syncobj_handles;
59 	int ret, i;
60 
61 	if (!num_syncobj_handles)
62 		return 0;
63 
64 	syncobj_handles = memdup_user(u64_to_user_ptr(syncobj_handles_array),
65 				      size_mul(sizeof(uint32_t), num_syncobj_handles));
66 	if (IS_ERR(syncobj_handles))
67 		return PTR_ERR(syncobj_handles);
68 
69 	for (i = 0; i < num_syncobj_handles; i++) {
70 
71 		if (!syncobj_handles[i]) {
72 			ret = -EINVAL;
73 			goto free_memdup;
74 		}
75 
76 		ret = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 0, &fence);
77 		if (ret)
78 			goto free_memdup;
79 
80 		dma_fence_wait(fence, false);
81 
82 		/* TODO: optimize async handling */
83 		dma_fence_put(fence);
84 	}
85 
86 free_memdup:
87 	kfree(syncobj_handles);
88 	return ret;
89 }
90 
91 static int
92 amdgpu_gem_update_timeline_node(struct drm_file *filp,
93 				uint32_t syncobj_handle,
94 				uint64_t point,
95 				struct drm_syncobj **syncobj,
96 				struct dma_fence_chain **chain)
97 {
98 	if (!syncobj_handle)
99 		return 0;
100 
101 	/* Find the sync object */
102 	*syncobj = drm_syncobj_find(filp, syncobj_handle);
103 	if (!*syncobj)
104 		return -ENOENT;
105 
106 	if (!point)
107 		return 0;
108 
109 	/* Allocate the chain node */
110 	*chain = dma_fence_chain_alloc();
111 	if (!*chain) {
112 		drm_syncobj_put(*syncobj);
113 		*syncobj = NULL;
114 		return -ENOMEM;
115 	}
116 
117 	return 0;
118 }
119 
120 static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
121 {
122 	struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
123 	struct drm_device *ddev = bo->base.dev;
124 	vm_fault_t ret;
125 	int idx;
126 
127 	ret = ttm_bo_vm_reserve(bo, vmf);
128 	if (ret)
129 		return ret;
130 
131 	if (drm_dev_enter(ddev, &idx)) {
132 		pm_runtime_get_noresume(ddev->dev);
133 
134 		ret = amdgpu_bo_fault_reserve_notify(bo);
135 		if (ret) {
136 			drm_dev_exit(idx);
137 			goto unlock;
138 		}
139 
140 		ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
141 					       TTM_BO_VM_NUM_PREFAULT);
142 
143 		pm_runtime_put_noidle(ddev->dev);
144 		drm_dev_exit(idx);
145 	} else {
146 		ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
147 	}
148 	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
149 		return ret;
150 
151 unlock:
152 	dma_resv_unlock(bo->base.resv);
153 	return ret;
154 }
155 
156 static const struct vm_operations_struct amdgpu_gem_vm_ops = {
157 	.fault = amdgpu_gem_fault,
158 	.open = ttm_bo_vm_open,
159 	.close = ttm_bo_vm_close,
160 	.access = ttm_bo_vm_access
161 };
162 
163 static void amdgpu_gem_object_free(struct drm_gem_object *gobj)
164 {
165 	struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
166 
167 	amdgpu_hmm_unregister(aobj);
168 	ttm_bo_fini(&aobj->tbo);
169 }
170 
171 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
172 			     int alignment, u32 initial_domain,
173 			     u64 flags, enum ttm_bo_type type,
174 			     struct dma_resv *resv,
175 			     struct drm_gem_object **obj, int8_t xcp_id_plus1)
176 {
177 	struct amdgpu_bo *bo;
178 	struct amdgpu_bo_user *ubo;
179 	struct amdgpu_bo_param bp;
180 	int r;
181 
182 	memset(&bp, 0, sizeof(bp));
183 	*obj = NULL;
184 	flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
185 
186 	bp.size = size;
187 	bp.byte_align = alignment;
188 	bp.type = type;
189 	bp.resv = resv;
190 	bp.preferred_domain = initial_domain;
191 	bp.flags = flags;
192 	bp.domain = initial_domain;
193 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
194 	bp.xcp_id_plus1 = xcp_id_plus1;
195 
196 	r = amdgpu_bo_create_user(adev, &bp, &ubo);
197 	if (r)
198 		return r;
199 
200 	bo = &ubo->bo;
201 	*obj = &bo->tbo.base;
202 
203 	return 0;
204 }
205 
206 void amdgpu_gem_force_release(struct amdgpu_device *adev)
207 {
208 	struct drm_device *ddev = adev_to_drm(adev);
209 	struct drm_file *file;
210 
211 	mutex_lock(&ddev->filelist_mutex);
212 
213 	list_for_each_entry(file, &ddev->filelist, lhead) {
214 		struct drm_gem_object *gobj;
215 		int handle;
216 
217 		WARN_ONCE(1, "Still active user space clients!\n");
218 		spin_lock(&file->table_lock);
219 		idr_for_each_entry(&file->object_idr, gobj, handle) {
220 			WARN_ONCE(1, "And also active allocations!\n");
221 			drm_gem_object_put(gobj);
222 		}
223 		idr_destroy(&file->object_idr);
224 		spin_unlock(&file->table_lock);
225 	}
226 
227 	mutex_unlock(&ddev->filelist_mutex);
228 }
229 
230 /*
231  * Call from drm_gem_handle_create which appear in both new and open ioctl
232  * case.
233  */
234 static int amdgpu_gem_object_open(struct drm_gem_object *obj,
235 				  struct drm_file *file_priv)
236 {
237 	struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
238 	struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
239 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
240 	struct amdgpu_vm *vm = &fpriv->vm;
241 	struct amdgpu_bo_va *bo_va;
242 	struct mm_struct *mm;
243 	struct drm_exec exec;
244 	int r = 0;
245 
246 	mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
247 	if (mm && mm != current->mm)
248 		return -EPERM;
249 
250 	if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
251 	    !amdgpu_vm_is_bo_always_valid(vm, abo))
252 		return -EPERM;
253 
254 	/* Validate and add eviction fence to DMABuf imports with dynamic
255 	 * attachment in compute VMs. Re-validation will be done by
256 	 * amdgpu_vm_validate. Fences are on the reservation shared with the
257 	 * export, which is currently required to be validated and fenced
258 	 * already by amdgpu_amdkfd_gpuvm_restore_process_bos.
259 	 *
260 	 * This runs before the BO is added to the VM: drm_gem_handle_create_tail()
261 	 * does not call the close() callback when open() fails, so nothing that
262 	 * would need unwinding may exist yet.
263 	 *
264 	 * Nested locking below for the case that a GEM object is opened in
265 	 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports,
266 	 * but not for export, this is a different lock class that cannot lead to
267 	 * circular lock dependencies.
268 	 */
269 	if (vm->is_compute_context && vm->process_info &&
270 	    drm_gem_is_imported(obj) &&
271 	    dma_buf_is_dynamic(obj->import_attach->dmabuf)) {
272 		mutex_lock_nested(&vm->process_info->lock, 1);
273 		if (!WARN_ON(!vm->process_info->eviction_fence))
274 			r = amdgpu_amdkfd_bo_validate_and_fence(abo,
275 					AMDGPU_GEM_DOMAIN_GTT,
276 					&vm->process_info->eviction_fence->base);
277 		mutex_unlock(&vm->process_info->lock);
278 		if (r) {
279 			struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);
280 
281 			dev_warn(adev->dev, "validate_and_fence failed: %d\n", r);
282 			if (ti) {
283 				dev_warn(adev->dev, "pid %d\n", ti->task.pid);
284 				amdgpu_vm_put_task_info(ti);
285 			}
286 			return r;
287 		}
288 	}
289 
290 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
291 	drm_exec_until_all_locked(&exec) {
292 		r = drm_exec_prepare_obj(&exec, &abo->tbo.base, TTM_NUM_MOVE_FENCES + 1);
293 		drm_exec_retry_on_contention(&exec);
294 		if (unlikely(r))
295 			goto out_unlock;
296 
297 		r = amdgpu_vm_lock_pd(vm, &exec, 0);
298 		drm_exec_retry_on_contention(&exec);
299 		if (unlikely(r))
300 			goto out_unlock;
301 	}
302 
303 	amdgpu_vm_bo_update_shared(abo);
304 	bo_va = amdgpu_vm_bo_find(vm, abo);
305 	if (!bo_va) {
306 		bo_va = amdgpu_vm_bo_add(adev, vm, abo);
307 		if (!bo_va) {
308 			r = -ENOMEM;
309 			goto out_unlock;
310 		}
311 		r = amdgpu_evf_mgr_attach_fence(&fpriv->evf_mgr, abo);
312 		if (r) {
313 			amdgpu_vm_bo_del(adev, bo_va);
314 			goto out_unlock;
315 		}
316 	} else {
317 		++bo_va->ref_count;
318 	}
319 
320 	drm_exec_fini(&exec);
321 	return 0;
322 
323 out_unlock:
324 	drm_exec_fini(&exec);
325 	return r;
326 }
327 
328 static void amdgpu_gem_object_close(struct drm_gem_object *obj,
329 				    struct drm_file *file_priv)
330 {
331 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
332 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
333 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
334 	struct amdgpu_vm *vm = &fpriv->vm;
335 
336 	struct dma_fence *fence = NULL;
337 	struct amdgpu_bo_va *bo_va;
338 	struct drm_exec exec;
339 	long r;
340 
341 	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
342 	drm_exec_until_all_locked(&exec) {
343 		r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1);
344 		drm_exec_retry_on_contention(&exec);
345 		if (unlikely(r))
346 			goto out_unlock;
347 
348 		r = amdgpu_vm_lock_pd(vm, &exec, 0);
349 		drm_exec_retry_on_contention(&exec);
350 		if (unlikely(r))
351 			goto out_unlock;
352 	}
353 
354 	bo_va = amdgpu_vm_bo_find(vm, bo);
355 	/* the fence is attached once per bo_va, so keep it for other handles */
356 	if (bo_va && --bo_va->ref_count)
357 		goto out_unlock;
358 
359 	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
360 		amdgpu_evf_mgr_detach_fence(&fpriv->evf_mgr, bo);
361 	if (!bo_va)
362 		goto out_unlock;
363 
364 	amdgpu_vm_bo_del(adev, bo_va);
365 	amdgpu_vm_bo_update_shared(bo);
366 	if (!amdgpu_vm_ready(vm))
367 		goto out_unlock;
368 
369 	r = amdgpu_vm_clear_freed(adev, vm, &fence);
370 	if (unlikely(r < 0) && !drm_dev_is_unplugged(adev_to_drm(adev)))
371 		dev_err(adev->dev, "failed to clear page "
372 			"tables on GEM object close (%ld)\n", r);
373 	if (r || !fence)
374 		goto out_unlock;
375 
376 	amdgpu_bo_fence(bo, fence, true);
377 	dma_fence_put(fence);
378 
379 out_unlock:
380 	if (r && !drm_dev_is_unplugged(adev_to_drm(adev)))
381 		dev_err(adev->dev, "leaking bo va (%ld)\n", r);
382 	drm_exec_fini(&exec);
383 }
384 
385 static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
386 {
387 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
388 
389 	if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
390 		return -EPERM;
391 	if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
392 		return -EPERM;
393 
394 	/* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
395 	 * for debugger access to invisible VRAM. Should have used MAP_SHARED
396 	 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
397 	 * becoming writable and makes vma_is_cow_mapping(vma) false.
398 	 */
399 	if (vma_is_cow_mapping(vma) &&
400 	    !(vma->vm_flags & VM_ACCESS_FLAGS))
401 		vm_flags_clear(vma, VM_MAYWRITE);
402 
403 	return drm_gem_ttm_mmap(obj, vma);
404 }
405 
406 static void amdgpu_gem_object_handle_free(struct drm_gem_object *gobj)
407 {
408 	struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
409 
410 	amdgpu_ualink_revoke_exported_memory(aobj);
411 
412 }
413 
414 const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
415 	.free = amdgpu_gem_object_free,
416 	.open = amdgpu_gem_object_open,
417 	.close = amdgpu_gem_object_close,
418 	.export = amdgpu_gem_prime_export,
419 	.vmap = drm_gem_ttm_vmap,
420 	.vunmap = drm_gem_ttm_vunmap,
421 	.mmap = amdgpu_gem_object_mmap,
422 	.vm_ops = &amdgpu_gem_vm_ops,
423 	.handle_free = amdgpu_gem_object_handle_free
424 };
425 
426 static bool amdgpu_gem_are_domains_valid(u32 domains)
427 {
428 	u32 normal = AMDGPU_GEM_DOMAIN_CPU |
429 		     AMDGPU_GEM_DOMAIN_GTT |
430 		     AMDGPU_GEM_DOMAIN_VRAM;
431 	/* Treat all non CPU/GTT/VRAM domains as special domains. */
432 	u32 special = AMDGPU_GEM_DOMAIN_MASK & ~normal;
433 	u32 normal_mask = domains & normal;
434 	u32 special_mask = domains & special;
435 
436 	if (!special_mask)
437 		return true;
438 
439 	if (normal_mask)
440 		return false;
441 
442 	return !(special_mask & (special_mask - 1));
443 }
444 
445 /*
446  * GEM ioctls.
447  */
448 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
449 			    struct drm_file *filp)
450 {
451 	struct amdgpu_device *adev = drm_to_adev(dev);
452 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
453 	struct amdgpu_vm *vm = &fpriv->vm;
454 	union drm_amdgpu_gem_create *args = data;
455 	uint64_t flags = args->in.domain_flags;
456 	uint64_t size = args->in.bo_size;
457 	struct dma_resv *resv = NULL;
458 	struct drm_gem_object *gobj;
459 	uint32_t handle, initial_domain;
460 	int r;
461 
462 	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
463 
464 	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
465 	if (r)
466 		return r;
467 
468 	/* reject invalid gem flags */
469 	if (flags & ~AMDGPU_GEM_CREATE_SETTABLE_MASK)
470 		return -EINVAL;
471 
472 	/* reject invalid gem domains */
473 	if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
474 		return -EINVAL;
475 	if (!amdgpu_gem_are_domains_valid(args->in.domains))
476 		return -EINVAL;
477 
478 	if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
479 		DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
480 		return -EINVAL;
481 	}
482 
483 	/* always clear VRAM */
484 	flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
485 
486 	/* create a gem object to contain this object in */
487 	if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
488 	    AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
489 		if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
490 			/* if gds bo is created from user space, it must be
491 			 * passed to bo list
492 			 */
493 			DRM_ERROR("GDS bo cannot be per-vm-bo\n");
494 			return -EINVAL;
495 		}
496 		flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
497 	}
498 
499 	if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
500 		r = amdgpu_bo_reserve(vm->root.bo, false);
501 		if (r)
502 			return r;
503 
504 		resv = vm->root.bo->tbo.base.resv;
505 	}
506 
507 	initial_domain = (u32)(0xffffffff & args->in.domains);
508 retry:
509 	r = amdgpu_gem_object_create(adev, size, args->in.alignment,
510 				     initial_domain,
511 				     flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
512 	if (r && r != -ERESTARTSYS) {
513 		if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
514 			flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
515 			goto retry;
516 		}
517 
518 		if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
519 			initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
520 			goto retry;
521 		}
522 		DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
523 				size, initial_domain, args->in.alignment, r);
524 	}
525 
526 	if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
527 		if (!r) {
528 			struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
529 
530 			abo->parent = amdgpu_bo_ref(vm->root.bo);
531 		}
532 		amdgpu_bo_unreserve(vm->root.bo);
533 	}
534 	if (r)
535 		return r;
536 
537 	r = drm_gem_handle_create(filp, gobj, &handle);
538 	/* drop reference from allocate - handle holds it now */
539 	drm_gem_object_put(gobj);
540 	if (r)
541 		return r;
542 
543 	memset(args, 0, sizeof(*args));
544 	args->out.handle = handle;
545 	return 0;
546 }
547 
548 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
549 			     struct drm_file *filp)
550 {
551 	struct ttm_operation_ctx ctx = { true, false };
552 	struct amdgpu_device *adev = drm_to_adev(dev);
553 	struct drm_amdgpu_gem_userptr *args = data;
554 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
555 	struct drm_gem_object *gobj;
556 	struct amdgpu_hmm_range *range;
557 	struct amdgpu_bo *bo;
558 	uint32_t handle;
559 	int r;
560 
561 	args->addr = untagged_addr(args->addr);
562 
563 	if (offset_in_page(args->addr | args->size))
564 		return -EINVAL;
565 
566 	if (!access_ok((void __user *)(uintptr_t)args->addr, args->size))
567 		return -EFAULT;
568 
569 	/* reject unknown flag values */
570 	if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
571 	    AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
572 	    AMDGPU_GEM_USERPTR_REGISTER))
573 		return -EINVAL;
574 
575 	if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
576 	     !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
577 
578 		/* if we want to write to it we must install a MMU notifier */
579 		return -EACCES;
580 	}
581 
582 	/* create a gem object to contain this object in */
583 	r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
584 				     0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
585 	if (r)
586 		return r;
587 
588 	bo = gem_to_amdgpu_bo(gobj);
589 	bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
590 	bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
591 	bo->parent = amdgpu_bo_ref(fpriv->vm.root.bo);
592 	r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
593 	if (r)
594 		goto release_object;
595 
596 	r = amdgpu_hmm_register(bo, args->addr);
597 	if (r)
598 		goto release_object;
599 
600 	if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
601 		range = amdgpu_hmm_range_alloc(NULL);
602 		if (unlikely(!range)) {
603 			r = -ENOMEM;
604 			goto release_object;
605 		}
606 		r = amdgpu_ttm_tt_get_user_pages(bo, range);
607 		if (r) {
608 			amdgpu_hmm_range_free(range);
609 			goto release_object;
610 		}
611 
612 		r = pm_runtime_resume_and_get(adev->dev);
613 		if (r < 0)
614 			goto user_pages_done;
615 
616 		r = amdgpu_bo_reserve(bo, true);
617 		if (r)
618 			goto user_pages_done;
619 
620 		amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range);
621 
622 		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
623 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
624 		amdgpu_bo_unreserve(bo);
625 
626 		pm_runtime_put_autosuspend(adev->dev);
627 
628 		if (r)
629 			goto user_pages_done;
630 	}
631 
632 	r = drm_gem_handle_create(filp, gobj, &handle);
633 	if (r)
634 		goto user_pages_done;
635 
636 	args->handle = handle;
637 
638 user_pages_done:
639 	if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
640 		amdgpu_hmm_range_free(range);
641 release_object:
642 	drm_gem_object_put(gobj);
643 
644 	return r;
645 }
646 
647 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
648 			  struct drm_device *dev,
649 			  uint32_t handle, uint64_t *offset_p)
650 {
651 	struct drm_gem_object *gobj;
652 	struct amdgpu_bo *robj;
653 
654 	gobj = drm_gem_object_lookup(filp, handle);
655 	if (!gobj)
656 		return -ENOENT;
657 
658 	robj = gem_to_amdgpu_bo(gobj);
659 	if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
660 	    (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
661 		drm_gem_object_put(gobj);
662 		return -EPERM;
663 	}
664 	*offset_p = amdgpu_bo_mmap_offset(robj);
665 	drm_gem_object_put(gobj);
666 	return 0;
667 }
668 
669 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
670 			  struct drm_file *filp)
671 {
672 	union drm_amdgpu_gem_mmap *args = data;
673 	uint32_t handle = args->in.handle;
674 
675 	memset(args, 0, sizeof(*args));
676 	return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
677 }
678 
679 /**
680  * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
681  *
682  * @timeout_ns: timeout in ns
683  *
684  * Calculate the timeout in jiffies from an absolute timeout in ns.
685  */
686 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
687 {
688 	/* Map anything that doesn't fit in a s64 to an infinite wait */
689 	if (timeout_ns > S64_MAX)
690 		return MAX_SCHEDULE_TIMEOUT;
691 
692 	return drm_timeout_abs_to_jiffies(timeout_ns);
693 }
694 
695 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
696 			      struct drm_file *filp)
697 {
698 	union drm_amdgpu_gem_wait_idle *args = data;
699 	struct drm_gem_object *gobj;
700 	struct amdgpu_bo *robj;
701 	uint32_t handle = args->in.handle;
702 	unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
703 	int r = 0;
704 	long ret;
705 
706 	gobj = drm_gem_object_lookup(filp, handle);
707 	if (!gobj)
708 		return -ENOENT;
709 
710 	robj = gem_to_amdgpu_bo(gobj);
711 	ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,
712 				    true, timeout);
713 
714 	/* ret == 0 means not signaled,
715 	 * ret > 0 means signaled
716 	 * ret < 0 means interrupted before timeout
717 	 */
718 	if (ret >= 0) {
719 		memset(args, 0, sizeof(*args));
720 		args->out.status = (ret == 0);
721 	} else
722 		r = ret;
723 
724 	drm_gem_object_put(gobj);
725 	return r;
726 }
727 
728 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
729 				struct drm_file *filp)
730 {
731 	struct drm_amdgpu_gem_metadata *args = data;
732 	struct drm_gem_object *gobj;
733 	struct amdgpu_bo *robj;
734 	int r = -1;
735 
736 	DRM_DEBUG("%d\n", args->handle);
737 	gobj = drm_gem_object_lookup(filp, args->handle);
738 	if (gobj == NULL)
739 		return -ENOENT;
740 	robj = gem_to_amdgpu_bo(gobj);
741 
742 	r = amdgpu_bo_reserve(robj, false);
743 	if (unlikely(r != 0))
744 		goto out;
745 
746 	/* Reject MMIO_REMAP BOs at IOCTL level: metadata/tiling does not apply. */
747 	if (robj->tbo.resource &&
748 	    robj->tbo.resource->mem_type == AMDGPU_PL_MMIO_REMAP) {
749 		DRM_WARN("metadata ioctl on MMIO_REMAP BO (handle %d)\n",
750 			 args->handle);
751 		r = -EINVAL;
752 		goto unreserve;
753 	}
754 
755 	if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
756 		amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
757 		r = amdgpu_bo_get_metadata(robj, args->data.data,
758 					   sizeof(args->data.data),
759 					   &args->data.data_size_bytes,
760 					   &args->data.flags);
761 	} else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
762 		if (args->data.data_size_bytes > sizeof(args->data.data)) {
763 			r = -EINVAL;
764 			goto unreserve;
765 		}
766 		r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
767 		if (!r)
768 			r = amdgpu_bo_set_metadata(robj, args->data.data,
769 						   args->data.data_size_bytes,
770 						   args->data.flags);
771 	}
772 
773 unreserve:
774 	amdgpu_bo_unreserve(robj);
775 out:
776 	drm_gem_object_put(gobj);
777 	return r;
778 }
779 
780 /**
781  * amdgpu_gem_va_update_vm -update the bo_va in its VM
782  *
783  * @adev: amdgpu_device pointer
784  * @vm: vm to update
785  * @bo_va: bo_va to update
786  * @operation: map, unmap or clear
787  *
788  * Update the bo_va directly after setting its address. Errors are not
789  * vital here, so they are not reported back to userspace.
790  *
791  * Returns resulting fence if freed BO(s) got cleared from the PT.
792  * otherwise stub fence in case of error.
793  */
794 static struct dma_fence *
795 amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
796 			struct amdgpu_vm *vm,
797 			struct amdgpu_bo_va *bo_va,
798 			uint32_t operation)
799 {
800 	struct dma_fence *fence;
801 	int r = 0;
802 
803 	/* If the VM is not ready return only a stub. */
804 	if (!amdgpu_vm_ready(vm))
805 		return dma_fence_get_stub();
806 
807 
808 	/*
809 	 * First clean up any freed mappings in the VM.
810 	 *
811 	 * amdgpu_vm_clear_freed() may replace @fence with a new fence if it
812 	 * schedules GPU work. If nothing needs clearing, @fence can remain as
813 	 * the original vm->last_update.
814 	 */
815 	r = amdgpu_vm_clear_freed(adev, vm, &vm->last_update);
816 	if (r)
817 		goto error;
818 
819 	/* For MAP/REPLACE we also need to update the BO mappings. */
820 	if (operation == AMDGPU_VA_OP_MAP ||
821 	    operation == AMDGPU_VA_OP_REPLACE) {
822 		r = amdgpu_vm_bo_update(adev, bo_va, false);
823 		if (r)
824 			goto error;
825 	}
826 
827 	/* Always update PDEs after we touched the mappings. */
828 	r = amdgpu_vm_update_pdes(adev, vm, false);
829 	if (r)
830 		goto error;
831 
832 	if ((operation == AMDGPU_VA_OP_MAP ||
833 	     operation == AMDGPU_VA_OP_REPLACE) &&
834 	    !amdgpu_vm_is_bo_always_valid(vm, bo_va->base.bo)) {
835 
836 		/*
837 		 * For MAP/REPLACE of non per-VM BOs we need to sync to both the
838 		 * bo_va->last_pt_update and vm->last_update or otherwise we
839 		 * potentially miss the PDE updates.
840 		 */
841 		fence = dma_fence_unwrap_merge(vm->last_update,
842 					       bo_va->last_pt_update);
843 		if (!fence) {
844 			/* As fallback in OOM situations */
845 			dma_fence_wait(vm->last_update, false);
846 			dma_fence_wait(bo_va->last_pt_update, false);
847 			fence = dma_fence_get_stub();
848 		}
849 	} else {
850 		fence = dma_fence_get(vm->last_update);
851 	}
852 
853 	return fence;
854 
855 error:
856 	if (r && r != -ERESTARTSYS)
857 		DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
858 
859 	return dma_fence_get(vm->last_update);
860 }
861 
862 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
863 			  struct drm_file *filp)
864 {
865 	const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
866 		AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
867 		AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
868 		AMDGPU_VM_PAGE_NOALLOC;
869 	const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
870 		AMDGPU_VM_PAGE_PRT;
871 
872 	struct drm_amdgpu_gem_va *args = data;
873 	struct drm_gem_object *gobj;
874 	struct amdgpu_device *adev = drm_to_adev(dev);
875 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
876 	struct amdgpu_bo *abo;
877 	struct amdgpu_bo_va *bo_va;
878 	struct drm_syncobj *timeline_syncobj = NULL;
879 	struct dma_fence_chain *timeline_chain = NULL;
880 	struct drm_exec exec;
881 	uint64_t vm_size, tmp;
882 	int r = 0;
883 
884 	/* Validate virtual address range against reserved regions. */
885 	if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {
886 		dev_dbg(dev->dev,
887 			"va_address 0x%llx is in reserved area 0x%llx\n",
888 			args->va_address, AMDGPU_VA_RESERVED_BOTTOM);
889 		return -EINVAL;
890 	}
891 
892 	if (args->va_address >= AMDGPU_GMC_HOLE_START &&
893 	    args->va_address < AMDGPU_GMC_HOLE_END) {
894 		dev_dbg(dev->dev,
895 			"va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",
896 			args->va_address, AMDGPU_GMC_HOLE_START,
897 			AMDGPU_GMC_HOLE_END);
898 		return -EINVAL;
899 	}
900 
901 	args->va_address &= AMDGPU_GMC_HOLE_MASK;
902 
903 	vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
904 	vm_size -= AMDGPU_VA_RESERVED_TOP;
905 	if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) {
906 		dev_dbg(dev->dev,
907 			"va_address 0x%llx is in top reserved area 0x%llx\n",
908 			args->va_address + args->map_size, vm_size);
909 		return -EINVAL;
910 	}
911 
912 	if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
913 		dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
914 			args->flags);
915 		return -EINVAL;
916 	}
917 
918 	/* Validate operation type. */
919 	switch (args->operation) {
920 	case AMDGPU_VA_OP_MAP:
921 	case AMDGPU_VA_OP_UNMAP:
922 	case AMDGPU_VA_OP_CLEAR:
923 	case AMDGPU_VA_OP_REPLACE:
924 		break;
925 	default:
926 		dev_dbg(dev->dev, "unsupported operation %d\n",
927 			args->operation);
928 		return -EINVAL;
929 	}
930 
931 	if (args->flags & AMDGPU_VM_DELAY_UPDATE &&
932 	    args->vm_timeline_syncobj_out)
933 		return -EINVAL;
934 
935 	if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
936 	    !(args->flags & AMDGPU_VM_PAGE_PRT)) {
937 		gobj = drm_gem_object_lookup(filp, args->handle);
938 		if (gobj == NULL)
939 			return -ENOENT;
940 		abo = gem_to_amdgpu_bo(gobj);
941 	} else {
942 		gobj = NULL;
943 		abo = NULL;
944 	}
945 
946 	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
947 
948 	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
949 	if (r)
950 		goto error_put_gobj;
951 
952 	/* Add input syncobj fences (if any) for synchronization. */
953 	r = amdgpu_gem_add_input_fence(filp,
954 				       args->input_fence_syncobj_handles,
955 				       args->num_syncobj_handles);
956 	if (r)
957 		goto error_put_gobj;
958 
959 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
960 		      DRM_EXEC_IGNORE_DUPLICATES, 0);
961 	drm_exec_until_all_locked(&exec) {
962 		if (gobj) {
963 			r = drm_exec_lock_obj(&exec, gobj);
964 			drm_exec_retry_on_contention(&exec);
965 			if (unlikely(r))
966 				goto error;
967 		}
968 
969 		r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);
970 		drm_exec_retry_on_contention(&exec);
971 		if (unlikely(r))
972 			goto error;
973 	}
974 
975 	/* Resolve the BO-VA mapping for this VM/BO combination. */
976 	if (abo) {
977 		bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
978 		if (!bo_va) {
979 			r = -ENOENT;
980 			goto error;
981 		}
982 	} else if (args->operation != AMDGPU_VA_OP_CLEAR) {
983 		bo_va = fpriv->prt_va;
984 	} else {
985 		bo_va = NULL;
986 	}
987 
988 	/*
989 	 * Prepare the timeline syncobj node if the user requested a VM
990 	 * timeline update. This only allocates/looks up the syncobj and
991 	 * chain node; the actual fence is attached later.
992 	 */
993 	r = amdgpu_gem_update_timeline_node(filp,
994 					    args->vm_timeline_syncobj_out,
995 					    args->vm_timeline_point,
996 					    &timeline_syncobj,
997 					    &timeline_chain);
998 	if (r)
999 		goto error;
1000 
1001 	switch (args->operation) {
1002 	case AMDGPU_VA_OP_MAP:
1003 		r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
1004 				     args->offset_in_bo, args->map_size,
1005 				     args->flags);
1006 		break;
1007 	case AMDGPU_VA_OP_UNMAP:
1008 		r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
1009 		break;
1010 
1011 	case AMDGPU_VA_OP_CLEAR:
1012 		r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
1013 						args->va_address,
1014 						args->map_size);
1015 		break;
1016 	case AMDGPU_VA_OP_REPLACE:
1017 		r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
1018 					     args->offset_in_bo, args->map_size,
1019 					     args->flags);
1020 		break;
1021 	default:
1022 		break;
1023 	}
1024 
1025 	/*
1026 	 * Once the VA operation is done, update the VM and obtain the fence
1027 	 * that represents the last relevant update for this mapping. This
1028 	 * fence can then be exported to the user-visible VM timeline.
1029 	 */
1030 	if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) &&
1031 	    (!adev->debug_vm || timeline_syncobj)) {
1032 		struct dma_fence *fence;
1033 
1034 		fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
1035 						args->operation);
1036 		if (timeline_syncobj) {
1037 			if (!args->vm_timeline_point) {
1038 				/* Replace the existing fence when no point is given. */
1039 				drm_syncobj_replace_fence(timeline_syncobj,
1040 							  fence);
1041 			} else {
1042 				/* Attach the last-update fence at a specific point. */
1043 				drm_syncobj_add_point(timeline_syncobj,
1044 						      timeline_chain,
1045 						      fence,
1046 						      args->vm_timeline_point);
1047 				timeline_chain = NULL;
1048 			}
1049 		}
1050 		dma_fence_put(fence);
1051 
1052 	}
1053 
1054 error:
1055 	dma_fence_chain_free(timeline_chain);
1056 	if (timeline_syncobj)
1057 		drm_syncobj_put(timeline_syncobj);
1058 	drm_exec_fini(&exec);
1059 error_put_gobj:
1060 	drm_gem_object_put(gobj);
1061 	return r;
1062 }
1063 
1064 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
1065 			struct drm_file *filp)
1066 {
1067 	struct drm_amdgpu_gem_op *args = data;
1068 	struct drm_gem_object *gobj;
1069 	struct amdgpu_vm_bo_base *base;
1070 	struct amdgpu_bo *robj;
1071 	struct drm_exec exec;
1072 	struct amdgpu_fpriv *fpriv = filp->driver_priv;
1073 	int r;
1074 
1075 	if (args->padding)
1076 		return -EINVAL;
1077 
1078 	gobj = drm_gem_object_lookup(filp, args->handle);
1079 	if (!gobj)
1080 		return -ENOENT;
1081 
1082 	robj = gem_to_amdgpu_bo(gobj);
1083 
1084 	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
1085 			  DRM_EXEC_IGNORE_DUPLICATES, 0);
1086 	drm_exec_until_all_locked(&exec) {
1087 		r = drm_exec_lock_obj(&exec, gobj);
1088 		drm_exec_retry_on_contention(&exec);
1089 		if (r)
1090 			goto out_exec;
1091 
1092 		if (args->op == AMDGPU_GEM_OP_GET_MAPPING_INFO) {
1093 			r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 0);
1094 			drm_exec_retry_on_contention(&exec);
1095 			if (r)
1096 				goto out_exec;
1097 		}
1098 	}
1099 
1100 	switch (args->op) {
1101 	case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
1102 		struct drm_amdgpu_gem_create_in info;
1103 		void __user *out = u64_to_user_ptr(args->value);
1104 
1105 		info.bo_size = robj->tbo.base.size;
1106 		info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;
1107 		info.domains = robj->preferred_domains;
1108 		info.domain_flags = robj->flags;
1109 		drm_exec_fini(&exec);
1110 		if (copy_to_user(out, &info, sizeof(info)))
1111 			r = -EFAULT;
1112 		break;
1113 	}
1114 	case AMDGPU_GEM_OP_SET_PLACEMENT:
1115 		if (drm_gem_is_imported(&robj->tbo.base) &&
1116 		    args->value & AMDGPU_GEM_DOMAIN_VRAM) {
1117 			r = -EINVAL;
1118 			goto out_exec;
1119 		}
1120 		if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
1121 			r = -EPERM;
1122 			goto out_exec;
1123 		}
1124 		for (base = robj->vm_bo; base; base = base->next)
1125 			if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
1126 				amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {
1127 				r = -EINVAL;
1128 				goto out_exec;
1129 			}
1130 
1131 
1132 		robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
1133 							AMDGPU_GEM_DOMAIN_GTT |
1134 							AMDGPU_GEM_DOMAIN_CPU);
1135 		robj->allowed_domains = robj->preferred_domains;
1136 		if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
1137 			robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
1138 
1139 		if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
1140 			amdgpu_vm_bo_invalidate(robj, true);
1141 		drm_exec_fini(&exec);
1142 		break;
1143 	case AMDGPU_GEM_OP_GET_MAPPING_INFO: {
1144 		struct amdgpu_bo_va *bo_va = amdgpu_vm_bo_find(&fpriv->vm, robj);
1145 		struct drm_amdgpu_gem_vm_entry *vm_entries;
1146 		struct amdgpu_bo_va_mapping *mapping;
1147 		int num_mappings = 0;
1148 		/*
1149 		 * num_entries is set as an input to the size of the user-allocated array of
1150 		 * drm_amdgpu_gem_vm_entry stored at args->value.
1151 		 * num_entries is sent back as output as the number of mappings the bo has.
1152 		 * If that number is larger than the size of the array, the ioctl must
1153 		 * be retried.
1154 		 */
1155 		if (!bo_va) {
1156 			r = -ENOENT;
1157 			goto out_exec;
1158 		}
1159 
1160 		if (args->num_entries > INT_MAX / sizeof(*vm_entries)) {
1161 			r = -EINVAL;
1162 			goto out_exec;
1163 		}
1164 
1165 		vm_entries = kvcalloc(args->num_entries, sizeof(*vm_entries), GFP_KERNEL);
1166 		if (!vm_entries) {
1167 			r = -ENOMEM;
1168 			goto out_exec;
1169 		}
1170 
1171 		amdgpu_vm_bo_va_for_each_valid_mapping(bo_va, mapping) {
1172 			if (num_mappings < args->num_entries) {
1173 				vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE;
1174 				vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE;
1175 				vm_entries[num_mappings].offset = mapping->offset;
1176 				vm_entries[num_mappings].flags = mapping->flags;
1177 			}
1178 			num_mappings += 1;
1179 		}
1180 
1181 		amdgpu_vm_bo_va_for_each_invalid_mapping(bo_va, mapping) {
1182 			if (num_mappings < args->num_entries) {
1183 				vm_entries[num_mappings].addr = mapping->start * AMDGPU_GPU_PAGE_SIZE;
1184 				vm_entries[num_mappings].size = (mapping->last - mapping->start + 1) * AMDGPU_GPU_PAGE_SIZE;
1185 				vm_entries[num_mappings].offset = mapping->offset;
1186 				vm_entries[num_mappings].flags = mapping->flags;
1187 			}
1188 			num_mappings += 1;
1189 		}
1190 
1191 		drm_exec_fini(&exec);
1192 
1193 		if (num_mappings > 0 && num_mappings <= args->num_entries)
1194 			if (copy_to_user(u64_to_user_ptr(args->value), vm_entries, num_mappings * sizeof(*vm_entries)))
1195 				r = -EFAULT;
1196 
1197 		args->num_entries = num_mappings;
1198 
1199 		kvfree(vm_entries);
1200 		break;
1201 	}
1202 	default:
1203 		drm_exec_fini(&exec);
1204 		r = -EINVAL;
1205 	}
1206 
1207 	drm_gem_object_put(gobj);
1208 	return r;
1209 out_exec:
1210 	drm_exec_fini(&exec);
1211 	drm_gem_object_put(gobj);
1212 	return r;
1213 }
1214 
1215 /**
1216  * amdgpu_gem_list_handles_ioctl - get information about a process' buffer objects
1217  *
1218  * @dev: drm device pointer
1219  * @data: drm_amdgpu_gem_list_handles
1220  * @filp: drm file pointer
1221  *
1222  * num_entries is set as an input to the size of the entries array.
1223  * num_entries is sent back as output as the number of bos in the process.
1224  * If that number is larger than the size of the array, the ioctl must
1225  * be retried.
1226  *
1227  * Returns:
1228  * 0 for success, -errno for errors.
1229  */
1230 int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data,
1231 				  struct drm_file *filp)
1232 {
1233 	struct drm_amdgpu_gem_list_handles *args = data;
1234 	struct drm_amdgpu_gem_list_handles_entry *bo_entries;
1235 	struct drm_gem_object *gobj;
1236 	int id, ret = 0;
1237 	int bo_index = 0;
1238 	int num_bos = 0;
1239 
1240 	spin_lock(&filp->table_lock);
1241 	idr_for_each_entry(&filp->object_idr, gobj, id)
1242 		num_bos += 1;
1243 	spin_unlock(&filp->table_lock);
1244 
1245 	if (args->num_entries < num_bos) {
1246 		args->num_entries = num_bos;
1247 		return 0;
1248 	}
1249 
1250 	if (num_bos == 0) {
1251 		args->num_entries = 0;
1252 		return 0;
1253 	}
1254 
1255 	bo_entries = kvzalloc_objs(*bo_entries, num_bos);
1256 	if (!bo_entries)
1257 		return -ENOMEM;
1258 
1259 	spin_lock(&filp->table_lock);
1260 	idr_for_each_entry(&filp->object_idr, gobj, id) {
1261 		struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1262 		struct drm_amdgpu_gem_list_handles_entry *bo_entry;
1263 
1264 		if (bo_index >= num_bos) {
1265 			ret = -EAGAIN;
1266 			break;
1267 		}
1268 
1269 		bo_entry = &bo_entries[bo_index];
1270 
1271 		bo_entry->size = amdgpu_bo_size(bo);
1272 		bo_entry->alloc_flags = bo->flags & AMDGPU_GEM_CREATE_SETTABLE_MASK;
1273 		bo_entry->preferred_domains = bo->preferred_domains;
1274 		bo_entry->gem_handle = id;
1275 		bo_entry->alignment = (u64)bo->tbo.page_alignment << PAGE_SHIFT;
1276 
1277 		if (bo->tbo.base.import_attach)
1278 			bo_entry->flags |= AMDGPU_GEM_LIST_HANDLES_FLAG_IS_IMPORT;
1279 
1280 		bo_index += 1;
1281 	}
1282 	spin_unlock(&filp->table_lock);
1283 
1284 	args->num_entries = bo_index;
1285 
1286 	if (!ret)
1287 		if (copy_to_user(u64_to_user_ptr(args->entries), bo_entries, num_bos * sizeof(*bo_entries)))
1288 			ret = -EFAULT;
1289 
1290 	kvfree(bo_entries);
1291 
1292 	return ret;
1293 }
1294 
1295 static unsigned int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
1296 					   unsigned int width,
1297 					   unsigned int cpp,
1298 					   bool tiled)
1299 {
1300 	unsigned int aligned = width;
1301 	unsigned int pitch_mask = 0;
1302 	unsigned int pitch;
1303 
1304 	switch (cpp) {
1305 	case 1:
1306 		pitch_mask = 255;
1307 		break;
1308 	case 2:
1309 		pitch_mask = 127;
1310 		break;
1311 	case 3:
1312 	case 4:
1313 		pitch_mask = 63;
1314 		break;
1315 	}
1316 
1317 	if (check_add_overflow(aligned, pitch_mask, &aligned))
1318 		return 0;
1319 	aligned &= ~pitch_mask;
1320 	if (check_mul_overflow(aligned, cpp, &pitch))
1321 		return 0;
1322 	return pitch;
1323 }
1324 
1325 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
1326 			    struct drm_device *dev,
1327 			    struct drm_mode_create_dumb *args)
1328 {
1329 	struct amdgpu_device *adev = drm_to_adev(dev);
1330 	struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
1331 	struct drm_gem_object *gobj;
1332 	uint32_t handle;
1333 	u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1334 		    AMDGPU_GEM_CREATE_CPU_GTT_USWC |
1335 		    AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1336 	u32 domain;
1337 	int r;
1338 
1339 	/*
1340 	 * The buffer returned from this function should be cleared, but
1341 	 * it can only be done if the ring is enabled or we'll fail to
1342 	 * create the buffer.
1343 	 */
1344 	if (adev->mman.buffer_funcs_enabled)
1345 		flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
1346 
1347 	args->pitch = amdgpu_gem_align_pitch(adev, args->width,
1348 					     DIV_ROUND_UP(args->bpp, 8), 0);
1349 	if (!args->pitch)
1350 		return -EINVAL;
1351 	args->size = (u64)args->pitch * args->height;
1352 	args->size = ALIGN(args->size, PAGE_SIZE);
1353 	if (!args->size)
1354 		return -EINVAL;
1355 	domain = amdgpu_bo_get_preferred_domain(adev,
1356 				amdgpu_display_supported_domains(adev, flags));
1357 	r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
1358 				     ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
1359 	if (r)
1360 		return -ENOMEM;
1361 
1362 	r = drm_gem_handle_create(file_priv, gobj, &handle);
1363 	/* drop reference from allocate - handle holds it now */
1364 	drm_gem_object_put(gobj);
1365 	if (r)
1366 		return r;
1367 
1368 	args->handle = handle;
1369 	return 0;
1370 }
1371 
1372 #if defined(CONFIG_DEBUG_FS)
1373 static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
1374 {
1375 	struct amdgpu_device *adev = m->private;
1376 	struct drm_device *dev = adev_to_drm(adev);
1377 	struct drm_file *file;
1378 	int r;
1379 
1380 	r = mutex_lock_interruptible(&dev->filelist_mutex);
1381 	if (r)
1382 		return r;
1383 
1384 	list_for_each_entry(file, &dev->filelist, lhead) {
1385 		struct task_struct *task;
1386 		struct drm_gem_object *gobj;
1387 		struct pid *pid;
1388 		int id;
1389 
1390 		/*
1391 		 * Although we have a valid reference on file->pid, that does
1392 		 * not guarantee that the task_struct who called get_pid() is
1393 		 * still alive (e.g. get_pid(current) => fork() => exit()).
1394 		 * Therefore, we need to protect this ->comm access using RCU.
1395 		 */
1396 		rcu_read_lock();
1397 		pid = rcu_dereference(file->pid);
1398 		task = pid_task(pid, PIDTYPE_TGID);
1399 		seq_printf(m, "pid %8d command %s:\n", pid_nr(pid),
1400 			   task ? task->comm : "<unknown>");
1401 		rcu_read_unlock();
1402 
1403 		spin_lock(&file->table_lock);
1404 		idr_for_each_entry(&file->object_idr, gobj, id) {
1405 			struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1406 
1407 			amdgpu_bo_print_info(id, bo, m);
1408 		}
1409 		spin_unlock(&file->table_lock);
1410 	}
1411 
1412 	mutex_unlock(&dev->filelist_mutex);
1413 	return 0;
1414 }
1415 
1416 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);
1417 
1418 #endif
1419 
1420 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
1421 {
1422 #if defined(CONFIG_DEBUG_FS)
1423 	struct drm_minor *minor = adev_to_drm(adev)->primary;
1424 	struct dentry *root = minor->debugfs_root;
1425 
1426 	debugfs_create_file("amdgpu_gem_info", 0444, root, adev,
1427 			    &amdgpu_debugfs_gem_info_fops);
1428 #endif
1429 }
1430