1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 * Authors: Dave Airlie
25 * Alex Deucher
26 * Jerome Glisse
27 */
28 #include <linux/ktime.h>
29 #include <linux/module.h>
30 #include <linux/pagemap.h>
31 #include <linux/pci.h>
32 #include <linux/dma-buf.h>
33
34 #include <drm/amdgpu_drm.h>
35 #include <drm/drm_drv.h>
36 #include <drm/drm_exec.h>
37 #include <drm/drm_gem_ttm_helper.h>
38 #include <drm/ttm/ttm_tt.h>
39 #include <drm/drm_syncobj.h>
40
41 #include "amdgpu.h"
42 #include "amdgpu_display.h"
43 #include "amdgpu_dma_buf.h"
44 #include "amdgpu_hmm.h"
45 #include "amdgpu_xgmi.h"
46 #include "amdgpu_vm.h"
47
48 static int
amdgpu_gem_add_input_fence(struct drm_file * filp,uint64_t syncobj_handles_array,uint32_t num_syncobj_handles)49 amdgpu_gem_add_input_fence(struct drm_file *filp,
50 uint64_t syncobj_handles_array,
51 uint32_t num_syncobj_handles)
52 {
53 struct dma_fence *fence;
54 uint32_t *syncobj_handles;
55 int ret, i;
56
57 if (!num_syncobj_handles)
58 return 0;
59
60 syncobj_handles = memdup_user(u64_to_user_ptr(syncobj_handles_array),
61 size_mul(sizeof(uint32_t), num_syncobj_handles));
62 if (IS_ERR(syncobj_handles))
63 return PTR_ERR(syncobj_handles);
64
65 for (i = 0; i < num_syncobj_handles; i++) {
66
67 if (!syncobj_handles[i]) {
68 ret = -EINVAL;
69 goto free_memdup;
70 }
71
72 ret = drm_syncobj_find_fence(filp, syncobj_handles[i], 0, 0, &fence);
73 if (ret)
74 goto free_memdup;
75
76 dma_fence_wait(fence, false);
77
78 /* TODO: optimize async handling */
79 dma_fence_put(fence);
80 }
81
82 free_memdup:
83 kfree(syncobj_handles);
84 return ret;
85 }
86
87 static int
amdgpu_gem_update_timeline_node(struct drm_file * filp,uint32_t syncobj_handle,uint64_t point,struct drm_syncobj ** syncobj,struct dma_fence_chain ** chain)88 amdgpu_gem_update_timeline_node(struct drm_file *filp,
89 uint32_t syncobj_handle,
90 uint64_t point,
91 struct drm_syncobj **syncobj,
92 struct dma_fence_chain **chain)
93 {
94 if (!syncobj_handle)
95 return 0;
96
97 /* Find the sync object */
98 *syncobj = drm_syncobj_find(filp, syncobj_handle);
99 if (!*syncobj)
100 return -ENOENT;
101
102 if (!point)
103 return 0;
104
105 /* Allocate the chain node */
106 *chain = dma_fence_chain_alloc();
107 if (!*chain) {
108 drm_syncobj_put(*syncobj);
109 return -ENOMEM;
110 }
111
112 return 0;
113 }
114
115 static void
amdgpu_gem_update_bo_mapping(struct drm_file * filp,struct amdgpu_bo_va * bo_va,uint32_t operation,uint64_t point,struct dma_fence * fence,struct drm_syncobj * syncobj,struct dma_fence_chain * chain)116 amdgpu_gem_update_bo_mapping(struct drm_file *filp,
117 struct amdgpu_bo_va *bo_va,
118 uint32_t operation,
119 uint64_t point,
120 struct dma_fence *fence,
121 struct drm_syncobj *syncobj,
122 struct dma_fence_chain *chain)
123 {
124 struct amdgpu_bo *bo = bo_va ? bo_va->base.bo : NULL;
125 struct amdgpu_fpriv *fpriv = filp->driver_priv;
126 struct amdgpu_vm *vm = &fpriv->vm;
127 struct dma_fence *last_update;
128
129 if (!syncobj)
130 return;
131
132 /* Find the last update fence */
133 switch (operation) {
134 case AMDGPU_VA_OP_MAP:
135 case AMDGPU_VA_OP_REPLACE:
136 if (bo && (bo->tbo.base.resv == vm->root.bo->tbo.base.resv))
137 last_update = vm->last_update;
138 else
139 last_update = bo_va->last_pt_update;
140 break;
141 case AMDGPU_VA_OP_UNMAP:
142 case AMDGPU_VA_OP_CLEAR:
143 last_update = fence;
144 break;
145 default:
146 return;
147 }
148
149 /* Add fence to timeline */
150 if (!point)
151 drm_syncobj_replace_fence(syncobj, last_update);
152 else
153 drm_syncobj_add_point(syncobj, chain, last_update, point);
154 }
155
amdgpu_gem_fault(struct vm_fault * vmf)156 static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
157 {
158 struct ttm_buffer_object *bo = vmf->vma->vm_private_data;
159 struct drm_device *ddev = bo->base.dev;
160 vm_fault_t ret;
161 int idx;
162
163 ret = ttm_bo_vm_reserve(bo, vmf);
164 if (ret)
165 return ret;
166
167 if (drm_dev_enter(ddev, &idx)) {
168 ret = amdgpu_bo_fault_reserve_notify(bo);
169 if (ret) {
170 drm_dev_exit(idx);
171 goto unlock;
172 }
173
174 ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot,
175 TTM_BO_VM_NUM_PREFAULT);
176
177 drm_dev_exit(idx);
178 } else {
179 ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
180 }
181 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
182 return ret;
183
184 unlock:
185 dma_resv_unlock(bo->base.resv);
186 return ret;
187 }
188
189 static const struct vm_operations_struct amdgpu_gem_vm_ops = {
190 .fault = amdgpu_gem_fault,
191 .open = ttm_bo_vm_open,
192 .close = ttm_bo_vm_close,
193 .access = ttm_bo_vm_access
194 };
195
amdgpu_gem_object_free(struct drm_gem_object * gobj)196 static void amdgpu_gem_object_free(struct drm_gem_object *gobj)
197 {
198 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(gobj);
199
200 amdgpu_hmm_unregister(aobj);
201 ttm_bo_put(&aobj->tbo);
202 }
203
amdgpu_gem_object_create(struct amdgpu_device * adev,unsigned long size,int alignment,u32 initial_domain,u64 flags,enum ttm_bo_type type,struct dma_resv * resv,struct drm_gem_object ** obj,int8_t xcp_id_plus1)204 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size,
205 int alignment, u32 initial_domain,
206 u64 flags, enum ttm_bo_type type,
207 struct dma_resv *resv,
208 struct drm_gem_object **obj, int8_t xcp_id_plus1)
209 {
210 struct amdgpu_bo *bo;
211 struct amdgpu_bo_user *ubo;
212 struct amdgpu_bo_param bp;
213 int r;
214
215 memset(&bp, 0, sizeof(bp));
216 *obj = NULL;
217 flags |= AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
218
219 bp.size = size;
220 bp.byte_align = alignment;
221 bp.type = type;
222 bp.resv = resv;
223 bp.preferred_domain = initial_domain;
224 bp.flags = flags;
225 bp.domain = initial_domain;
226 bp.bo_ptr_size = sizeof(struct amdgpu_bo);
227 bp.xcp_id_plus1 = xcp_id_plus1;
228
229 r = amdgpu_bo_create_user(adev, &bp, &ubo);
230 if (r)
231 return r;
232
233 bo = &ubo->bo;
234 *obj = &bo->tbo.base;
235
236 return 0;
237 }
238
amdgpu_gem_force_release(struct amdgpu_device * adev)239 void amdgpu_gem_force_release(struct amdgpu_device *adev)
240 {
241 struct drm_device *ddev = adev_to_drm(adev);
242 struct drm_file *file;
243
244 mutex_lock(&ddev->filelist_mutex);
245
246 list_for_each_entry(file, &ddev->filelist, lhead) {
247 struct drm_gem_object *gobj;
248 int handle;
249
250 WARN_ONCE(1, "Still active user space clients!\n");
251 spin_lock(&file->table_lock);
252 idr_for_each_entry(&file->object_idr, gobj, handle) {
253 WARN_ONCE(1, "And also active allocations!\n");
254 drm_gem_object_put(gobj);
255 }
256 idr_destroy(&file->object_idr);
257 spin_unlock(&file->table_lock);
258 }
259
260 mutex_unlock(&ddev->filelist_mutex);
261 }
262
263 /*
264 * Call from drm_gem_handle_create which appear in both new and open ioctl
265 * case.
266 */
amdgpu_gem_object_open(struct drm_gem_object * obj,struct drm_file * file_priv)267 static int amdgpu_gem_object_open(struct drm_gem_object *obj,
268 struct drm_file *file_priv)
269 {
270 struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj);
271 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev);
272 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
273 struct amdgpu_vm *vm = &fpriv->vm;
274 struct amdgpu_bo_va *bo_va;
275 struct mm_struct *mm;
276 int r;
277
278 mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm);
279 if (mm && mm != current->mm)
280 return -EPERM;
281
282 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID &&
283 !amdgpu_vm_is_bo_always_valid(vm, abo))
284 return -EPERM;
285
286 r = amdgpu_bo_reserve(abo, false);
287 if (r)
288 return r;
289
290 amdgpu_vm_bo_update_shared(abo);
291 bo_va = amdgpu_vm_bo_find(vm, abo);
292 if (!bo_va)
293 bo_va = amdgpu_vm_bo_add(adev, vm, abo);
294 else
295 ++bo_va->ref_count;
296
297 /* attach gfx eviction fence */
298 r = amdgpu_eviction_fence_attach(&fpriv->evf_mgr, abo);
299 if (r) {
300 DRM_DEBUG_DRIVER("Failed to attach eviction fence to BO\n");
301 amdgpu_bo_unreserve(abo);
302 return r;
303 }
304
305 amdgpu_bo_unreserve(abo);
306
307 /* Validate and add eviction fence to DMABuf imports with dynamic
308 * attachment in compute VMs. Re-validation will be done by
309 * amdgpu_vm_validate. Fences are on the reservation shared with the
310 * export, which is currently required to be validated and fenced
311 * already by amdgpu_amdkfd_gpuvm_restore_process_bos.
312 *
313 * Nested locking below for the case that a GEM object is opened in
314 * kfd_mem_export_dmabuf. Since the lock below is only taken for imports,
315 * but not for export, this is a different lock class that cannot lead to
316 * circular lock dependencies.
317 */
318 if (!vm->is_compute_context || !vm->process_info)
319 return 0;
320 if (!obj->import_attach ||
321 !dma_buf_is_dynamic(obj->import_attach->dmabuf))
322 return 0;
323 mutex_lock_nested(&vm->process_info->lock, 1);
324 if (!WARN_ON(!vm->process_info->eviction_fence)) {
325 r = amdgpu_amdkfd_bo_validate_and_fence(abo, AMDGPU_GEM_DOMAIN_GTT,
326 &vm->process_info->eviction_fence->base);
327 if (r) {
328 struct amdgpu_task_info *ti = amdgpu_vm_get_task_info_vm(vm);
329
330 dev_warn(adev->dev, "validate_and_fence failed: %d\n", r);
331 if (ti) {
332 dev_warn(adev->dev, "pid %d\n", ti->pid);
333 amdgpu_vm_put_task_info(ti);
334 }
335 }
336 }
337 mutex_unlock(&vm->process_info->lock);
338
339 return r;
340 }
341
amdgpu_gem_object_close(struct drm_gem_object * obj,struct drm_file * file_priv)342 static void amdgpu_gem_object_close(struct drm_gem_object *obj,
343 struct drm_file *file_priv)
344 {
345 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
346 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
347 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
348 struct amdgpu_vm *vm = &fpriv->vm;
349
350 struct dma_fence *fence = NULL;
351 struct amdgpu_bo_va *bo_va;
352 struct drm_exec exec;
353 long r;
354
355 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0);
356 drm_exec_until_all_locked(&exec) {
357 r = drm_exec_prepare_obj(&exec, &bo->tbo.base, 1);
358 drm_exec_retry_on_contention(&exec);
359 if (unlikely(r))
360 goto out_unlock;
361
362 r = amdgpu_vm_lock_pd(vm, &exec, 0);
363 drm_exec_retry_on_contention(&exec);
364 if (unlikely(r))
365 goto out_unlock;
366 }
367
368 if (!amdgpu_vm_is_bo_always_valid(vm, bo))
369 amdgpu_eviction_fence_detach(&fpriv->evf_mgr, bo);
370
371 bo_va = amdgpu_vm_bo_find(vm, bo);
372 if (!bo_va || --bo_va->ref_count)
373 goto out_unlock;
374
375 amdgpu_vm_bo_del(adev, bo_va);
376 amdgpu_vm_bo_update_shared(bo);
377 if (!amdgpu_vm_ready(vm))
378 goto out_unlock;
379
380 r = amdgpu_vm_clear_freed(adev, vm, &fence);
381 if (unlikely(r < 0))
382 dev_err(adev->dev, "failed to clear page "
383 "tables on GEM object close (%ld)\n", r);
384 if (r || !fence)
385 goto out_unlock;
386
387 amdgpu_bo_fence(bo, fence, true);
388 dma_fence_put(fence);
389
390 out_unlock:
391 if (r)
392 dev_err(adev->dev, "leaking bo va (%ld)\n", r);
393 drm_exec_fini(&exec);
394 }
395
amdgpu_gem_object_mmap(struct drm_gem_object * obj,struct vm_area_struct * vma)396 static int amdgpu_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
397 {
398 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
399
400 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm))
401 return -EPERM;
402 if (bo->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)
403 return -EPERM;
404
405 /* Workaround for Thunk bug creating PROT_NONE,MAP_PRIVATE mappings
406 * for debugger access to invisible VRAM. Should have used MAP_SHARED
407 * instead. Clearing VM_MAYWRITE prevents the mapping from ever
408 * becoming writable and makes is_cow_mapping(vm_flags) false.
409 */
410 if (is_cow_mapping(vma->vm_flags) &&
411 !(vma->vm_flags & VM_ACCESS_FLAGS))
412 vm_flags_clear(vma, VM_MAYWRITE);
413
414 return drm_gem_ttm_mmap(obj, vma);
415 }
416
417 const struct drm_gem_object_funcs amdgpu_gem_object_funcs = {
418 .free = amdgpu_gem_object_free,
419 .open = amdgpu_gem_object_open,
420 .close = amdgpu_gem_object_close,
421 .export = amdgpu_gem_prime_export,
422 .vmap = drm_gem_ttm_vmap,
423 .vunmap = drm_gem_ttm_vunmap,
424 .mmap = amdgpu_gem_object_mmap,
425 .vm_ops = &amdgpu_gem_vm_ops,
426 };
427
428 /*
429 * GEM ioctls.
430 */
amdgpu_gem_create_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)431 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
432 struct drm_file *filp)
433 {
434 struct amdgpu_device *adev = drm_to_adev(dev);
435 struct amdgpu_fpriv *fpriv = filp->driver_priv;
436 struct amdgpu_vm *vm = &fpriv->vm;
437 union drm_amdgpu_gem_create *args = data;
438 uint64_t flags = args->in.domain_flags;
439 uint64_t size = args->in.bo_size;
440 struct dma_resv *resv = NULL;
441 struct drm_gem_object *gobj;
442 uint32_t handle, initial_domain;
443 int r;
444
445 /* reject invalid gem flags */
446 if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
447 AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
448 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
449 AMDGPU_GEM_CREATE_VRAM_CLEARED |
450 AMDGPU_GEM_CREATE_VM_ALWAYS_VALID |
451 AMDGPU_GEM_CREATE_EXPLICIT_SYNC |
452 AMDGPU_GEM_CREATE_ENCRYPTED |
453 AMDGPU_GEM_CREATE_GFX12_DCC |
454 AMDGPU_GEM_CREATE_DISCARDABLE))
455 return -EINVAL;
456
457 /* reject invalid gem domains */
458 if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK)
459 return -EINVAL;
460
461 if (!amdgpu_is_tmz(adev) && (flags & AMDGPU_GEM_CREATE_ENCRYPTED)) {
462 DRM_NOTE_ONCE("Cannot allocate secure buffer since TMZ is disabled\n");
463 return -EINVAL;
464 }
465
466 /* always clear VRAM */
467 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
468
469 /* create a gem object to contain this object in */
470 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS |
471 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) {
472 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
473 /* if gds bo is created from user space, it must be
474 * passed to bo list
475 */
476 DRM_ERROR("GDS bo cannot be per-vm-bo\n");
477 return -EINVAL;
478 }
479 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
480 }
481
482 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
483 r = amdgpu_bo_reserve(vm->root.bo, false);
484 if (r)
485 return r;
486
487 resv = vm->root.bo->tbo.base.resv;
488 }
489
490 initial_domain = (u32)(0xffffffff & args->in.domains);
491 retry:
492 r = amdgpu_gem_object_create(adev, size, args->in.alignment,
493 initial_domain,
494 flags, ttm_bo_type_device, resv, &gobj, fpriv->xcp_id + 1);
495 if (r && r != -ERESTARTSYS) {
496 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) {
497 flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
498 goto retry;
499 }
500
501 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) {
502 initial_domain |= AMDGPU_GEM_DOMAIN_GTT;
503 goto retry;
504 }
505 DRM_DEBUG("Failed to allocate GEM object (%llu, %d, %llu, %d)\n",
506 size, initial_domain, args->in.alignment, r);
507 }
508
509 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) {
510 if (!r) {
511 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj);
512
513 abo->parent = amdgpu_bo_ref(vm->root.bo);
514 }
515 amdgpu_bo_unreserve(vm->root.bo);
516 }
517 if (r)
518 return r;
519
520 r = drm_gem_handle_create(filp, gobj, &handle);
521 /* drop reference from allocate - handle holds it now */
522 drm_gem_object_put(gobj);
523 if (r)
524 return r;
525
526 memset(args, 0, sizeof(*args));
527 args->out.handle = handle;
528 return 0;
529 }
530
amdgpu_gem_userptr_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)531 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
532 struct drm_file *filp)
533 {
534 struct ttm_operation_ctx ctx = { true, false };
535 struct amdgpu_device *adev = drm_to_adev(dev);
536 struct drm_amdgpu_gem_userptr *args = data;
537 struct amdgpu_fpriv *fpriv = filp->driver_priv;
538 struct drm_gem_object *gobj;
539 struct hmm_range *range;
540 struct amdgpu_bo *bo;
541 uint32_t handle;
542 int r;
543
544 args->addr = untagged_addr(args->addr);
545
546 if (offset_in_page(args->addr | args->size))
547 return -EINVAL;
548
549 /* reject unknown flag values */
550 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
551 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
552 AMDGPU_GEM_USERPTR_REGISTER))
553 return -EINVAL;
554
555 if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) &&
556 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) {
557
558 /* if we want to write to it we must install a MMU notifier */
559 return -EACCES;
560 }
561
562 /* create a gem object to contain this object in */
563 r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU,
564 0, ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
565 if (r)
566 return r;
567
568 bo = gem_to_amdgpu_bo(gobj);
569 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT;
570 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT;
571 r = amdgpu_ttm_tt_set_userptr(&bo->tbo, args->addr, args->flags);
572 if (r)
573 goto release_object;
574
575 r = amdgpu_hmm_register(bo, args->addr);
576 if (r)
577 goto release_object;
578
579 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) {
580 r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages,
581 &range);
582 if (r)
583 goto release_object;
584
585 r = amdgpu_bo_reserve(bo, true);
586 if (r)
587 goto user_pages_done;
588
589 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
590 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
591 amdgpu_bo_unreserve(bo);
592 if (r)
593 goto user_pages_done;
594 }
595
596 r = drm_gem_handle_create(filp, gobj, &handle);
597 if (r)
598 goto user_pages_done;
599
600 args->handle = handle;
601
602 user_pages_done:
603 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE)
604 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm, range);
605
606 release_object:
607 drm_gem_object_put(gobj);
608
609 return r;
610 }
611
amdgpu_mode_dumb_mmap(struct drm_file * filp,struct drm_device * dev,uint32_t handle,uint64_t * offset_p)612 int amdgpu_mode_dumb_mmap(struct drm_file *filp,
613 struct drm_device *dev,
614 uint32_t handle, uint64_t *offset_p)
615 {
616 struct drm_gem_object *gobj;
617 struct amdgpu_bo *robj;
618
619 gobj = drm_gem_object_lookup(filp, handle);
620 if (!gobj)
621 return -ENOENT;
622
623 robj = gem_to_amdgpu_bo(gobj);
624 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) ||
625 (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) {
626 drm_gem_object_put(gobj);
627 return -EPERM;
628 }
629 *offset_p = amdgpu_bo_mmap_offset(robj);
630 drm_gem_object_put(gobj);
631 return 0;
632 }
633
amdgpu_gem_mmap_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)634 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data,
635 struct drm_file *filp)
636 {
637 union drm_amdgpu_gem_mmap *args = data;
638 uint32_t handle = args->in.handle;
639
640 memset(args, 0, sizeof(*args));
641 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr);
642 }
643
644 /**
645 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value
646 *
647 * @timeout_ns: timeout in ns
648 *
649 * Calculate the timeout in jiffies from an absolute timeout in ns.
650 */
amdgpu_gem_timeout(uint64_t timeout_ns)651 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns)
652 {
653 unsigned long timeout_jiffies;
654 ktime_t timeout;
655
656 /* clamp timeout if it's to large */
657 if (((int64_t)timeout_ns) < 0)
658 return MAX_SCHEDULE_TIMEOUT;
659
660 timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get());
661 if (ktime_to_ns(timeout) < 0)
662 return 0;
663
664 timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout));
665 /* clamp timeout to avoid unsigned-> signed overflow */
666 if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT)
667 return MAX_SCHEDULE_TIMEOUT - 1;
668
669 return timeout_jiffies;
670 }
671
amdgpu_gem_wait_idle_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)672 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
673 struct drm_file *filp)
674 {
675 union drm_amdgpu_gem_wait_idle *args = data;
676 struct drm_gem_object *gobj;
677 struct amdgpu_bo *robj;
678 uint32_t handle = args->in.handle;
679 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout);
680 int r = 0;
681 long ret;
682
683 gobj = drm_gem_object_lookup(filp, handle);
684 if (!gobj)
685 return -ENOENT;
686
687 robj = gem_to_amdgpu_bo(gobj);
688 ret = dma_resv_wait_timeout(robj->tbo.base.resv, DMA_RESV_USAGE_READ,
689 true, timeout);
690
691 /* ret == 0 means not signaled,
692 * ret > 0 means signaled
693 * ret < 0 means interrupted before timeout
694 */
695 if (ret >= 0) {
696 memset(args, 0, sizeof(*args));
697 args->out.status = (ret == 0);
698 } else
699 r = ret;
700
701 drm_gem_object_put(gobj);
702 return r;
703 }
704
amdgpu_gem_metadata_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)705 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data,
706 struct drm_file *filp)
707 {
708 struct drm_amdgpu_gem_metadata *args = data;
709 struct drm_gem_object *gobj;
710 struct amdgpu_bo *robj;
711 int r = -1;
712
713 DRM_DEBUG("%d\n", args->handle);
714 gobj = drm_gem_object_lookup(filp, args->handle);
715 if (gobj == NULL)
716 return -ENOENT;
717 robj = gem_to_amdgpu_bo(gobj);
718
719 r = amdgpu_bo_reserve(robj, false);
720 if (unlikely(r != 0))
721 goto out;
722
723 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) {
724 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info);
725 r = amdgpu_bo_get_metadata(robj, args->data.data,
726 sizeof(args->data.data),
727 &args->data.data_size_bytes,
728 &args->data.flags);
729 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) {
730 if (args->data.data_size_bytes > sizeof(args->data.data)) {
731 r = -EINVAL;
732 goto unreserve;
733 }
734 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info);
735 if (!r)
736 r = amdgpu_bo_set_metadata(robj, args->data.data,
737 args->data.data_size_bytes,
738 args->data.flags);
739 }
740
741 unreserve:
742 amdgpu_bo_unreserve(robj);
743 out:
744 drm_gem_object_put(gobj);
745 return r;
746 }
747
748 /**
749 * amdgpu_gem_va_update_vm -update the bo_va in its VM
750 *
751 * @adev: amdgpu_device pointer
752 * @vm: vm to update
753 * @bo_va: bo_va to update
754 * @operation: map, unmap or clear
755 *
756 * Update the bo_va directly after setting its address. Errors are not
757 * vital here, so they are not reported back to userspace.
758 *
759 * Returns resulting fence if freed BO(s) got cleared from the PT.
760 * otherwise stub fence in case of error.
761 */
762 static struct dma_fence *
amdgpu_gem_va_update_vm(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo_va * bo_va,uint32_t operation)763 amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
764 struct amdgpu_vm *vm,
765 struct amdgpu_bo_va *bo_va,
766 uint32_t operation)
767 {
768 struct dma_fence *fence = dma_fence_get_stub();
769 int r;
770
771 if (!amdgpu_vm_ready(vm))
772 return fence;
773
774 r = amdgpu_vm_clear_freed(adev, vm, &fence);
775 if (r)
776 goto error;
777
778 if (operation == AMDGPU_VA_OP_MAP ||
779 operation == AMDGPU_VA_OP_REPLACE) {
780 r = amdgpu_vm_bo_update(adev, bo_va, false);
781 if (r)
782 goto error;
783 }
784
785 r = amdgpu_vm_update_pdes(adev, vm, false);
786
787 error:
788 if (r && r != -ERESTARTSYS)
789 DRM_ERROR("Couldn't update BO_VA (%d)\n", r);
790
791 return fence;
792 }
793
794 /**
795 * amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags
796 *
797 * @adev: amdgpu_device pointer
798 * @flags: GEM UAPI flags
799 *
800 * Returns the GEM UAPI flags mapped into hardware for the ASIC.
801 */
amdgpu_gem_va_map_flags(struct amdgpu_device * adev,uint32_t flags)802 uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags)
803 {
804 uint64_t pte_flag = 0;
805
806 if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
807 pte_flag |= AMDGPU_PTE_EXECUTABLE;
808 if (flags & AMDGPU_VM_PAGE_READABLE)
809 pte_flag |= AMDGPU_PTE_READABLE;
810 if (flags & AMDGPU_VM_PAGE_WRITEABLE)
811 pte_flag |= AMDGPU_PTE_WRITEABLE;
812 if (flags & AMDGPU_VM_PAGE_PRT)
813 pte_flag |= AMDGPU_PTE_PRT_FLAG(adev);
814 if (flags & AMDGPU_VM_PAGE_NOALLOC)
815 pte_flag |= AMDGPU_PTE_NOALLOC;
816
817 if (adev->gmc.gmc_funcs->map_mtype)
818 pte_flag |= amdgpu_gmc_map_mtype(adev,
819 flags & AMDGPU_VM_MTYPE_MASK);
820
821 return pte_flag;
822 }
823
amdgpu_gem_va_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)824 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
825 struct drm_file *filp)
826 {
827 const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
828 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
829 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
830 AMDGPU_VM_PAGE_NOALLOC;
831 const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
832 AMDGPU_VM_PAGE_PRT;
833
834 struct drm_amdgpu_gem_va *args = data;
835 struct drm_gem_object *gobj;
836 struct amdgpu_device *adev = drm_to_adev(dev);
837 struct amdgpu_fpriv *fpriv = filp->driver_priv;
838 struct amdgpu_bo *abo;
839 struct amdgpu_bo_va *bo_va;
840 struct drm_syncobj *timeline_syncobj = NULL;
841 struct dma_fence_chain *timeline_chain = NULL;
842 struct dma_fence *fence;
843 struct drm_exec exec;
844 uint64_t va_flags;
845 uint64_t vm_size;
846 int r = 0;
847
848 if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {
849 dev_dbg(dev->dev,
850 "va_address 0x%llx is in reserved area 0x%llx\n",
851 args->va_address, AMDGPU_VA_RESERVED_BOTTOM);
852 return -EINVAL;
853 }
854
855 if (args->va_address >= AMDGPU_GMC_HOLE_START &&
856 args->va_address < AMDGPU_GMC_HOLE_END) {
857 dev_dbg(dev->dev,
858 "va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",
859 args->va_address, AMDGPU_GMC_HOLE_START,
860 AMDGPU_GMC_HOLE_END);
861 return -EINVAL;
862 }
863
864 args->va_address &= AMDGPU_GMC_HOLE_MASK;
865
866 vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
867 vm_size -= AMDGPU_VA_RESERVED_TOP;
868 if (args->va_address + args->map_size > vm_size) {
869 dev_dbg(dev->dev,
870 "va_address 0x%llx is in top reserved area 0x%llx\n",
871 args->va_address + args->map_size, vm_size);
872 return -EINVAL;
873 }
874
875 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
876 dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
877 args->flags);
878 return -EINVAL;
879 }
880
881 switch (args->operation) {
882 case AMDGPU_VA_OP_MAP:
883 case AMDGPU_VA_OP_UNMAP:
884 case AMDGPU_VA_OP_CLEAR:
885 case AMDGPU_VA_OP_REPLACE:
886 break;
887 default:
888 dev_dbg(dev->dev, "unsupported operation %d\n",
889 args->operation);
890 return -EINVAL;
891 }
892
893 if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
894 !(args->flags & AMDGPU_VM_PAGE_PRT)) {
895 gobj = drm_gem_object_lookup(filp, args->handle);
896 if (gobj == NULL)
897 return -ENOENT;
898 abo = gem_to_amdgpu_bo(gobj);
899 } else {
900 gobj = NULL;
901 abo = NULL;
902 }
903
904 r = amdgpu_gem_add_input_fence(filp,
905 args->input_fence_syncobj_handles,
906 args->num_syncobj_handles);
907 if (r)
908 goto error_put_gobj;
909
910 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
911 DRM_EXEC_IGNORE_DUPLICATES, 0);
912 drm_exec_until_all_locked(&exec) {
913 if (gobj) {
914 r = drm_exec_lock_obj(&exec, gobj);
915 drm_exec_retry_on_contention(&exec);
916 if (unlikely(r))
917 goto error;
918 }
919
920 r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);
921 drm_exec_retry_on_contention(&exec);
922 if (unlikely(r))
923 goto error;
924 }
925
926 if (abo) {
927 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
928 if (!bo_va) {
929 r = -ENOENT;
930 goto error;
931 }
932 } else if (args->operation != AMDGPU_VA_OP_CLEAR) {
933 bo_va = fpriv->prt_va;
934 } else {
935 bo_va = NULL;
936 }
937
938 r = amdgpu_gem_update_timeline_node(filp,
939 args->vm_timeline_syncobj_out,
940 args->vm_timeline_point,
941 &timeline_syncobj,
942 &timeline_chain);
943 if (r)
944 goto error;
945
946 switch (args->operation) {
947 case AMDGPU_VA_OP_MAP:
948 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
949 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
950 args->offset_in_bo, args->map_size,
951 va_flags);
952 break;
953 case AMDGPU_VA_OP_UNMAP:
954 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
955 break;
956
957 case AMDGPU_VA_OP_CLEAR:
958 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
959 args->va_address,
960 args->map_size);
961 break;
962 case AMDGPU_VA_OP_REPLACE:
963 va_flags = amdgpu_gem_va_map_flags(adev, args->flags);
964 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
965 args->offset_in_bo, args->map_size,
966 va_flags);
967 break;
968 default:
969 break;
970 }
971 if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !adev->debug_vm) {
972 fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
973 args->operation);
974
975 if (timeline_syncobj)
976 amdgpu_gem_update_bo_mapping(filp, bo_va,
977 args->operation,
978 args->vm_timeline_point,
979 fence, timeline_syncobj,
980 timeline_chain);
981 else
982 dma_fence_put(fence);
983
984 }
985
986 error:
987 drm_exec_fini(&exec);
988 error_put_gobj:
989 drm_gem_object_put(gobj);
990 return r;
991 }
992
amdgpu_gem_op_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)993 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data,
994 struct drm_file *filp)
995 {
996 struct drm_amdgpu_gem_op *args = data;
997 struct drm_gem_object *gobj;
998 struct amdgpu_vm_bo_base *base;
999 struct amdgpu_bo *robj;
1000 int r;
1001
1002 gobj = drm_gem_object_lookup(filp, args->handle);
1003 if (!gobj)
1004 return -ENOENT;
1005
1006 robj = gem_to_amdgpu_bo(gobj);
1007
1008 r = amdgpu_bo_reserve(robj, false);
1009 if (unlikely(r))
1010 goto out;
1011
1012 switch (args->op) {
1013 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: {
1014 struct drm_amdgpu_gem_create_in info;
1015 void __user *out = u64_to_user_ptr(args->value);
1016
1017 info.bo_size = robj->tbo.base.size;
1018 info.alignment = robj->tbo.page_alignment << PAGE_SHIFT;
1019 info.domains = robj->preferred_domains;
1020 info.domain_flags = robj->flags;
1021 amdgpu_bo_unreserve(robj);
1022 if (copy_to_user(out, &info, sizeof(info)))
1023 r = -EFAULT;
1024 break;
1025 }
1026 case AMDGPU_GEM_OP_SET_PLACEMENT:
1027 if (robj->tbo.base.import_attach &&
1028 args->value & AMDGPU_GEM_DOMAIN_VRAM) {
1029 r = -EINVAL;
1030 amdgpu_bo_unreserve(robj);
1031 break;
1032 }
1033 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) {
1034 r = -EPERM;
1035 amdgpu_bo_unreserve(robj);
1036 break;
1037 }
1038 for (base = robj->vm_bo; base; base = base->next)
1039 if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev),
1040 amdgpu_ttm_adev(base->vm->root.bo->tbo.bdev))) {
1041 r = -EINVAL;
1042 amdgpu_bo_unreserve(robj);
1043 goto out;
1044 }
1045
1046
1047 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM |
1048 AMDGPU_GEM_DOMAIN_GTT |
1049 AMDGPU_GEM_DOMAIN_CPU);
1050 robj->allowed_domains = robj->preferred_domains;
1051 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM)
1052 robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT;
1053
1054 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID)
1055 amdgpu_vm_bo_invalidate(robj, true);
1056
1057 amdgpu_bo_unreserve(robj);
1058 break;
1059 default:
1060 amdgpu_bo_unreserve(robj);
1061 r = -EINVAL;
1062 }
1063
1064 out:
1065 drm_gem_object_put(gobj);
1066 return r;
1067 }
1068
amdgpu_gem_align_pitch(struct amdgpu_device * adev,int width,int cpp,bool tiled)1069 static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
1070 int width,
1071 int cpp,
1072 bool tiled)
1073 {
1074 int aligned = width;
1075 int pitch_mask = 0;
1076
1077 switch (cpp) {
1078 case 1:
1079 pitch_mask = 255;
1080 break;
1081 case 2:
1082 pitch_mask = 127;
1083 break;
1084 case 3:
1085 case 4:
1086 pitch_mask = 63;
1087 break;
1088 }
1089
1090 aligned += pitch_mask;
1091 aligned &= ~pitch_mask;
1092 return aligned * cpp;
1093 }
1094
amdgpu_mode_dumb_create(struct drm_file * file_priv,struct drm_device * dev,struct drm_mode_create_dumb * args)1095 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
1096 struct drm_device *dev,
1097 struct drm_mode_create_dumb *args)
1098 {
1099 struct amdgpu_device *adev = drm_to_adev(dev);
1100 struct amdgpu_fpriv *fpriv = file_priv->driver_priv;
1101 struct drm_gem_object *gobj;
1102 uint32_t handle;
1103 u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED |
1104 AMDGPU_GEM_CREATE_CPU_GTT_USWC |
1105 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
1106 u32 domain;
1107 int r;
1108
1109 /*
1110 * The buffer returned from this function should be cleared, but
1111 * it can only be done if the ring is enabled or we'll fail to
1112 * create the buffer.
1113 */
1114 if (adev->mman.buffer_funcs_enabled)
1115 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED;
1116
1117 args->pitch = amdgpu_gem_align_pitch(adev, args->width,
1118 DIV_ROUND_UP(args->bpp, 8), 0);
1119 args->size = (u64)args->pitch * args->height;
1120 args->size = ALIGN(args->size, PAGE_SIZE);
1121 domain = amdgpu_bo_get_preferred_domain(adev,
1122 amdgpu_display_supported_domains(adev, flags));
1123 r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
1124 ttm_bo_type_device, NULL, &gobj, fpriv->xcp_id + 1);
1125 if (r)
1126 return -ENOMEM;
1127
1128 r = drm_gem_handle_create(file_priv, gobj, &handle);
1129 /* drop reference from allocate - handle holds it now */
1130 drm_gem_object_put(gobj);
1131 if (r)
1132 return r;
1133
1134 args->handle = handle;
1135 return 0;
1136 }
1137
1138 #if defined(CONFIG_DEBUG_FS)
amdgpu_debugfs_gem_info_show(struct seq_file * m,void * unused)1139 static int amdgpu_debugfs_gem_info_show(struct seq_file *m, void *unused)
1140 {
1141 struct amdgpu_device *adev = m->private;
1142 struct drm_device *dev = adev_to_drm(adev);
1143 struct drm_file *file;
1144 int r;
1145
1146 r = mutex_lock_interruptible(&dev->filelist_mutex);
1147 if (r)
1148 return r;
1149
1150 list_for_each_entry(file, &dev->filelist, lhead) {
1151 struct task_struct *task;
1152 struct drm_gem_object *gobj;
1153 struct pid *pid;
1154 int id;
1155
1156 /*
1157 * Although we have a valid reference on file->pid, that does
1158 * not guarantee that the task_struct who called get_pid() is
1159 * still alive (e.g. get_pid(current) => fork() => exit()).
1160 * Therefore, we need to protect this ->comm access using RCU.
1161 */
1162 rcu_read_lock();
1163 pid = rcu_dereference(file->pid);
1164 task = pid_task(pid, PIDTYPE_TGID);
1165 seq_printf(m, "pid %8d command %s:\n", pid_nr(pid),
1166 task ? task->comm : "<unknown>");
1167 rcu_read_unlock();
1168
1169 spin_lock(&file->table_lock);
1170 idr_for_each_entry(&file->object_idr, gobj, id) {
1171 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj);
1172
1173 amdgpu_bo_print_info(id, bo, m);
1174 }
1175 spin_unlock(&file->table_lock);
1176 }
1177
1178 mutex_unlock(&dev->filelist_mutex);
1179 return 0;
1180 }
1181
1182 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_gem_info);
1183
1184 #endif
1185
amdgpu_debugfs_gem_init(struct amdgpu_device * adev)1186 void amdgpu_debugfs_gem_init(struct amdgpu_device *adev)
1187 {
1188 #if defined(CONFIG_DEBUG_FS)
1189 struct drm_minor *minor = adev_to_drm(adev)->primary;
1190 struct dentry *root = minor->debugfs_root;
1191
1192 debugfs_create_file("amdgpu_gem_info", 0444, root, adev,
1193 &amdgpu_debugfs_gem_info_fops);
1194 #endif
1195 }
1196