1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2014-2018 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 #include <linux/dma-buf.h> 24 #include <linux/list.h> 25 #include <linux/pagemap.h> 26 #include <linux/sched/mm.h> 27 #include <linux/sched/task.h> 28 #include <drm/ttm/ttm_tt.h> 29 30 #include <drm/drm_exec.h> 31 32 #include "amdgpu_object.h" 33 #include "amdgpu_gem.h" 34 #include "amdgpu_vm.h" 35 #include "amdgpu_hmm.h" 36 #include "amdgpu_amdkfd.h" 37 #include "amdgpu_dma_buf.h" 38 #include <uapi/linux/kfd_ioctl.h> 39 #include "amdgpu_xgmi.h" 40 #include "kfd_priv.h" 41 #include "kfd_smi_events.h" 42 43 /* Userptr restore delay, just long enough to allow consecutive VM 44 * changes to accumulate 45 */ 46 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 47 #define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29) 48 49 /* Impose limit on how much memory KFD can use */ 50 static struct { 51 uint64_t max_system_mem_limit; 52 uint64_t max_ttm_mem_limit; 53 int64_t system_mem_used; 54 int64_t ttm_mem_used; 55 spinlock_t mem_limit_lock; 56 } kfd_mem_limit; 57 58 static const char * const domain_bit_to_string[] = { 59 "CPU", 60 "GTT", 61 "VRAM", 62 "GDS", 63 "GWS", 64 "OA" 65 }; 66 67 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] 68 69 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); 70 static size_t __get_available_memory(struct amdgpu_device *adev, 71 uint8_t xcp_id, int64_t vram_used); 72 73 static bool kfd_mem_is_attached(struct amdgpu_vm *avm, 74 struct kgd_mem *mem) 75 { 76 struct kfd_mem_attachment *entry; 77 78 list_for_each_entry(entry, &mem->attachments, list) 79 if (entry->bo_va->base.vm == avm) 80 return true; 81 82 return false; 83 } 84 85 /** 86 * reuse_dmamap() - Check whether adev can share the original 87 * userptr BO 88 * 89 * If both adev and bo_adev are in direct mapping or 90 * in the same iommu group, they can share the original BO. 91 * 92 * @adev: Device to which can or cannot share the original BO 93 * @bo_adev: Device to which allocated BO belongs to 94 * 95 * Return: returns true if adev can share original userptr BO, 96 * false otherwise. 97 */ 98 static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev) 99 { 100 return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) || 101 (adev->dev->iommu_group == bo_adev->dev->iommu_group); 102 } 103 104 /* Set memory usage limits. Current, limits are 105 * System (TTM + userptr) memory - 15/16th System RAM 106 * TTM memory - 3/8th System RAM 107 */ 108 void amdgpu_amdkfd_gpuvm_init_mem_limits(void) 109 { 110 struct sysinfo si; 111 uint64_t mem; 112 113 if (kfd_mem_limit.max_system_mem_limit) 114 return; 115 116 si_meminfo(&si); 117 mem = si.totalram - si.totalhigh; 118 mem *= si.mem_unit; 119 120 spin_lock_init(&kfd_mem_limit.mem_limit_lock); 121 kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6); 122 if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT) 123 kfd_mem_limit.max_system_mem_limit >>= 1; 124 else 125 kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT; 126 127 kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT; 128 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", 129 (kfd_mem_limit.max_system_mem_limit >> 20), 130 (kfd_mem_limit.max_ttm_mem_limit >> 20)); 131 } 132 133 void amdgpu_amdkfd_reserve_system_mem(uint64_t size) 134 { 135 kfd_mem_limit.system_mem_used += size; 136 } 137 138 /* Estimate page table size needed to represent a given memory size 139 * 140 * With 4KB pages, we need one 8 byte PTE for each 4KB of memory 141 * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB 142 * of memory (factor 256K, >> 18). ROCm user mode tries to optimize 143 * for 2MB pages for TLB efficiency. However, small allocations and 144 * fragmented system memory still need some 4KB pages. We choose a 145 * compromise that should work in most cases without reserving too 146 * much memory for page tables unnecessarily (factor 16K, >> 14). 147 */ 148 149 #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) 150 151 /** 152 * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size 153 * of buffer. 154 * 155 * @adev: Device to which allocated BO belongs to 156 * @size: Size of buffer, in bytes, encapsulated by B0. This should be 157 * equivalent to amdgpu_bo_size(BO) 158 * @alloc_flag: Flag used in allocating a BO as noted above 159 * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is 160 * managed as one compute node in driver for app 161 * 162 * Return: 163 * returns -ENOMEM in case of error, ZERO otherwise 164 */ 165 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, 166 uint64_t size, u32 alloc_flag, int8_t xcp_id) 167 { 168 size_t system_mem_needed, ttm_mem_needed, vram_needed; 169 int ret = 0; 170 171 system_mem_needed = 0; 172 ttm_mem_needed = 0; 173 vram_needed = 0; 174 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 175 system_mem_needed = size; 176 ttm_mem_needed = size; 177 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 178 /* 179 * Conservatively round up the allocation requirement to 2 MB 180 * to avoid fragmentation caused by 4K allocations in the tail 181 * 2M BO chunk. 182 */ 183 vram_needed = size; 184 /* 185 * For GFX 9.4.3, get the VRAM size from XCP structs 186 */ 187 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) 188 return -EINVAL; 189 190 if (adev->apu_prefer_gtt) { 191 system_mem_needed = size; 192 ttm_mem_needed = size; 193 } 194 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 195 system_mem_needed = size; 196 } else if (!(alloc_flag & 197 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 198 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 199 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 200 return -ENOMEM; 201 } 202 203 spin_lock(&kfd_mem_limit.mem_limit_lock); 204 205 if (kfd_mem_limit.system_mem_used + system_mem_needed > 206 kfd_mem_limit.max_system_mem_limit) { 207 pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); 208 if (!no_system_mem_limit) { 209 ret = -ENOMEM; 210 goto release; 211 } 212 } 213 214 if (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > 215 kfd_mem_limit.max_ttm_mem_limit) { 216 ret = -ENOMEM; 217 goto release; 218 } 219 220 /*if is_app_apu is false and apu_prefer_gtt is true, it is an APU with 221 * carve out < gtt. In that case, VRAM allocation will go to gtt domain, skip 222 * VRAM check since ttm_mem_limit check already cover this allocation 223 */ 224 225 if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu) && 226 vram_needed > __get_available_memory(adev, xcp_id, 227 adev->kfd.vram_used[xcp_id])) { 228 ret = -ENOMEM; 229 goto release; 230 } 231 232 /* Update memory accounting by decreasing available system 233 * memory, TTM memory and GPU memory as computed above 234 */ 235 WARN_ONCE(vram_needed && !adev, 236 "adev reference can't be null when vram is used"); 237 if (adev && xcp_id >= 0) { 238 adev->kfd.vram_used[xcp_id] += vram_needed; 239 adev->kfd.vram_used_aligned[xcp_id] += 240 adev->apu_prefer_gtt ? 241 vram_needed : 242 ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); 243 } 244 kfd_mem_limit.system_mem_used += system_mem_needed; 245 kfd_mem_limit.ttm_mem_used += ttm_mem_needed; 246 247 release: 248 spin_unlock(&kfd_mem_limit.mem_limit_lock); 249 return ret; 250 } 251 252 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, 253 uint64_t size, u32 alloc_flag, int8_t xcp_id) 254 { 255 spin_lock(&kfd_mem_limit.mem_limit_lock); 256 257 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 258 kfd_mem_limit.system_mem_used -= size; 259 kfd_mem_limit.ttm_mem_used -= size; 260 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 261 WARN_ONCE(!adev, 262 "adev reference can't be null when alloc mem flags vram is set"); 263 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) 264 goto release; 265 266 if (adev) { 267 adev->kfd.vram_used[xcp_id] -= size; 268 if (adev->apu_prefer_gtt) { 269 adev->kfd.vram_used_aligned[xcp_id] -= size; 270 kfd_mem_limit.system_mem_used -= size; 271 kfd_mem_limit.ttm_mem_used -= size; 272 } else { 273 adev->kfd.vram_used_aligned[xcp_id] -= 274 ALIGN(size, VRAM_AVAILABLITY_ALIGN); 275 } 276 } 277 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 278 kfd_mem_limit.system_mem_used -= size; 279 } else if (!(alloc_flag & 280 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 281 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 282 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 283 goto release; 284 } 285 WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0, 286 "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id); 287 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, 288 "KFD TTM memory accounting unbalanced"); 289 WARN_ONCE(kfd_mem_limit.system_mem_used < 0, 290 "KFD system memory accounting unbalanced"); 291 292 release: 293 spin_unlock(&kfd_mem_limit.mem_limit_lock); 294 } 295 296 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) 297 { 298 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 299 u32 alloc_flags = bo->kfd_bo->alloc_flags; 300 u64 size = amdgpu_bo_size(bo); 301 302 amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags, 303 bo->xcp_id); 304 305 kfree(bo->kfd_bo); 306 } 307 308 void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id) 309 { 310 if (!bo->kfd_bo) 311 return; 312 313 if (WARN_ONCE(xcp_id < 0, "KFD BO pinned in VRAM with no partition")) 314 return; 315 316 atomic64_add(amdgpu_bo_size(bo), 317 &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]); 318 } 319 320 void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id) 321 { 322 if (!bo->kfd_bo) 323 return; 324 325 if (WARN_ONCE(xcp_id < 0, "KFD BO unpinned in VRAM with no partition")) 326 return; 327 328 atomic64_sub(amdgpu_bo_size(bo), 329 &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]); 330 } 331 332 /** 333 * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information 334 * about USERPTR or DOOREBELL or MMIO BO. 335 * 336 * @adev: Device for which dmamap BO is being created 337 * @mem: BO of peer device that is being DMA mapped. Provides parameters 338 * in building the dmamap BO 339 * @bo_out: Output parameter updated with handle of dmamap BO 340 */ 341 static int 342 create_dmamap_sg_bo(struct amdgpu_device *adev, 343 struct kgd_mem *mem, struct amdgpu_bo **bo_out) 344 { 345 struct drm_gem_object *gem_obj; 346 int ret; 347 uint64_t flags = 0; 348 349 ret = amdgpu_bo_reserve(mem->bo, false); 350 if (ret) 351 return ret; 352 353 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) 354 flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT | 355 AMDGPU_GEM_CREATE_UNCACHED); 356 357 ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1, 358 AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags, 359 ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0); 360 361 amdgpu_bo_unreserve(mem->bo); 362 363 if (ret) { 364 pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); 365 return -EINVAL; 366 } 367 368 *bo_out = gem_to_amdgpu_bo(gem_obj); 369 (*bo_out)->parent = amdgpu_bo_ref(mem->bo); 370 return ret; 371 } 372 373 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's 374 * reservation object. 375 * 376 * @bo: [IN] Remove eviction fence(s) from this BO 377 * @ef: [IN] This eviction fence is removed if it 378 * is present in the shared list. 379 * 380 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. 381 */ 382 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, 383 struct amdgpu_amdkfd_fence *ef) 384 { 385 struct dma_fence *replacement; 386 387 if (!ef) 388 return -EINVAL; 389 390 /* TODO: Instead of block before we should use the fence of the page 391 * table update and TLB flush here directly. 392 */ 393 replacement = dma_fence_get_stub(); 394 dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, 395 replacement, DMA_RESV_USAGE_BOOKKEEP); 396 dma_fence_put(replacement); 397 return 0; 398 } 399 400 /** 401 * amdgpu_amdkfd_remove_all_eviction_fences - Remove all eviction fences 402 * @bo: the BO where to remove the evictions fences from. 403 * 404 * This functions should only be used on release when all references to the BO 405 * are already dropped. We remove the eviction fence from the private copy of 406 * the dma_resv object here since that is what is used during release to 407 * determine of the BO is idle or not. 408 */ 409 void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo) 410 { 411 struct dma_resv *resv = &bo->tbo.base._resv; 412 struct dma_fence *fence, *stub; 413 struct dma_resv_iter cursor; 414 415 dma_resv_assert_held(resv); 416 417 stub = dma_fence_get_stub(); 418 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) { 419 if (!to_amdgpu_amdkfd_fence(fence)) 420 continue; 421 422 dma_resv_replace_fences(resv, fence->context, stub, 423 DMA_RESV_USAGE_BOOKKEEP); 424 } 425 dma_fence_put(stub); 426 } 427 428 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, 429 bool wait) 430 { 431 struct ttm_operation_ctx ctx = { false, false }; 432 int ret; 433 434 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), 435 "Called with userptr BO")) 436 return -EINVAL; 437 438 /* bo has been pinned, not need validate it */ 439 if (bo->tbo.pin_count) 440 return 0; 441 442 amdgpu_bo_placement_from_domain(bo, domain); 443 444 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 445 if (ret) 446 goto validate_fail; 447 if (wait) 448 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 449 450 validate_fail: 451 return ret; 452 } 453 454 int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, 455 uint32_t domain, 456 struct dma_fence *fence) 457 { 458 int ret = amdgpu_bo_reserve(bo, false); 459 460 if (ret) 461 return ret; 462 463 ret = amdgpu_amdkfd_bo_validate(bo, domain, true); 464 if (ret) 465 goto unreserve_out; 466 467 ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1); 468 if (ret) 469 goto unreserve_out; 470 471 dma_resv_add_fence(bo->tbo.base.resv, fence, 472 DMA_RESV_USAGE_BOOKKEEP); 473 474 unreserve_out: 475 amdgpu_bo_unreserve(bo); 476 477 return ret; 478 } 479 480 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) 481 { 482 return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); 483 } 484 485 /* vm_validate_pt_pd_bos - Validate page table and directory BOs 486 * 487 * Page directories are not updated here because huge page handling 488 * during page table updates can invalidate page directory entries 489 * again. Page directories are only updated after updating page 490 * tables. 491 */ 492 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm, 493 struct ww_acquire_ctx *ticket) 494 { 495 struct amdgpu_bo *pd = vm->root.bo; 496 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 497 int ret; 498 499 ret = amdgpu_vm_validate(adev, vm, ticket, 500 amdgpu_amdkfd_validate_vm_bo, NULL); 501 if (ret) { 502 pr_err("failed to validate PT BOs\n"); 503 return ret; 504 } 505 506 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); 507 508 return 0; 509 } 510 511 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) 512 { 513 struct amdgpu_bo *pd = vm->root.bo; 514 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 515 int ret; 516 517 ret = amdgpu_vm_update_pdes(adev, vm, false); 518 if (ret) 519 return ret; 520 521 return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL); 522 } 523 524 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct amdgpu_vm *vm, 525 struct kgd_mem *mem) 526 { 527 uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE | 528 AMDGPU_VM_MTYPE_DEFAULT; 529 530 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) 531 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; 532 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) 533 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; 534 535 return mapping_flags; 536 } 537 538 /** 539 * create_sg_table() - Create an sg_table for a contiguous DMA addr range 540 * @addr: The starting address to point to 541 * @size: Size of memory area in bytes being pointed to 542 * 543 * Allocates an instance of sg_table and initializes it to point to memory 544 * area specified by input parameters. The address used to build is assumed 545 * to be DMA mapped, if needed. 546 * 547 * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table 548 * because they are physically contiguous. 549 * 550 * Return: Initialized instance of SG Table or NULL 551 */ 552 static struct sg_table *create_sg_table(uint64_t addr, uint32_t size) 553 { 554 struct sg_table *sg = kmalloc_obj(*sg); 555 556 if (!sg) 557 return NULL; 558 if (sg_alloc_table(sg, 1, GFP_KERNEL)) { 559 kfree(sg); 560 return NULL; 561 } 562 sg_dma_address(sg->sgl) = addr; 563 sg->sgl->length = size; 564 #ifdef CONFIG_NEED_SG_DMA_LENGTH 565 sg->sgl->dma_length = size; 566 #endif 567 return sg; 568 } 569 570 static int 571 kfd_mem_dmamap_userptr(struct kgd_mem *mem, 572 struct kfd_mem_attachment *attachment) 573 { 574 enum dma_data_direction direction = 575 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 576 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 577 struct ttm_operation_ctx ctx = {.interruptible = true}; 578 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 579 struct amdgpu_device *adev = attachment->adev; 580 struct ttm_tt *src_ttm = mem->bo->tbo.ttm; 581 struct ttm_tt *ttm = bo->tbo.ttm; 582 int ret; 583 584 if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) 585 return -EINVAL; 586 587 ttm->sg = kmalloc_obj(*ttm->sg); 588 if (unlikely(!ttm->sg)) 589 return -ENOMEM; 590 591 /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ 592 ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, 593 ttm->num_pages, 0, 594 (u64)ttm->num_pages << PAGE_SHIFT, 595 GFP_KERNEL); 596 if (unlikely(ret)) 597 goto free_sg; 598 599 ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); 600 if (unlikely(ret)) 601 goto release_sg; 602 603 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 604 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 605 if (ret) 606 goto unmap_sg; 607 608 return 0; 609 610 unmap_sg: 611 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 612 release_sg: 613 pr_err("DMA map userptr failed: %d\n", ret); 614 sg_free_table(ttm->sg); 615 free_sg: 616 kfree(ttm->sg); 617 ttm->sg = NULL; 618 return ret; 619 } 620 621 static int 622 kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment) 623 { 624 struct ttm_operation_ctx ctx = {.interruptible = true}; 625 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 626 627 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 628 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 629 } 630 631 /** 632 * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO 633 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device 634 * @attachment: Virtual address attachment of the BO on accessing device 635 * 636 * An access request from the device that owns DOORBELL does not require DMA mapping. 637 * This is because the request doesn't go through PCIe root complex i.e. it instead 638 * loops back. The need to DMA map arises only when accessing peer device's DOORBELL 639 * 640 * In contrast, all access requests for MMIO need to be DMA mapped without regard to 641 * device ownership. This is because access requests for MMIO go through PCIe root 642 * complex. 643 * 644 * This is accomplished in two steps: 645 * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used 646 * in updating requesting device's page table 647 * - Signal TTM to mark memory pointed to by requesting device's BO as GPU 648 * accessible. This allows an update of requesting device's page table 649 * with entries associated with DOOREBELL or MMIO memory 650 * 651 * This method is invoked in the following contexts: 652 * - Mapping of DOORBELL or MMIO BO of same or peer device 653 * - Validating an evicted DOOREBELL or MMIO BO on device seeking access 654 * 655 * Return: ZERO if successful, NON-ZERO otherwise 656 */ 657 static int 658 kfd_mem_dmamap_sg_bo(struct kgd_mem *mem, 659 struct kfd_mem_attachment *attachment) 660 { 661 struct ttm_operation_ctx ctx = {.interruptible = true}; 662 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 663 struct amdgpu_device *adev = attachment->adev; 664 struct ttm_tt *ttm = bo->tbo.ttm; 665 enum dma_data_direction dir; 666 dma_addr_t dma_addr; 667 bool mmio; 668 int ret; 669 670 /* Expect SG Table of dmapmap BO to be NULL */ 671 mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP); 672 if (unlikely(ttm->sg)) { 673 pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio); 674 return -EINVAL; 675 } 676 677 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 678 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 679 dma_addr = mem->bo->tbo.sg->sgl->dma_address; 680 pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length); 681 pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr); 682 dma_addr = dma_map_resource(adev->dev, dma_addr, 683 mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); 684 ret = dma_mapping_error(adev->dev, dma_addr); 685 if (unlikely(ret)) 686 return ret; 687 pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr); 688 689 ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length); 690 if (unlikely(!ttm->sg)) { 691 ret = -ENOMEM; 692 goto unmap_sg; 693 } 694 695 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 696 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 697 if (unlikely(ret)) 698 goto free_sg; 699 700 return ret; 701 702 free_sg: 703 sg_free_table(ttm->sg); 704 kfree(ttm->sg); 705 ttm->sg = NULL; 706 unmap_sg: 707 dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length, 708 dir, DMA_ATTR_SKIP_CPU_SYNC); 709 return ret; 710 } 711 712 static int 713 kfd_mem_dmamap_attachment(struct kgd_mem *mem, 714 struct kfd_mem_attachment *attachment) 715 { 716 switch (attachment->type) { 717 case KFD_MEM_ATT_SHARED: 718 return 0; 719 case KFD_MEM_ATT_USERPTR: 720 return kfd_mem_dmamap_userptr(mem, attachment); 721 case KFD_MEM_ATT_DMABUF: 722 return kfd_mem_dmamap_dmabuf(attachment); 723 case KFD_MEM_ATT_SG: 724 return kfd_mem_dmamap_sg_bo(mem, attachment); 725 default: 726 WARN_ON_ONCE(1); 727 } 728 return -EINVAL; 729 } 730 731 static void 732 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem, 733 struct kfd_mem_attachment *attachment) 734 { 735 enum dma_data_direction direction = 736 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 737 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 738 struct ttm_operation_ctx ctx = {.interruptible = false}; 739 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 740 struct amdgpu_device *adev = attachment->adev; 741 struct ttm_tt *ttm = bo->tbo.ttm; 742 743 if (unlikely(!ttm->sg)) 744 return; 745 746 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 747 (void)ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 748 749 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 750 sg_free_table(ttm->sg); 751 kfree(ttm->sg); 752 ttm->sg = NULL; 753 } 754 755 static void 756 kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment) 757 { 758 /* This is a no-op. We don't want to trigger eviction fences when 759 * unmapping DMABufs. Therefore the invalidation (moving to system 760 * domain) is done in kfd_mem_dmamap_dmabuf. 761 */ 762 } 763 764 /** 765 * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO 766 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device 767 * @attachment: Virtual address attachment of the BO on accessing device 768 * 769 * The method performs following steps: 770 * - Signal TTM to mark memory pointed to by BO as GPU inaccessible 771 * - Free SG Table that is used to encapsulate DMA mapped memory of 772 * peer device's DOORBELL or MMIO memory 773 * 774 * This method is invoked in the following contexts: 775 * UNMapping of DOORBELL or MMIO BO on a device having access to its memory 776 * Eviction of DOOREBELL or MMIO BO on device having access to its memory 777 * 778 * Return: void 779 */ 780 static void 781 kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem, 782 struct kfd_mem_attachment *attachment) 783 { 784 struct ttm_operation_ctx ctx = {.interruptible = true}; 785 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 786 struct amdgpu_device *adev = attachment->adev; 787 struct ttm_tt *ttm = bo->tbo.ttm; 788 enum dma_data_direction dir; 789 790 if (unlikely(!ttm->sg)) { 791 pr_debug("SG Table of BO is NULL"); 792 return; 793 } 794 795 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 796 (void)ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 797 798 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 799 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 800 dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address, 801 ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); 802 sg_free_table(ttm->sg); 803 kfree(ttm->sg); 804 ttm->sg = NULL; 805 bo->tbo.sg = NULL; 806 } 807 808 static void 809 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem, 810 struct kfd_mem_attachment *attachment) 811 { 812 switch (attachment->type) { 813 case KFD_MEM_ATT_SHARED: 814 break; 815 case KFD_MEM_ATT_USERPTR: 816 kfd_mem_dmaunmap_userptr(mem, attachment); 817 break; 818 case KFD_MEM_ATT_DMABUF: 819 kfd_mem_dmaunmap_dmabuf(attachment); 820 break; 821 case KFD_MEM_ATT_SG: 822 kfd_mem_dmaunmap_sg_bo(mem, attachment); 823 break; 824 default: 825 WARN_ON_ONCE(1); 826 } 827 } 828 829 static int kfd_mem_export_dmabuf(struct kgd_mem *mem) 830 { 831 if (!mem->dmabuf) { 832 struct amdgpu_device *bo_adev; 833 struct dma_buf *dmabuf; 834 835 bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 836 dmabuf = drm_gem_prime_handle_to_dmabuf(&bo_adev->ddev, bo_adev->kfd.client.file, 837 mem->gem_handle, 838 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 839 DRM_RDWR : 0); 840 if (IS_ERR(dmabuf)) 841 return PTR_ERR(dmabuf); 842 mem->dmabuf = dmabuf; 843 } 844 845 return 0; 846 } 847 848 static int 849 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem, 850 struct amdgpu_bo **bo) 851 { 852 struct drm_gem_object *gobj; 853 int ret; 854 855 ret = kfd_mem_export_dmabuf(mem); 856 if (ret) 857 return ret; 858 859 gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf); 860 if (IS_ERR(gobj)) 861 return PTR_ERR(gobj); 862 863 *bo = gem_to_amdgpu_bo(gobj); 864 (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE; 865 866 return 0; 867 } 868 869 /* kfd_mem_attach - Add a BO to a VM 870 * 871 * Everything that needs to bo done only once when a BO is first added 872 * to a VM. It can later be mapped and unmapped many times without 873 * repeating these steps. 874 * 875 * 0. Create BO for DMA mapping, if needed 876 * 1. Allocate and initialize BO VA entry data structure 877 * 2. Add BO to the VM 878 * 3. Determine ASIC-specific PTE flags 879 * 4. Alloc page tables and directories if needed 880 * 4a. Validate new page tables and directories 881 */ 882 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem, 883 struct amdgpu_vm *vm, bool is_aql) 884 { 885 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 886 unsigned long bo_size = mem->bo->tbo.base.size; 887 uint64_t va = mem->va; 888 struct kfd_mem_attachment *attachment[2] = {NULL, NULL}; 889 struct amdgpu_bo *bo[2] = {NULL, NULL}; 890 struct amdgpu_bo_va *bo_va; 891 bool same_hive = false; 892 struct drm_exec exec; 893 int i, ret; 894 895 if (!va) { 896 pr_err("Invalid VA when adding BO to VM\n"); 897 return -EINVAL; 898 } 899 900 /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices 901 * 902 * The access path of MMIO and DOORBELL BOs of is always over PCIe. 903 * In contrast the access path of VRAM BOs depens upon the type of 904 * link that connects the peer device. Access over PCIe is allowed 905 * if peer device has large BAR. In contrast, access over xGMI is 906 * allowed for both small and large BAR configurations of peer device 907 */ 908 if ((adev != bo_adev && !adev->apu_prefer_gtt) && 909 ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) || 910 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) || 911 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 912 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM) 913 same_hive = amdgpu_xgmi_same_hive(adev, bo_adev); 914 if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev)) 915 return -EINVAL; 916 } 917 918 for (i = 0; i <= is_aql; i++) { 919 attachment[i] = kzalloc_obj(*attachment[i]); 920 if (unlikely(!attachment[i])) { 921 ret = -ENOMEM; 922 goto unwind; 923 } 924 925 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, 926 va + bo_size, vm); 927 928 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) || 929 (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) || 930 (mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) || 931 same_hive) { 932 /* Mappings on the local GPU, or VRAM mappings in the 933 * local hive, or userptr, or GTT mapping can reuse dma map 934 * address space share the original BO 935 */ 936 attachment[i]->type = KFD_MEM_ATT_SHARED; 937 bo[i] = mem->bo; 938 drm_gem_object_get(&bo[i]->tbo.base); 939 } else if (i > 0) { 940 /* Multiple mappings on the same GPU share the BO */ 941 attachment[i]->type = KFD_MEM_ATT_SHARED; 942 bo[i] = bo[0]; 943 drm_gem_object_get(&bo[i]->tbo.base); 944 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { 945 /* Create an SG BO to DMA-map userptrs on other GPUs */ 946 attachment[i]->type = KFD_MEM_ATT_USERPTR; 947 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); 948 if (ret) 949 goto unwind; 950 /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */ 951 } else if (mem->bo->tbo.type == ttm_bo_type_sg) { 952 WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL || 953 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP), 954 "Handing invalid SG BO in ATTACH request"); 955 attachment[i]->type = KFD_MEM_ATT_SG; 956 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); 957 if (ret) 958 goto unwind; 959 /* Enable acces to GTT and VRAM BOs of peer devices */ 960 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT || 961 mem->domain == AMDGPU_GEM_DOMAIN_VRAM) { 962 attachment[i]->type = KFD_MEM_ATT_DMABUF; 963 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]); 964 if (ret) 965 goto unwind; 966 pr_debug("Employ DMABUF mechanism to enable peer GPU access\n"); 967 } else { 968 WARN_ONCE(true, "Handling invalid ATTACH request"); 969 ret = -EINVAL; 970 goto unwind; 971 } 972 973 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 974 drm_exec_until_all_locked(&exec) { 975 ret = amdgpu_vm_lock_pd(vm, &exec, 0); 976 drm_exec_retry_on_contention(&exec); 977 if (unlikely(ret)) 978 goto unlock; 979 ret = drm_exec_lock_obj(&exec, &bo[i]->tbo.base); 980 drm_exec_retry_on_contention(&exec); 981 if (unlikely(ret)) 982 goto unlock; 983 } 984 985 bo_va = amdgpu_vm_bo_find(vm, bo[i]); 986 if (!bo_va) 987 bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]); 988 else 989 ++bo_va->ref_count; 990 attachment[i]->bo_va = bo_va; 991 drm_exec_fini(&exec); 992 if (unlikely(!attachment[i]->bo_va)) { 993 ret = -ENOMEM; 994 pr_err("Failed to add BO object to VM. ret == %d\n", 995 ret); 996 goto unwind; 997 } 998 attachment[i]->va = va; 999 attachment[i]->pte_flags = get_pte_flags(adev, vm, mem); 1000 attachment[i]->adev = adev; 1001 list_add(&attachment[i]->list, &mem->attachments); 1002 1003 va += bo_size; 1004 } 1005 1006 return 0; 1007 1008 unlock: 1009 drm_exec_fini(&exec); 1010 unwind: 1011 for (; i >= 0; i--) { 1012 if (!attachment[i]) 1013 continue; 1014 if (attachment[i]->bo_va) { 1015 (void)amdgpu_bo_reserve(bo[i], true); 1016 if (--attachment[i]->bo_va->ref_count == 0) 1017 amdgpu_vm_bo_del(adev, attachment[i]->bo_va); 1018 amdgpu_bo_unreserve(bo[i]); 1019 list_del(&attachment[i]->list); 1020 } 1021 if (bo[i]) 1022 drm_gem_object_put(&bo[i]->tbo.base); 1023 kfree(attachment[i]); 1024 } 1025 return ret; 1026 } 1027 1028 static void kfd_mem_detach(struct kfd_mem_attachment *attachment) 1029 { 1030 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 1031 1032 pr_debug("\t remove VA 0x%llx in entry %p\n", 1033 attachment->va, attachment); 1034 if (--attachment->bo_va->ref_count == 0) 1035 amdgpu_vm_bo_del(attachment->adev, attachment->bo_va); 1036 drm_gem_object_put(&bo->tbo.base); 1037 list_del(&attachment->list); 1038 kfree(attachment); 1039 } 1040 1041 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, 1042 struct amdkfd_process_info *process_info, 1043 bool userptr) 1044 { 1045 mutex_lock(&process_info->lock); 1046 if (userptr) 1047 list_add_tail(&mem->validate_list, 1048 &process_info->userptr_valid_list); 1049 else 1050 list_add_tail(&mem->validate_list, &process_info->kfd_bo_list); 1051 mutex_unlock(&process_info->lock); 1052 } 1053 1054 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem, 1055 struct amdkfd_process_info *process_info) 1056 { 1057 mutex_lock(&process_info->lock); 1058 list_del(&mem->validate_list); 1059 mutex_unlock(&process_info->lock); 1060 } 1061 1062 /* Initializes user pages. It registers the MMU notifier and validates 1063 * the userptr BO in the GTT domain. 1064 * 1065 * The BO must already be on the userptr_valid_list. Otherwise an 1066 * eviction and restore may happen that leaves the new BO unmapped 1067 * with the user mode queues running. 1068 * 1069 * Takes the process_info->lock to protect against concurrent restore 1070 * workers. 1071 * 1072 * Returns 0 for success, negative errno for errors. 1073 */ 1074 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr, 1075 bool criu_resume) 1076 { 1077 struct amdkfd_process_info *process_info = mem->process_info; 1078 struct amdgpu_bo *bo = mem->bo; 1079 struct ttm_operation_ctx ctx = { true, false }; 1080 struct amdgpu_hmm_range *range; 1081 int ret = 0; 1082 1083 mutex_lock(&process_info->lock); 1084 1085 ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0); 1086 if (ret) { 1087 pr_err("%s: Failed to set userptr: %d\n", __func__, ret); 1088 goto out; 1089 } 1090 1091 ret = amdgpu_hmm_register(bo, user_addr); 1092 if (ret) { 1093 pr_err("%s: Failed to register MMU notifier: %d\n", 1094 __func__, ret); 1095 goto out; 1096 } 1097 1098 if (criu_resume) { 1099 /* 1100 * During a CRIU restore operation, the userptr buffer objects 1101 * will be validated in the restore_userptr_work worker at a 1102 * later stage when it is scheduled by another ioctl called by 1103 * CRIU master process for the target pid for restore. 1104 */ 1105 mutex_lock(&process_info->notifier_lock); 1106 mem->invalid++; 1107 mutex_unlock(&process_info->notifier_lock); 1108 mutex_unlock(&process_info->lock); 1109 return 0; 1110 } 1111 1112 range = amdgpu_hmm_range_alloc(NULL); 1113 if (unlikely(!range)) { 1114 ret = -ENOMEM; 1115 goto unregister_out; 1116 } 1117 1118 ret = amdgpu_ttm_tt_get_user_pages(bo, range); 1119 if (ret) { 1120 amdgpu_hmm_range_free(range); 1121 if (ret == -EAGAIN) 1122 pr_debug("Failed to get user pages, try again\n"); 1123 else 1124 pr_err("%s: Failed to get user pages: %d\n", __func__, ret); 1125 goto unregister_out; 1126 } 1127 1128 ret = amdgpu_bo_reserve(bo, true); 1129 if (ret) { 1130 pr_err("%s: Failed to reserve BO\n", __func__); 1131 goto release_out; 1132 } 1133 1134 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range); 1135 1136 amdgpu_bo_placement_from_domain(bo, mem->domain); 1137 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1138 if (ret) 1139 pr_err("%s: failed to validate BO\n", __func__); 1140 amdgpu_bo_unreserve(bo); 1141 1142 release_out: 1143 amdgpu_hmm_range_free(range); 1144 unregister_out: 1145 if (ret) 1146 amdgpu_hmm_unregister(bo); 1147 out: 1148 mutex_unlock(&process_info->lock); 1149 return ret; 1150 } 1151 1152 /* Reserving a BO and its page table BOs must happen atomically to 1153 * avoid deadlocks. Some operations update multiple VMs at once. Track 1154 * all the reservation info in a context structure. Optionally a sync 1155 * object can track VM updates. 1156 */ 1157 struct bo_vm_reservation_context { 1158 /* DRM execution context for the reservation */ 1159 struct drm_exec exec; 1160 /* Number of VMs reserved */ 1161 unsigned int n_vms; 1162 /* Pointer to sync object */ 1163 struct amdgpu_sync *sync; 1164 }; 1165 1166 enum bo_vm_match { 1167 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */ 1168 BO_VM_MAPPED, /* Match VMs where a BO is mapped */ 1169 BO_VM_ALL, /* Match all VMs a BO was added to */ 1170 }; 1171 1172 /** 1173 * reserve_bo_and_vm - reserve a BO and a VM unconditionally. 1174 * @mem: KFD BO structure. 1175 * @vm: the VM to reserve. 1176 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 1177 */ 1178 static int reserve_bo_and_vm(struct kgd_mem *mem, 1179 struct amdgpu_vm *vm, 1180 struct bo_vm_reservation_context *ctx) 1181 { 1182 struct amdgpu_bo *bo = mem->bo; 1183 int ret; 1184 1185 WARN_ON(!vm); 1186 1187 ctx->n_vms = 1; 1188 ctx->sync = &mem->sync; 1189 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 1190 drm_exec_until_all_locked(&ctx->exec) { 1191 ret = amdgpu_vm_lock_pd(vm, &ctx->exec, 2); 1192 drm_exec_retry_on_contention(&ctx->exec); 1193 if (unlikely(ret)) 1194 goto error; 1195 1196 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); 1197 drm_exec_retry_on_contention(&ctx->exec); 1198 if (unlikely(ret)) 1199 goto error; 1200 } 1201 return 0; 1202 1203 error: 1204 pr_err("Failed to reserve buffers in ttm.\n"); 1205 drm_exec_fini(&ctx->exec); 1206 return ret; 1207 } 1208 1209 /** 1210 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally 1211 * @mem: KFD BO structure. 1212 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO 1213 * is used. Otherwise, a single VM associated with the BO. 1214 * @map_type: the mapping status that will be used to filter the VMs. 1215 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 1216 * 1217 * Returns 0 for success, negative for failure. 1218 */ 1219 static int reserve_bo_and_cond_vms(struct kgd_mem *mem, 1220 struct amdgpu_vm *vm, enum bo_vm_match map_type, 1221 struct bo_vm_reservation_context *ctx) 1222 { 1223 struct kfd_mem_attachment *entry; 1224 struct amdgpu_bo *bo = mem->bo; 1225 int ret; 1226 1227 ctx->sync = &mem->sync; 1228 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT | 1229 DRM_EXEC_IGNORE_DUPLICATES, 0); 1230 drm_exec_until_all_locked(&ctx->exec) { 1231 ctx->n_vms = 0; 1232 list_for_each_entry(entry, &mem->attachments, list) { 1233 if ((vm && vm != entry->bo_va->base.vm) || 1234 (entry->is_mapped != map_type 1235 && map_type != BO_VM_ALL)) 1236 continue; 1237 1238 ret = amdgpu_vm_lock_pd(entry->bo_va->base.vm, 1239 &ctx->exec, 2); 1240 drm_exec_retry_on_contention(&ctx->exec); 1241 if (unlikely(ret)) 1242 goto error; 1243 ++ctx->n_vms; 1244 } 1245 1246 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); 1247 drm_exec_retry_on_contention(&ctx->exec); 1248 if (unlikely(ret)) 1249 goto error; 1250 } 1251 return 0; 1252 1253 error: 1254 pr_err("Failed to reserve buffers in ttm.\n"); 1255 drm_exec_fini(&ctx->exec); 1256 return ret; 1257 } 1258 1259 /** 1260 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context 1261 * @ctx: Reservation context to unreserve 1262 * @wait: Optionally wait for a sync object representing pending VM updates 1263 * @intr: Whether the wait is interruptible 1264 * 1265 * Also frees any resources allocated in 1266 * reserve_bo_and_(cond_)vm(s). Returns the status from 1267 * amdgpu_sync_wait. 1268 */ 1269 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx, 1270 bool wait, bool intr) 1271 { 1272 int ret = 0; 1273 1274 if (wait) 1275 ret = amdgpu_sync_wait(ctx->sync, intr); 1276 1277 drm_exec_fini(&ctx->exec); 1278 ctx->sync = NULL; 1279 return ret; 1280 } 1281 1282 static int unmap_bo_from_gpuvm(struct kgd_mem *mem, 1283 struct kfd_mem_attachment *entry, 1284 struct amdgpu_sync *sync) 1285 { 1286 struct amdgpu_bo_va *bo_va = entry->bo_va; 1287 struct amdgpu_device *adev = entry->adev; 1288 struct amdgpu_vm *vm = bo_va->base.vm; 1289 1290 if (bo_va->queue_refcount) { 1291 pr_debug("bo_va->queue_refcount %d\n", bo_va->queue_refcount); 1292 return -EBUSY; 1293 } 1294 1295 (void)amdgpu_vm_bo_unmap(adev, bo_va, entry->va); 1296 1297 /* VM entity stopped if process killed, don't clear freed pt bo */ 1298 if (!amdgpu_vm_ready(vm)) 1299 return 0; 1300 1301 (void)amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); 1302 1303 (void)amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL); 1304 1305 return 0; 1306 } 1307 1308 static int update_gpuvm_pte(struct kgd_mem *mem, 1309 struct kfd_mem_attachment *entry, 1310 struct amdgpu_sync *sync) 1311 { 1312 struct amdgpu_bo_va *bo_va = entry->bo_va; 1313 struct amdgpu_device *adev = entry->adev; 1314 int ret; 1315 1316 ret = kfd_mem_dmamap_attachment(mem, entry); 1317 if (ret) 1318 return ret; 1319 1320 /* Update the page tables */ 1321 ret = amdgpu_vm_bo_update(adev, bo_va, false); 1322 if (ret) { 1323 pr_err("amdgpu_vm_bo_update failed\n"); 1324 return ret; 1325 } 1326 1327 return amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL); 1328 } 1329 1330 static int map_bo_to_gpuvm(struct kgd_mem *mem, 1331 struct kfd_mem_attachment *entry, 1332 struct amdgpu_sync *sync, 1333 bool no_update_pte) 1334 { 1335 int ret; 1336 1337 /* Set virtual address for the allocation */ 1338 ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0, 1339 amdgpu_bo_size(entry->bo_va->base.bo), 1340 entry->pte_flags); 1341 if (ret) { 1342 pr_err("Failed to map VA 0x%llx in vm. ret %d\n", 1343 entry->va, ret); 1344 return ret; 1345 } 1346 1347 if (no_update_pte) 1348 return 0; 1349 1350 ret = update_gpuvm_pte(mem, entry, sync); 1351 if (ret) { 1352 pr_err("update_gpuvm_pte() failed\n"); 1353 goto update_gpuvm_pte_failed; 1354 } 1355 1356 return 0; 1357 1358 update_gpuvm_pte_failed: 1359 unmap_bo_from_gpuvm(mem, entry, sync); 1360 kfd_mem_dmaunmap_attachment(mem, entry); 1361 return ret; 1362 } 1363 1364 static int process_validate_vms(struct amdkfd_process_info *process_info, 1365 struct ww_acquire_ctx *ticket) 1366 { 1367 struct amdgpu_vm *peer_vm; 1368 int ret; 1369 1370 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1371 vm_list_node) { 1372 ret = vm_validate_pt_pd_bos(peer_vm, ticket); 1373 if (ret) 1374 return ret; 1375 } 1376 1377 return 0; 1378 } 1379 1380 static int process_sync_pds_resv(struct amdkfd_process_info *process_info, 1381 struct amdgpu_sync *sync) 1382 { 1383 struct amdgpu_vm *peer_vm; 1384 int ret; 1385 1386 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1387 vm_list_node) { 1388 struct amdgpu_bo *pd = peer_vm->root.bo; 1389 1390 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv, 1391 AMDGPU_SYNC_NE_OWNER, 1392 AMDGPU_FENCE_OWNER_KFD); 1393 if (ret) 1394 return ret; 1395 } 1396 1397 return 0; 1398 } 1399 1400 static int process_update_pds(struct amdkfd_process_info *process_info, 1401 struct amdgpu_sync *sync) 1402 { 1403 struct amdgpu_vm *peer_vm; 1404 int ret; 1405 1406 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1407 vm_list_node) { 1408 ret = vm_update_pds(peer_vm, sync); 1409 if (ret) 1410 return ret; 1411 } 1412 1413 return 0; 1414 } 1415 1416 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, 1417 struct dma_fence **ef) 1418 { 1419 struct amdkfd_process_info *info = NULL; 1420 struct kfd_process *process = NULL; 1421 int ret; 1422 1423 process = container_of(process_info, struct kfd_process, kgd_process_info); 1424 if (!*process_info) { 1425 info = kzalloc_obj(*info); 1426 if (!info) 1427 return -ENOMEM; 1428 1429 mutex_init(&info->lock); 1430 mutex_init(&info->notifier_lock); 1431 INIT_LIST_HEAD(&info->vm_list_head); 1432 INIT_LIST_HEAD(&info->kfd_bo_list); 1433 INIT_LIST_HEAD(&info->userptr_valid_list); 1434 INIT_LIST_HEAD(&info->userptr_inval_list); 1435 1436 info->eviction_fence = 1437 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), 1438 current->mm, 1439 process->context_id); 1440 if (!info->eviction_fence) { 1441 pr_err("Failed to create eviction fence\n"); 1442 ret = -ENOMEM; 1443 goto create_evict_fence_fail; 1444 } 1445 1446 info->pid = get_task_pid(current, PIDTYPE_TGID); 1447 INIT_DELAYED_WORK(&info->restore_userptr_work, 1448 amdgpu_amdkfd_restore_userptr_worker); 1449 1450 info->context_id = process->context_id; 1451 1452 *process_info = info; 1453 } 1454 1455 if (cmpxchg(&vm->process_info, NULL, *process_info) != NULL) { 1456 ret = -EINVAL; 1457 goto already_acquired; 1458 } 1459 1460 /* Validate page directory and attach eviction fence */ 1461 ret = amdgpu_bo_reserve(vm->root.bo, true); 1462 if (ret) 1463 goto reserve_pd_fail; 1464 ret = vm_validate_pt_pd_bos(vm, NULL); 1465 if (ret) { 1466 pr_err("validate_pt_pd_bos() failed\n"); 1467 goto validate_pd_fail; 1468 } 1469 ret = amdgpu_bo_sync_wait(vm->root.bo, 1470 AMDGPU_FENCE_OWNER_KFD, false); 1471 if (ret) 1472 goto wait_pd_fail; 1473 ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); 1474 if (ret) 1475 goto reserve_shared_fail; 1476 dma_resv_add_fence(vm->root.bo->tbo.base.resv, 1477 &vm->process_info->eviction_fence->base, 1478 DMA_RESV_USAGE_BOOKKEEP); 1479 amdgpu_bo_unreserve(vm->root.bo); 1480 1481 /* Update process info */ 1482 mutex_lock(&vm->process_info->lock); 1483 list_add_tail(&vm->vm_list_node, 1484 &(vm->process_info->vm_list_head)); 1485 vm->process_info->n_vms++; 1486 if (ef) 1487 *ef = dma_fence_get(&vm->process_info->eviction_fence->base); 1488 mutex_unlock(&vm->process_info->lock); 1489 1490 return 0; 1491 1492 reserve_shared_fail: 1493 wait_pd_fail: 1494 validate_pd_fail: 1495 amdgpu_bo_unreserve(vm->root.bo); 1496 reserve_pd_fail: 1497 vm->process_info = NULL; 1498 already_acquired: 1499 if (info) { 1500 dma_fence_put(&info->eviction_fence->base); 1501 *process_info = NULL; 1502 put_pid(info->pid); 1503 create_evict_fence_fail: 1504 mutex_destroy(&info->lock); 1505 mutex_destroy(&info->notifier_lock); 1506 kfree(info); 1507 } 1508 return ret; 1509 } 1510 1511 /** 1512 * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria 1513 * @bo: Handle of buffer object being pinned 1514 * @domain: Domain into which BO should be pinned 1515 * 1516 * - USERPTR BOs are UNPINNABLE and will return error 1517 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1518 * PIN count incremented. It is valid to PIN a BO multiple times 1519 * 1520 * Return: ZERO if successful in pinning, Non-Zero in case of error. 1521 */ 1522 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) 1523 { 1524 int ret = 0; 1525 1526 ret = amdgpu_bo_reserve(bo, false); 1527 if (unlikely(ret)) 1528 return ret; 1529 1530 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) { 1531 /* 1532 * If bo is not contiguous on VRAM, move to system memory first to ensure 1533 * we can get contiguous VRAM space after evicting other BOs. 1534 */ 1535 if (!(bo->tbo.resource->placement & TTM_PL_FLAG_CONTIGUOUS)) { 1536 struct ttm_operation_ctx ctx = { true, false }; 1537 1538 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 1539 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1540 if (unlikely(ret)) { 1541 pr_debug("validate bo 0x%p to GTT failed %d\n", &bo->tbo, ret); 1542 goto out; 1543 } 1544 } 1545 } 1546 1547 ret = amdgpu_bo_pin(bo, domain); 1548 if (ret) 1549 pr_err("Error in Pinning BO to domain: %d\n", domain); 1550 1551 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 1552 1553 out: 1554 amdgpu_bo_unreserve(bo); 1555 return ret; 1556 } 1557 1558 /** 1559 * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria 1560 * @bo: Handle of buffer object being unpinned 1561 * 1562 * - Is a illegal request for USERPTR BOs and is ignored 1563 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1564 * PIN count decremented. Calls to UNPIN must balance calls to PIN 1565 */ 1566 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo) 1567 { 1568 int ret = 0; 1569 1570 ret = amdgpu_bo_reserve(bo, false); 1571 if (unlikely(ret)) 1572 return; 1573 1574 amdgpu_bo_unpin(bo); 1575 1576 amdgpu_bo_unreserve(bo); 1577 } 1578 1579 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev, 1580 struct amdgpu_vm *avm, 1581 void **process_info, 1582 struct dma_fence **ef) 1583 { 1584 int ret; 1585 1586 /* Already a compute VM? */ 1587 if (avm->process_info) 1588 return -EINVAL; 1589 1590 /* Convert VM into a compute VM */ 1591 ret = amdgpu_vm_make_compute(adev, avm); 1592 if (ret) 1593 return ret; 1594 1595 /* Initialize KFD part of the VM and process info */ 1596 ret = init_kfd_vm(avm, process_info, ef); 1597 if (ret) 1598 return ret; 1599 1600 amdgpu_vm_set_task_info(avm); 1601 1602 return 0; 1603 } 1604 1605 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, 1606 struct amdgpu_vm *vm) 1607 { 1608 struct amdkfd_process_info *process_info = vm->process_info; 1609 1610 if (!process_info) 1611 return; 1612 1613 /* Update process info */ 1614 mutex_lock(&process_info->lock); 1615 process_info->n_vms--; 1616 list_del(&vm->vm_list_node); 1617 mutex_unlock(&process_info->lock); 1618 1619 vm->process_info = NULL; 1620 1621 /* Release per-process resources when last compute VM is destroyed */ 1622 if (!process_info->n_vms) { 1623 WARN_ON(!list_empty(&process_info->kfd_bo_list)); 1624 WARN_ON(!list_empty(&process_info->userptr_valid_list)); 1625 WARN_ON(!list_empty(&process_info->userptr_inval_list)); 1626 1627 dma_fence_put(&process_info->eviction_fence->base); 1628 cancel_delayed_work_sync(&process_info->restore_userptr_work); 1629 put_pid(process_info->pid); 1630 mutex_destroy(&process_info->lock); 1631 mutex_destroy(&process_info->notifier_lock); 1632 kfree(process_info); 1633 } 1634 } 1635 1636 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv) 1637 { 1638 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1639 struct amdgpu_bo *pd = avm->root.bo; 1640 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 1641 1642 if (adev->asic_type < CHIP_VEGA10) 1643 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT; 1644 return avm->pd_phys_addr; 1645 } 1646 1647 void amdgpu_amdkfd_block_mmu_notifications(void *p) 1648 { 1649 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; 1650 1651 mutex_lock(&pinfo->lock); 1652 WRITE_ONCE(pinfo->block_mmu_notifications, true); 1653 mutex_unlock(&pinfo->lock); 1654 } 1655 1656 int amdgpu_amdkfd_criu_resume(void *p) 1657 { 1658 int ret = 0; 1659 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; 1660 1661 mutex_lock(&pinfo->lock); 1662 pr_debug("scheduling work\n"); 1663 mutex_lock(&pinfo->notifier_lock); 1664 pinfo->evicted_bos++; 1665 mutex_unlock(&pinfo->notifier_lock); 1666 if (!READ_ONCE(pinfo->block_mmu_notifications)) { 1667 ret = -EINVAL; 1668 goto out_unlock; 1669 } 1670 WRITE_ONCE(pinfo->block_mmu_notifications, false); 1671 queue_delayed_work(system_freezable_wq, 1672 &pinfo->restore_userptr_work, 0); 1673 1674 out_unlock: 1675 mutex_unlock(&pinfo->lock); 1676 return ret; 1677 } 1678 1679 /* Caller must hold kfd_mem_limit.mem_limit_lock. */ 1680 static size_t __get_available_memory(struct amdgpu_device *adev, 1681 uint8_t xcp_id, int64_t vram_used) 1682 { 1683 uint64_t reserved_for_pt = 1684 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); 1685 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1686 uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0); 1687 ssize_t available; 1688 uint64_t vram_available, system_mem_available, ttm_mem_available; 1689 1690 lockdep_assert_held(&kfd_mem_limit.mem_limit_lock); 1691 1692 if (adev->apu_prefer_gtt && !adev->gmc.is_app_apu) 1693 vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id) 1694 - vram_used; 1695 else 1696 vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id) 1697 - vram_used 1698 - atomic64_read(&adev->vram_pin_size_xcp[xcp_id]) 1699 - atomic64_read(&adev->vram_pin_size_common) 1700 + atomic64_read(&adev->kfd.vram_pinned[xcp_id]) 1701 - reserved_for_pt 1702 - reserved_for_ras; 1703 1704 if (adev->apu_prefer_gtt) { 1705 system_mem_available = no_system_mem_limit ? 1706 kfd_mem_limit.max_system_mem_limit : 1707 kfd_mem_limit.max_system_mem_limit - 1708 kfd_mem_limit.system_mem_used; 1709 1710 ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit - 1711 kfd_mem_limit.ttm_mem_used; 1712 1713 available = min3(system_mem_available, ttm_mem_available, 1714 vram_available); 1715 } else { 1716 available = vram_available; 1717 } 1718 1719 if (available < 0) 1720 available = 0; 1721 1722 return available; 1723 } 1724 1725 size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev, 1726 uint8_t xcp_id) 1727 { 1728 size_t available; 1729 1730 spin_lock(&kfd_mem_limit.mem_limit_lock); 1731 available = __get_available_memory(adev, xcp_id, 1732 adev->kfd.vram_used_aligned[xcp_id]); 1733 spin_unlock(&kfd_mem_limit.mem_limit_lock); 1734 1735 return available; 1736 } 1737 1738 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( 1739 struct amdgpu_device *adev, uint64_t va, uint64_t size, 1740 void *drm_priv, struct kgd_mem **mem, 1741 uint64_t *offset, uint32_t flags, bool criu_resume) 1742 { 1743 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1744 struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm); 1745 enum ttm_bo_type bo_type = ttm_bo_type_device; 1746 struct sg_table *sg = NULL; 1747 uint64_t user_addr = 0; 1748 struct amdgpu_bo *bo; 1749 struct drm_gem_object *gobj = NULL; 1750 u32 domain, alloc_domain; 1751 uint64_t aligned_size; 1752 int8_t xcp_id = -1; 1753 u64 alloc_flags; 1754 int ret; 1755 1756 /* 1757 * Check on which domain to allocate BO 1758 */ 1759 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 1760 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM; 1761 1762 if (adev->apu_prefer_gtt) { 1763 domain = AMDGPU_GEM_DOMAIN_GTT; 1764 alloc_domain = AMDGPU_GEM_DOMAIN_GTT; 1765 alloc_flags = 0; 1766 } else { 1767 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE | 1768 AMDGPU_GEM_CREATE_VRAM_CLEARED; 1769 alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ? 1770 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0; 1771 1772 /* For contiguous VRAM allocation */ 1773 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS) 1774 alloc_flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1775 } 1776 xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? 1777 0 : fpriv->xcp_id; 1778 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 1779 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT; 1780 alloc_flags = 0; 1781 } else { 1782 domain = AMDGPU_GEM_DOMAIN_GTT; 1783 alloc_domain = AMDGPU_GEM_DOMAIN_CPU; 1784 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE; 1785 1786 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 1787 if (!offset || !*offset) 1788 return -EINVAL; 1789 user_addr = untagged_addr(*offset); 1790 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1791 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1792 bo_type = ttm_bo_type_sg; 1793 if (size > UINT_MAX) 1794 return -EINVAL; 1795 sg = create_sg_table(*offset, size); 1796 if (!sg) 1797 return -ENOMEM; 1798 } else { 1799 return -EINVAL; 1800 } 1801 } 1802 1803 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT) 1804 alloc_flags |= AMDGPU_GEM_CREATE_COHERENT; 1805 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT) 1806 alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT; 1807 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED) 1808 alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED; 1809 1810 *mem = kzalloc_obj(struct kgd_mem); 1811 if (!*mem) { 1812 ret = -ENOMEM; 1813 goto err; 1814 } 1815 INIT_LIST_HEAD(&(*mem)->attachments); 1816 mutex_init(&(*mem)->lock); 1817 (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM); 1818 1819 /* Workaround for AQL queue wraparound bug. Map the same 1820 * memory twice. That means we only actually allocate half 1821 * the memory. 1822 */ 1823 if ((*mem)->aql_queue) 1824 size >>= 1; 1825 aligned_size = PAGE_ALIGN(size); 1826 1827 /* reject AQL queue with size < 2 */ 1828 if (!aligned_size) { 1829 ret = -EINVAL; 1830 goto err_alignment_size; 1831 } 1832 1833 (*mem)->alloc_flags = flags; 1834 1835 amdgpu_sync_create(&(*mem)->sync); 1836 1837 ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags, 1838 xcp_id); 1839 if (ret) { 1840 pr_debug("Insufficient memory\n"); 1841 goto err_reserve_limit; 1842 } 1843 1844 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n", 1845 va, (*mem)->aql_queue ? size << 1 : size, 1846 domain_string(alloc_domain), xcp_id); 1847 1848 ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags, 1849 bo_type, NULL, &gobj, xcp_id + 1); 1850 if (ret) { 1851 pr_debug("Failed to create BO on domain %s. ret %d\n", 1852 domain_string(alloc_domain), ret); 1853 goto err_bo_create; 1854 } 1855 ret = drm_vma_node_allow(&gobj->vma_node, drm_priv); 1856 if (ret) { 1857 pr_debug("Failed to allow vma node access. ret %d\n", ret); 1858 goto err_node_allow; 1859 } 1860 ret = drm_gem_handle_create(adev->kfd.client.file, gobj, &(*mem)->gem_handle); 1861 if (ret) 1862 goto err_gem_handle_create; 1863 bo = gem_to_amdgpu_bo(gobj); 1864 if (bo_type == ttm_bo_type_sg) { 1865 bo->tbo.sg = sg; 1866 bo->tbo.ttm->sg = sg; 1867 } 1868 bo->kfd_bo = *mem; 1869 (*mem)->bo = bo; 1870 if (user_addr) 1871 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO; 1872 1873 (*mem)->va = va; 1874 (*mem)->domain = domain; 1875 (*mem)->mapped_to_gpu_memory = 0; 1876 (*mem)->process_info = avm->process_info; 1877 1878 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr); 1879 1880 if (user_addr) { 1881 pr_debug("creating userptr BO for user_addr = %llx\n", user_addr); 1882 ret = init_user_pages(*mem, user_addr, criu_resume); 1883 if (ret) 1884 goto allocate_init_user_pages_failed; 1885 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1886 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1887 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT); 1888 if (ret) { 1889 pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n"); 1890 goto err_pin_bo; 1891 } 1892 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 1893 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 1894 } else { 1895 mutex_lock(&avm->process_info->lock); 1896 if (avm->process_info->eviction_fence && 1897 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base)) 1898 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain, 1899 &avm->process_info->eviction_fence->base); 1900 mutex_unlock(&avm->process_info->lock); 1901 if (ret) 1902 goto err_validate_bo; 1903 } 1904 1905 if (offset) 1906 *offset = amdgpu_bo_mmap_offset(bo); 1907 1908 return 0; 1909 1910 allocate_init_user_pages_failed: 1911 err_pin_bo: 1912 err_validate_bo: 1913 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); 1914 drm_gem_handle_delete(adev->kfd.client.file, (*mem)->gem_handle); 1915 err_gem_handle_create: 1916 drm_vma_node_revoke(&gobj->vma_node, drm_priv); 1917 err_node_allow: 1918 /* Don't unreserve system mem limit twice */ 1919 goto err_reserve_limit; 1920 err_bo_create: 1921 amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id); 1922 err_reserve_limit: 1923 amdgpu_sync_free(&(*mem)->sync); 1924 err_alignment_size: 1925 mutex_destroy(&(*mem)->lock); 1926 if (gobj) 1927 drm_gem_object_put(gobj); 1928 else 1929 kfree(*mem); 1930 err: 1931 if (sg) { 1932 sg_free_table(sg); 1933 kfree(sg); 1934 } 1935 return ret; 1936 } 1937 1938 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( 1939 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv, 1940 uint64_t *size) 1941 { 1942 struct amdkfd_process_info *process_info = mem->process_info; 1943 unsigned long bo_size = mem->bo->tbo.base.size; 1944 bool use_release_notifier = (mem->bo->kfd_bo == mem); 1945 struct kfd_mem_attachment *entry, *tmp; 1946 struct bo_vm_reservation_context ctx; 1947 unsigned int mapped_to_gpu_memory; 1948 int ret; 1949 bool is_imported = false; 1950 1951 mutex_lock(&mem->lock); 1952 1953 mapped_to_gpu_memory = mem->mapped_to_gpu_memory; 1954 is_imported = mem->is_imported; 1955 mutex_unlock(&mem->lock); 1956 /* lock is not needed after this, since mem is unused and will 1957 * be freed anyway 1958 */ 1959 1960 if (mapped_to_gpu_memory > 0) { 1961 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n", 1962 mem->va, bo_size); 1963 return -EBUSY; 1964 } 1965 1966 /* At this point the BO is guaranteed to be freed, so unpin the 1967 * MMIO/DOORBELL BOs that were pinned during allocation. 1968 */ 1969 if (mem->alloc_flags & 1970 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1971 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1972 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); 1973 } 1974 1975 /* Make sure restore workers don't access the BO any more */ 1976 mutex_lock(&process_info->lock); 1977 if (!list_empty(&mem->validate_list)) 1978 list_del_init(&mem->validate_list); 1979 mutex_unlock(&process_info->lock); 1980 1981 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); 1982 if (unlikely(ret)) 1983 return ret; 1984 1985 /* Cleanup user pages and MMU notifiers */ 1986 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 1987 amdgpu_hmm_unregister(mem->bo); 1988 amdgpu_hmm_range_free(mem->range); 1989 mem->range = NULL; 1990 } 1991 1992 amdgpu_amdkfd_remove_eviction_fence(mem->bo, 1993 process_info->eviction_fence); 1994 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, 1995 mem->va + bo_size * (1 + mem->aql_queue)); 1996 1997 /* Remove from VM internal data structures */ 1998 list_for_each_entry_safe(entry, tmp, &mem->attachments, list) { 1999 kfd_mem_dmaunmap_attachment(mem, entry); 2000 kfd_mem_detach(entry); 2001 } 2002 2003 ret = unreserve_bo_and_vms(&ctx, false, false); 2004 2005 /* Free the sync object */ 2006 amdgpu_sync_free(&mem->sync); 2007 2008 /* If the SG is not NULL, it's one we created for a doorbell or mmio 2009 * remap BO. We need to free it. 2010 */ 2011 if (mem->bo->tbo.sg) { 2012 sg_free_table(mem->bo->tbo.sg); 2013 kfree(mem->bo->tbo.sg); 2014 } 2015 2016 /* Update the size of the BO being freed if it was allocated from 2017 * VRAM and is not imported. For APP APU VRAM allocations are done 2018 * in GTT domain 2019 */ 2020 if (size) { 2021 if (!is_imported && 2022 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) 2023 *size = bo_size; 2024 else 2025 *size = 0; 2026 } 2027 2028 /* Free the BO*/ 2029 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv); 2030 drm_gem_handle_delete(adev->kfd.client.file, mem->gem_handle); 2031 if (mem->dmabuf) { 2032 dma_buf_put(mem->dmabuf); 2033 mem->dmabuf = NULL; 2034 } 2035 mutex_destroy(&mem->lock); 2036 2037 /* If this releases the last reference, it will end up calling 2038 * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why 2039 * this needs to be the last call here. 2040 */ 2041 drm_gem_object_put(&mem->bo->tbo.base); 2042 2043 /* 2044 * For kgd_mem allocated in import_obj_create() via 2045 * amdgpu_amdkfd_gpuvm_import_dmabuf_fd(), 2046 * explicitly free it here. 2047 */ 2048 if (!use_release_notifier) 2049 kfree(mem); 2050 2051 return ret; 2052 } 2053 2054 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( 2055 struct amdgpu_device *adev, struct kgd_mem *mem, 2056 void *drm_priv) 2057 { 2058 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2059 int ret; 2060 struct amdgpu_bo *bo; 2061 uint32_t domain; 2062 struct kfd_mem_attachment *entry; 2063 struct bo_vm_reservation_context ctx; 2064 unsigned long bo_size; 2065 bool is_invalid_userptr = false; 2066 2067 bo = mem->bo; 2068 if (!bo) { 2069 pr_err("Invalid BO when mapping memory to GPU\n"); 2070 return -EINVAL; 2071 } 2072 2073 /* Make sure restore is not running concurrently. Since we 2074 * don't map invalid userptr BOs, we rely on the next restore 2075 * worker to do the mapping 2076 */ 2077 mutex_lock(&mem->process_info->lock); 2078 2079 /* Lock notifier lock. If we find an invalid userptr BO, we can be 2080 * sure that the MMU notifier is no longer running 2081 * concurrently and the queues are actually stopped 2082 */ 2083 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 2084 mutex_lock(&mem->process_info->notifier_lock); 2085 is_invalid_userptr = !!mem->invalid; 2086 mutex_unlock(&mem->process_info->notifier_lock); 2087 } 2088 2089 mutex_lock(&mem->lock); 2090 2091 domain = mem->domain; 2092 bo_size = bo->tbo.base.size; 2093 2094 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", 2095 mem->va, 2096 mem->va + bo_size * (1 + mem->aql_queue), 2097 avm, domain_string(domain)); 2098 2099 if (!kfd_mem_is_attached(avm, mem)) { 2100 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue); 2101 if (ret) 2102 goto out; 2103 } 2104 2105 ret = reserve_bo_and_vm(mem, avm, &ctx); 2106 if (unlikely(ret)) 2107 goto out; 2108 2109 /* Userptr can be marked as "not invalid", but not actually be 2110 * validated yet (still in the system domain). In that case 2111 * the queues are still stopped and we can leave mapping for 2112 * the next restore worker 2113 */ 2114 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && 2115 bo->tbo.resource->mem_type == TTM_PL_SYSTEM) 2116 is_invalid_userptr = true; 2117 2118 ret = vm_validate_pt_pd_bos(avm, NULL); 2119 if (unlikely(ret)) 2120 goto out_unreserve; 2121 2122 list_for_each_entry(entry, &mem->attachments, list) { 2123 if (entry->bo_va->base.vm != avm || entry->is_mapped) 2124 continue; 2125 2126 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n", 2127 entry->va, entry->va + bo_size, entry); 2128 2129 ret = map_bo_to_gpuvm(mem, entry, ctx.sync, 2130 is_invalid_userptr); 2131 if (ret) { 2132 pr_err("Failed to map bo to gpuvm\n"); 2133 goto out_unreserve; 2134 } 2135 2136 ret = vm_update_pds(avm, ctx.sync); 2137 if (ret) { 2138 pr_err("Failed to update page directories\n"); 2139 goto out_unreserve; 2140 } 2141 2142 entry->is_mapped = true; 2143 mem->mapped_to_gpu_memory++; 2144 pr_debug("\t INC mapping count %d\n", 2145 mem->mapped_to_gpu_memory); 2146 } 2147 2148 ret = unreserve_bo_and_vms(&ctx, false, false); 2149 2150 goto out; 2151 2152 out_unreserve: 2153 unreserve_bo_and_vms(&ctx, false, false); 2154 out: 2155 mutex_unlock(&mem->process_info->lock); 2156 mutex_unlock(&mem->lock); 2157 return ret; 2158 } 2159 2160 int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv) 2161 { 2162 struct kfd_mem_attachment *entry; 2163 struct amdgpu_vm *vm; 2164 int ret; 2165 2166 vm = drm_priv_to_vm(drm_priv); 2167 2168 mutex_lock(&mem->lock); 2169 2170 ret = amdgpu_bo_reserve(mem->bo, true); 2171 if (ret) 2172 goto out; 2173 2174 list_for_each_entry(entry, &mem->attachments, list) { 2175 if (entry->bo_va->base.vm != vm) 2176 continue; 2177 if (entry->bo_va->base.bo->tbo.ttm && 2178 !entry->bo_va->base.bo->tbo.ttm->sg) 2179 continue; 2180 2181 kfd_mem_dmaunmap_attachment(mem, entry); 2182 } 2183 2184 amdgpu_bo_unreserve(mem->bo); 2185 out: 2186 mutex_unlock(&mem->lock); 2187 2188 return ret; 2189 } 2190 2191 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( 2192 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv) 2193 { 2194 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2195 unsigned long bo_size = mem->bo->tbo.base.size; 2196 struct kfd_mem_attachment *entry; 2197 struct bo_vm_reservation_context ctx; 2198 int ret; 2199 2200 mutex_lock(&mem->lock); 2201 2202 ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx); 2203 if (unlikely(ret)) 2204 goto out; 2205 /* If no VMs were reserved, it means the BO wasn't actually mapped */ 2206 if (ctx.n_vms == 0) { 2207 ret = -EINVAL; 2208 goto unreserve_out; 2209 } 2210 2211 ret = vm_validate_pt_pd_bos(avm, NULL); 2212 if (unlikely(ret)) 2213 goto unreserve_out; 2214 2215 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n", 2216 mem->va, 2217 mem->va + bo_size * (1 + mem->aql_queue), 2218 avm); 2219 2220 list_for_each_entry(entry, &mem->attachments, list) { 2221 if (entry->bo_va->base.vm != avm || !entry->is_mapped) 2222 continue; 2223 2224 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n", 2225 entry->va, entry->va + bo_size, entry); 2226 2227 ret = unmap_bo_from_gpuvm(mem, entry, ctx.sync); 2228 if (ret) 2229 goto unreserve_out; 2230 2231 entry->is_mapped = false; 2232 2233 mem->mapped_to_gpu_memory--; 2234 pr_debug("\t DEC mapping count %d\n", 2235 mem->mapped_to_gpu_memory); 2236 } 2237 2238 unreserve_out: 2239 unreserve_bo_and_vms(&ctx, false, false); 2240 out: 2241 mutex_unlock(&mem->lock); 2242 return ret; 2243 } 2244 2245 int amdgpu_amdkfd_gpuvm_sync_memory( 2246 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr) 2247 { 2248 struct amdgpu_sync sync; 2249 int ret; 2250 2251 amdgpu_sync_create(&sync); 2252 2253 mutex_lock(&mem->lock); 2254 amdgpu_sync_clone(&mem->sync, &sync); 2255 mutex_unlock(&mem->lock); 2256 2257 ret = amdgpu_sync_wait(&sync, intr); 2258 amdgpu_sync_free(&sync); 2259 return ret; 2260 } 2261 2262 /** 2263 * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count 2264 * @bo: Buffer object to be mapped 2265 * @bo_gart: Return bo reference 2266 * @process_info: KFD process info of the queue the BO is for 2267 * 2268 * Before return, bo reference count is incremented. To release the reference and unpin/ 2269 * unmap the BO, call amdgpu_amdkfd_free_kernel_mem. 2270 */ 2271 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart, 2272 struct amdkfd_process_info *process_info) 2273 { 2274 int ret; 2275 2276 ret = amdgpu_bo_reserve(bo, true); 2277 if (ret) { 2278 pr_err("Failed to reserve bo. ret %d\n", ret); 2279 goto err_reserve_bo_failed; 2280 } 2281 2282 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 2283 if (ret) { 2284 pr_err("Failed to pin bo. ret %d\n", ret); 2285 goto err_pin_bo_failed; 2286 } 2287 2288 ret = amdgpu_ttm_alloc_gart(&bo->tbo); 2289 if (ret) { 2290 pr_err("Failed to bind bo to GART. ret %d\n", ret); 2291 goto err_map_bo_gart_failed; 2292 } 2293 2294 /* bo->vm_bo can be the VM of any file which opened the BO */ 2295 amdgpu_amdkfd_remove_eviction_fence(bo, process_info->eviction_fence); 2296 2297 amdgpu_bo_unreserve(bo); 2298 2299 *bo_gart = amdgpu_bo_ref(bo); 2300 2301 return 0; 2302 2303 err_map_bo_gart_failed: 2304 amdgpu_bo_unpin(bo); 2305 err_pin_bo_failed: 2306 amdgpu_bo_unreserve(bo); 2307 err_reserve_bo_failed: 2308 2309 return ret; 2310 } 2311 2312 /** amdgpu_amdkfd_gpuvm_map_bo_to_kernel() - Map GTT or VRAM BO for kernel CPU access 2313 * 2314 * @mem: Buffer object to be mapped for CPU access 2315 * @kptr[out]: pointer in kernel CPU address space 2316 * @size[out]: size of the buffer 2317 * @domain[IN]: domain for pinning (AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_DOMAIN_VRAM, 2318 * or their combination to let the driver choose). CPU visibility is 2319 * automatically enforced by amdgpu_bo_pin() 2320 * 2321 * Pins the BO and maps it for kernel CPU access. The eviction fence is removed 2322 * from the BO, since pinned BOs cannot be evicted. The bo must remain on the 2323 * validate_list, so the GPU mapping can be restored after a page table was 2324 * evicted. 2325 * 2326 * Return: 0 on success, error code on failure 2327 */ 2328 int amdgpu_amdkfd_gpuvm_map_bo_to_kernel(struct kgd_mem *mem, void **kptr, 2329 u64 *size, u32 domain) 2330 { 2331 int ret; 2332 struct amdgpu_bo *bo = mem->bo; 2333 2334 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 2335 pr_err("userptr can't be mapped to kernel\n"); 2336 return -EINVAL; 2337 } 2338 2339 if (!(domain & (AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM))) { 2340 pr_debug("Invalid domain 0x%x for kernel mapping\n", domain); 2341 return -EINVAL; 2342 } 2343 2344 mutex_lock(&mem->process_info->lock); 2345 2346 ret = amdgpu_bo_reserve(bo, true); 2347 if (ret) { 2348 pr_err("Failed to reserve bo. ret %d\n", ret); 2349 goto bo_reserve_failed; 2350 } 2351 2352 ret = amdgpu_bo_pin(bo, domain); 2353 if (ret) { 2354 pr_err("Failed to pin bo. ret %d\n", ret); 2355 goto pin_failed; 2356 } 2357 2358 ret = amdgpu_bo_kmap(bo, kptr); 2359 if (ret) { 2360 pr_err("Failed to map bo to kernel. ret %d\n", ret); 2361 goto kmap_failed; 2362 } 2363 2364 amdgpu_amdkfd_remove_eviction_fence( 2365 bo, mem->process_info->eviction_fence); 2366 2367 if (size) 2368 *size = amdgpu_bo_size(bo); 2369 2370 amdgpu_bo_unreserve(bo); 2371 2372 mutex_unlock(&mem->process_info->lock); 2373 return 0; 2374 2375 kmap_failed: 2376 amdgpu_bo_unpin(bo); 2377 pin_failed: 2378 amdgpu_bo_unreserve(bo); 2379 bo_reserve_failed: 2380 mutex_unlock(&mem->process_info->lock); 2381 2382 return ret; 2383 } 2384 2385 /** amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel() - Unmap GTT or VRAM BO for kernel CPU access 2386 * 2387 * @mem: Buffer object to be unmapped for CPU access 2388 * 2389 * Removes the kernel CPU mapping and unpins the BO. It does not restore the 2390 * eviction fence, so this function should only be used for cleanup before the 2391 * BO is destroyed. 2392 */ 2393 void amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(struct kgd_mem *mem) 2394 { 2395 struct amdgpu_bo *bo = mem->bo; 2396 2397 (void)amdgpu_bo_reserve(bo, true); 2398 amdgpu_bo_kunmap(bo); 2399 amdgpu_bo_unpin(bo); 2400 amdgpu_bo_unreserve(bo); 2401 } 2402 2403 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev, 2404 struct kfd_vm_fault_info *mem) 2405 { 2406 if (atomic_read_acquire(&adev->gmc.vm_fault_info_updated) == 1) { 2407 *mem = *adev->gmc.vm_fault_info; 2408 atomic_set_release(&adev->gmc.vm_fault_info_updated, 0); 2409 } 2410 return 0; 2411 } 2412 2413 static int import_obj_create(struct amdgpu_device *adev, 2414 struct dma_buf *dma_buf, 2415 struct drm_gem_object *obj, 2416 uint64_t va, void *drm_priv, 2417 struct kgd_mem **mem, uint64_t *size, 2418 uint64_t *mmap_offset) 2419 { 2420 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2421 struct amdgpu_bo *bo; 2422 int ret; 2423 2424 bo = gem_to_amdgpu_bo(obj); 2425 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | 2426 AMDGPU_GEM_DOMAIN_GTT))) 2427 /* Only VRAM and GTT BOs are supported */ 2428 return -EINVAL; 2429 2430 *mem = kzalloc_obj(struct kgd_mem); 2431 if (!*mem) 2432 return -ENOMEM; 2433 2434 ret = drm_vma_node_allow(&obj->vma_node, drm_priv); 2435 if (ret) 2436 goto err_free_mem; 2437 2438 if (size) 2439 *size = amdgpu_bo_size(bo); 2440 2441 if (mmap_offset) 2442 *mmap_offset = amdgpu_bo_mmap_offset(bo); 2443 2444 INIT_LIST_HEAD(&(*mem)->attachments); 2445 mutex_init(&(*mem)->lock); 2446 2447 (*mem)->alloc_flags = 2448 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? 2449 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT) 2450 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE 2451 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 2452 2453 get_dma_buf(dma_buf); 2454 (*mem)->dmabuf = dma_buf; 2455 (*mem)->bo = bo; 2456 (*mem)->va = va; 2457 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && 2458 !adev->apu_prefer_gtt ? 2459 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT; 2460 2461 (*mem)->mapped_to_gpu_memory = 0; 2462 (*mem)->process_info = avm->process_info; 2463 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false); 2464 amdgpu_sync_create(&(*mem)->sync); 2465 (*mem)->is_imported = true; 2466 2467 mutex_lock(&avm->process_info->lock); 2468 if (avm->process_info->eviction_fence && 2469 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base)) 2470 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain, 2471 &avm->process_info->eviction_fence->base); 2472 mutex_unlock(&avm->process_info->lock); 2473 if (ret) 2474 goto err_remove_mem; 2475 2476 return 0; 2477 2478 err_remove_mem: 2479 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); 2480 amdgpu_sync_free(&(*mem)->sync); 2481 mutex_destroy(&(*mem)->lock); 2482 dma_buf_put(dma_buf); 2483 drm_vma_node_revoke(&obj->vma_node, drm_priv); 2484 err_free_mem: 2485 kfree(*mem); 2486 return ret; 2487 } 2488 2489 int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd, 2490 uint64_t va, void *drm_priv, 2491 struct kgd_mem **mem, uint64_t *size, 2492 uint64_t *mmap_offset) 2493 { 2494 struct drm_gem_object *obj; 2495 uint32_t handle; 2496 int ret; 2497 2498 ret = drm_gem_prime_fd_to_handle(&adev->ddev, adev->kfd.client.file, fd, 2499 &handle); 2500 if (ret) 2501 return ret; 2502 obj = drm_gem_object_lookup(adev->kfd.client.file, handle); 2503 if (!obj) { 2504 ret = -EINVAL; 2505 goto err_release_handle; 2506 } 2507 2508 ret = import_obj_create(adev, obj->dma_buf, obj, va, drm_priv, mem, size, 2509 mmap_offset); 2510 if (ret) 2511 goto err_put_obj; 2512 2513 (*mem)->gem_handle = handle; 2514 2515 return 0; 2516 2517 err_put_obj: 2518 drm_gem_object_put(obj); 2519 err_release_handle: 2520 drm_gem_handle_delete(adev->kfd.client.file, handle); 2521 return ret; 2522 } 2523 2524 int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem, 2525 struct dma_buf **dma_buf) 2526 { 2527 int ret; 2528 2529 mutex_lock(&mem->lock); 2530 ret = kfd_mem_export_dmabuf(mem); 2531 if (ret) 2532 goto out; 2533 2534 get_dma_buf(mem->dmabuf); 2535 *dma_buf = mem->dmabuf; 2536 out: 2537 mutex_unlock(&mem->lock); 2538 return ret; 2539 } 2540 2541 /* Evict a userptr BO by stopping the queues if necessary 2542 * 2543 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it 2544 * cannot do any memory allocations, and cannot take any locks that 2545 * are held elsewhere while allocating memory. 2546 * 2547 * It doesn't do anything to the BO itself. The real work happens in 2548 * restore, where we get updated page addresses. This function only 2549 * ensures that GPU access to the BO is stopped. 2550 */ 2551 int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni, 2552 unsigned long cur_seq, struct kgd_mem *mem) 2553 { 2554 struct amdkfd_process_info *process_info = mem->process_info; 2555 int r = 0; 2556 2557 /* Do not process MMU notifications during CRIU restore until 2558 * KFD_CRIU_OP_RESUME IOCTL is received 2559 */ 2560 if (READ_ONCE(process_info->block_mmu_notifications)) 2561 return 0; 2562 2563 mutex_lock(&process_info->notifier_lock); 2564 mmu_interval_set_seq(mni, cur_seq); 2565 2566 mem->invalid++; 2567 if (++process_info->evicted_bos == 1) { 2568 /* First eviction, stop the queues */ 2569 r = kgd2kfd_quiesce_mm(mni->mm, 2570 KFD_QUEUE_EVICTION_TRIGGER_USERPTR); 2571 2572 if (r && r != -ESRCH) 2573 pr_err("Failed to quiesce KFD\n"); 2574 2575 if (r != -ESRCH) 2576 queue_delayed_work(system_freezable_wq, 2577 &process_info->restore_userptr_work, 2578 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2579 } 2580 mutex_unlock(&process_info->notifier_lock); 2581 2582 return r; 2583 } 2584 2585 /* Update invalid userptr BOs 2586 * 2587 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to 2588 * userptr_inval_list and updates user pages for all BOs that have 2589 * been invalidated since their last update. 2590 */ 2591 static int update_invalid_user_pages(struct amdkfd_process_info *process_info, 2592 struct mm_struct *mm) 2593 { 2594 struct kgd_mem *mem, *tmp_mem; 2595 struct amdgpu_bo *bo; 2596 struct ttm_operation_ctx ctx = { false, false }; 2597 uint32_t invalid; 2598 int ret = 0; 2599 2600 mutex_lock(&process_info->notifier_lock); 2601 2602 /* Move all invalidated BOs to the userptr_inval_list */ 2603 list_for_each_entry_safe(mem, tmp_mem, 2604 &process_info->userptr_valid_list, 2605 validate_list) 2606 if (mem->invalid) 2607 list_move_tail(&mem->validate_list, 2608 &process_info->userptr_inval_list); 2609 2610 /* Go through userptr_inval_list and update any invalid user_pages */ 2611 list_for_each_entry(mem, &process_info->userptr_inval_list, 2612 validate_list) { 2613 invalid = mem->invalid; 2614 if (!invalid) 2615 /* BO hasn't been invalidated since the last 2616 * revalidation attempt. Keep its page list. 2617 */ 2618 continue; 2619 2620 bo = mem->bo; 2621 2622 amdgpu_hmm_range_free(mem->range); 2623 mem->range = NULL; 2624 2625 /* BO reservations and getting user pages (hmm_range_fault) 2626 * must happen outside the notifier lock 2627 */ 2628 mutex_unlock(&process_info->notifier_lock); 2629 2630 /* Move the BO to system (CPU) domain if necessary to unmap 2631 * and free the SG table 2632 */ 2633 if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) { 2634 if (amdgpu_bo_reserve(bo, true)) 2635 return -EAGAIN; 2636 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 2637 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2638 amdgpu_bo_unreserve(bo); 2639 if (ret) { 2640 pr_err("%s: Failed to invalidate userptr BO\n", 2641 __func__); 2642 return -EAGAIN; 2643 } 2644 } 2645 2646 mem->range = amdgpu_hmm_range_alloc(NULL); 2647 if (unlikely(!mem->range)) 2648 return -ENOMEM; 2649 /* Get updated user pages */ 2650 ret = amdgpu_ttm_tt_get_user_pages(bo, mem->range); 2651 if (ret) { 2652 amdgpu_hmm_range_free(mem->range); 2653 mem->range = NULL; 2654 pr_debug("Failed %d to get user pages\n", ret); 2655 2656 /* Return -EFAULT bad address error as success. It will 2657 * fail later with a VM fault if the GPU tries to access 2658 * it. Better than hanging indefinitely with stalled 2659 * user mode queues. 2660 * 2661 * Return other error -EBUSY or -ENOMEM to retry restore 2662 */ 2663 if (ret != -EFAULT) 2664 return ret; 2665 2666 /* If applications unmap memory before destroying the userptr 2667 * from the KFD, trigger a segmentation fault in VM debug mode. 2668 */ 2669 if (amdgpu_ttm_adev(bo->tbo.bdev)->debug_vm_userptr) { 2670 struct kfd_process *p; 2671 2672 pr_err("Pid %d unmapped memory before destroying userptr at GPU addr 0x%llx\n", 2673 pid_nr(process_info->pid), mem->va); 2674 2675 // Send GPU VM fault to user space 2676 p = kfd_lookup_process_by_pid(process_info->pid); 2677 if (p) { 2678 kfd_signal_vm_fault_event_with_userptr(p, mem->va); 2679 kfd_unref_process(p); 2680 } 2681 } 2682 2683 ret = 0; 2684 } 2685 2686 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->range); 2687 2688 mutex_lock(&process_info->notifier_lock); 2689 2690 /* Mark the BO as valid unless it was invalidated 2691 * again concurrently. 2692 */ 2693 if (mem->invalid != invalid) { 2694 ret = -EAGAIN; 2695 goto unlock_out; 2696 } 2697 /* set mem valid if mem has hmm range associated */ 2698 if (mem->range) 2699 mem->invalid = 0; 2700 } 2701 2702 unlock_out: 2703 mutex_unlock(&process_info->notifier_lock); 2704 2705 return ret; 2706 } 2707 2708 /* Validate invalid userptr BOs 2709 * 2710 * Validates BOs on the userptr_inval_list. Also updates GPUVM page tables 2711 * with new page addresses and waits for the page table updates to complete. 2712 */ 2713 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) 2714 { 2715 struct ttm_operation_ctx ctx = { false, false }; 2716 struct amdgpu_sync sync; 2717 struct drm_exec exec; 2718 2719 struct amdgpu_vm *peer_vm; 2720 struct kgd_mem *mem, *tmp_mem; 2721 struct amdgpu_bo *bo; 2722 int ret; 2723 2724 amdgpu_sync_create(&sync); 2725 2726 drm_exec_init(&exec, 0, 0); 2727 /* Reserve all BOs and page tables for validation */ 2728 drm_exec_until_all_locked(&exec) { 2729 /* Reserve all the page directories */ 2730 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2731 vm_list_node) { 2732 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2); 2733 drm_exec_retry_on_contention(&exec); 2734 if (unlikely(ret)) 2735 goto unreserve_out; 2736 } 2737 2738 /* Reserve the userptr_inval_list entries to resv_list */ 2739 list_for_each_entry(mem, &process_info->userptr_inval_list, 2740 validate_list) { 2741 struct drm_gem_object *gobj; 2742 2743 gobj = &mem->bo->tbo.base; 2744 ret = drm_exec_prepare_obj(&exec, gobj, 1); 2745 drm_exec_retry_on_contention(&exec); 2746 if (unlikely(ret)) 2747 goto unreserve_out; 2748 } 2749 } 2750 2751 ret = process_validate_vms(process_info, NULL); 2752 if (ret) 2753 goto unreserve_out; 2754 2755 /* Validate BOs and update GPUVM page tables */ 2756 list_for_each_entry_safe(mem, tmp_mem, 2757 &process_info->userptr_inval_list, 2758 validate_list) { 2759 struct kfd_mem_attachment *attachment; 2760 2761 bo = mem->bo; 2762 2763 /* Validate the BO if we got user pages */ 2764 if (bo->tbo.ttm->pages[0]) { 2765 amdgpu_bo_placement_from_domain(bo, mem->domain); 2766 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2767 if (ret) { 2768 pr_err("%s: failed to validate BO\n", __func__); 2769 goto unreserve_out; 2770 } 2771 } 2772 2773 /* Update mapping. If the BO was not validated 2774 * (because we couldn't get user pages), this will 2775 * clear the page table entries, which will result in 2776 * VM faults if the GPU tries to access the invalid 2777 * memory. 2778 */ 2779 list_for_each_entry(attachment, &mem->attachments, list) { 2780 if (!attachment->is_mapped) 2781 continue; 2782 2783 kfd_mem_dmaunmap_attachment(mem, attachment); 2784 ret = update_gpuvm_pte(mem, attachment, &sync); 2785 if (ret) { 2786 pr_err("%s: update PTE failed\n", __func__); 2787 /* make sure this gets validated again */ 2788 mutex_lock(&process_info->notifier_lock); 2789 mem->invalid++; 2790 mutex_unlock(&process_info->notifier_lock); 2791 goto unreserve_out; 2792 } 2793 } 2794 } 2795 2796 /* Update page directories */ 2797 ret = process_update_pds(process_info, &sync); 2798 2799 unreserve_out: 2800 drm_exec_fini(&exec); 2801 amdgpu_sync_wait(&sync, false); 2802 amdgpu_sync_free(&sync); 2803 2804 return ret; 2805 } 2806 2807 /* Confirm that all user pages are valid while holding the notifier lock 2808 * 2809 * Moves valid BOs from the userptr_inval_list back to userptr_val_list. 2810 */ 2811 static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info) 2812 { 2813 struct kgd_mem *mem, *tmp_mem; 2814 int ret = 0; 2815 2816 list_for_each_entry_safe(mem, tmp_mem, 2817 &process_info->userptr_inval_list, 2818 validate_list) { 2819 bool valid; 2820 2821 /* keep mem without hmm range at userptr_inval_list */ 2822 if (!mem->range) 2823 continue; 2824 2825 /* Only check mem with hmm range associated */ 2826 valid = amdgpu_hmm_range_valid(mem->range); 2827 amdgpu_hmm_range_free(mem->range); 2828 2829 mem->range = NULL; 2830 if (!valid) { 2831 WARN(!mem->invalid, "Invalid BO not marked invalid"); 2832 ret = -EAGAIN; 2833 continue; 2834 } 2835 2836 if (mem->invalid) { 2837 WARN(1, "Valid BO is marked invalid"); 2838 ret = -EAGAIN; 2839 continue; 2840 } 2841 2842 list_move_tail(&mem->validate_list, 2843 &process_info->userptr_valid_list); 2844 } 2845 2846 return ret; 2847 } 2848 2849 /* Worker callback to restore evicted userptr BOs 2850 * 2851 * Tries to update and validate all userptr BOs. If successful and no 2852 * concurrent evictions happened, the queues are restarted. Otherwise, 2853 * reschedule for another attempt later. 2854 */ 2855 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) 2856 { 2857 struct delayed_work *dwork = to_delayed_work(work); 2858 struct amdkfd_process_info *process_info = 2859 container_of(dwork, struct amdkfd_process_info, 2860 restore_userptr_work); 2861 struct task_struct *usertask; 2862 struct mm_struct *mm; 2863 uint32_t evicted_bos; 2864 int ret; 2865 2866 mutex_lock(&process_info->notifier_lock); 2867 evicted_bos = process_info->evicted_bos; 2868 mutex_unlock(&process_info->notifier_lock); 2869 if (!evicted_bos) 2870 return; 2871 2872 /* Reference task and mm in case of concurrent process termination */ 2873 usertask = get_pid_task(process_info->pid, PIDTYPE_PID); 2874 if (!usertask) 2875 return; 2876 mm = get_task_mm(usertask); 2877 if (!mm) { 2878 put_task_struct(usertask); 2879 return; 2880 } 2881 2882 mutex_lock(&process_info->lock); 2883 2884 ret = update_invalid_user_pages(process_info, mm); 2885 if (ret) 2886 goto unlock_out; 2887 /* userptr_inval_list can be empty if all evicted userptr BOs 2888 * have been freed. In that case there is nothing to validate 2889 * and we can just restart the queues. 2890 */ 2891 if (!list_empty(&process_info->userptr_inval_list)) { 2892 ret = validate_invalid_user_pages(process_info); 2893 if (ret) 2894 goto unlock_out; 2895 } 2896 /* Final check for concurrent evicton and atomic update. If 2897 * another eviction happens after successful update, it will 2898 * be a first eviction that calls quiesce_mm. The eviction 2899 * reference counting inside KFD will handle this case. 2900 */ 2901 mutex_lock(&process_info->notifier_lock); 2902 if (process_info->evicted_bos != evicted_bos) { 2903 ret = -EAGAIN; 2904 goto unlock_notifier_out; 2905 } 2906 2907 if (confirm_valid_user_pages_locked(process_info)) { 2908 WARN(1, "User pages unexpectedly invalid"); 2909 ret = -EINVAL; 2910 goto unlock_notifier_out; 2911 } 2912 2913 process_info->evicted_bos = evicted_bos = 0; 2914 2915 if (kgd2kfd_resume_mm(mm)) { 2916 pr_err("%s: Failed to resume KFD\n", __func__); 2917 /* No recovery from this failure. Probably the CP is 2918 * hanging. No point trying again. 2919 */ 2920 } 2921 2922 unlock_notifier_out: 2923 mutex_unlock(&process_info->notifier_lock); 2924 unlock_out: 2925 mutex_unlock(&process_info->lock); 2926 2927 /* If validation failed, reschedule another attempt, unless the 2928 * device is gone for good 2929 */ 2930 if (evicted_bos && ret != -ENODEV) { 2931 queue_delayed_work(system_freezable_wq, 2932 &process_info->restore_userptr_work, 2933 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2934 2935 kfd_smi_event_queue_restore_rescheduled(mm); 2936 } 2937 mmput(mm); 2938 put_task_struct(usertask); 2939 } 2940 2941 static void replace_eviction_fence(struct dma_fence __rcu **ef, 2942 struct dma_fence *new_ef) 2943 { 2944 struct dma_fence *old_ef = rcu_replace_pointer(*ef, new_ef, true 2945 /* protected by process_info->lock */); 2946 2947 /* If we're replacing an unsignaled eviction fence, that fence will 2948 * never be signaled, and if anyone is still waiting on that fence, 2949 * they will hang forever. This should never happen. We should only 2950 * replace the fence in restore_work that only gets scheduled after 2951 * eviction work signaled the fence. 2952 */ 2953 WARN_ONCE(!dma_fence_is_signaled(old_ef), 2954 "Replacing unsignaled eviction fence"); 2955 dma_fence_put(old_ef); 2956 } 2957 2958 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given 2959 * KFD process identified by process_info 2960 * 2961 * @process_info: amdkfd_process_info of the KFD process 2962 * 2963 * After memory eviction, restore thread calls this function. The function 2964 * should be called when the Process is still valid. BO restore involves - 2965 * 2966 * 1. Release old eviction fence and create new one 2967 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list. 2968 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of 2969 * BOs that need to be reserved. 2970 * 4. Reserve all the BOs 2971 * 5. Validate of PD and PT BOs. 2972 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence 2973 * 7. Add fence to all PD and PT BOs. 2974 * 8. Unreserve all BOs 2975 */ 2976 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu **ef) 2977 { 2978 struct amdkfd_process_info *process_info = info; 2979 struct amdgpu_vm *peer_vm; 2980 struct kgd_mem *mem; 2981 struct list_head duplicate_save; 2982 struct amdgpu_sync sync_obj; 2983 unsigned long failed_size = 0; 2984 unsigned long total_size = 0; 2985 struct drm_exec exec; 2986 int ret; 2987 2988 INIT_LIST_HEAD(&duplicate_save); 2989 2990 mutex_lock(&process_info->lock); 2991 2992 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 2993 drm_exec_until_all_locked(&exec) { 2994 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2995 vm_list_node) { 2996 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2); 2997 drm_exec_retry_on_contention(&exec); 2998 if (unlikely(ret)) { 2999 pr_err("Locking VM PD failed, ret: %d\n", ret); 3000 goto ttm_reserve_fail; 3001 } 3002 } 3003 3004 /* Reserve all BOs and page tables/directory. Add all BOs from 3005 * kfd_bo_list to ctx.list 3006 */ 3007 list_for_each_entry(mem, &process_info->kfd_bo_list, 3008 validate_list) { 3009 struct drm_gem_object *gobj; 3010 3011 gobj = &mem->bo->tbo.base; 3012 ret = drm_exec_prepare_obj(&exec, gobj, 1); 3013 drm_exec_retry_on_contention(&exec); 3014 if (unlikely(ret)) { 3015 pr_err("drm_exec_prepare_obj failed, ret: %d\n", ret); 3016 goto ttm_reserve_fail; 3017 } 3018 } 3019 } 3020 3021 amdgpu_sync_create(&sync_obj); 3022 3023 /* Validate BOs managed by KFD */ 3024 list_for_each_entry(mem, &process_info->kfd_bo_list, 3025 validate_list) { 3026 3027 struct amdgpu_bo *bo = mem->bo; 3028 uint32_t domain = mem->domain; 3029 struct dma_resv_iter cursor; 3030 struct dma_fence *fence; 3031 3032 total_size += amdgpu_bo_size(bo); 3033 3034 ret = amdgpu_amdkfd_bo_validate(bo, domain, false); 3035 if (ret) { 3036 pr_debug("Memory eviction: Validate BOs failed\n"); 3037 failed_size += amdgpu_bo_size(bo); 3038 ret = amdgpu_amdkfd_bo_validate(bo, 3039 AMDGPU_GEM_DOMAIN_GTT, false); 3040 if (ret) { 3041 pr_debug("Memory eviction: Try again\n"); 3042 goto validate_map_fail; 3043 } 3044 } 3045 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv, 3046 DMA_RESV_USAGE_KERNEL, fence) { 3047 ret = amdgpu_sync_fence(&sync_obj, fence, GFP_KERNEL); 3048 if (ret) { 3049 pr_debug("Memory eviction: Sync BO fence failed. Try again\n"); 3050 goto validate_map_fail; 3051 } 3052 } 3053 } 3054 3055 if (failed_size) 3056 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size); 3057 3058 /* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO 3059 * validations above would invalidate DMABuf imports again. 3060 */ 3061 ret = process_validate_vms(process_info, drm_exec_ticket(&exec)); 3062 if (ret) { 3063 pr_debug("Validating VMs failed, ret: %d\n", ret); 3064 goto validate_map_fail; 3065 } 3066 3067 /* Update mappings managed by KFD. */ 3068 list_for_each_entry(mem, &process_info->kfd_bo_list, 3069 validate_list) { 3070 struct kfd_mem_attachment *attachment; 3071 3072 list_for_each_entry(attachment, &mem->attachments, list) { 3073 if (!attachment->is_mapped) 3074 continue; 3075 3076 kfd_mem_dmaunmap_attachment(mem, attachment); 3077 ret = update_gpuvm_pte(mem, attachment, &sync_obj); 3078 if (ret) { 3079 pr_debug("Memory eviction: update PTE failed. Try again\n"); 3080 goto validate_map_fail; 3081 } 3082 } 3083 } 3084 3085 /* Update mappings not managed by KFD */ 3086 list_for_each_entry(peer_vm, &process_info->vm_list_head, 3087 vm_list_node) { 3088 struct amdgpu_device *adev = amdgpu_ttm_adev( 3089 peer_vm->root.bo->tbo.bdev); 3090 3091 struct amdgpu_fpriv *fpriv = 3092 container_of(peer_vm, struct amdgpu_fpriv, vm); 3093 3094 ret = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 3095 if (ret) { 3096 dev_dbg(adev->dev, 3097 "Memory eviction: handle PRT moved failed, pid %8d. Try again.\n", 3098 pid_nr(process_info->pid)); 3099 goto validate_map_fail; 3100 } 3101 3102 ret = amdgpu_vm_handle_moved(adev, peer_vm, drm_exec_ticket(&exec)); 3103 if (ret) { 3104 dev_dbg(adev->dev, 3105 "Memory eviction: handle moved failed, pid %8d. Try again.\n", 3106 pid_nr(process_info->pid)); 3107 goto validate_map_fail; 3108 } 3109 } 3110 3111 /* Update page directories */ 3112 ret = process_update_pds(process_info, &sync_obj); 3113 if (ret) { 3114 pr_debug("Memory eviction: update PDs failed. Try again\n"); 3115 goto validate_map_fail; 3116 } 3117 3118 /* Sync with fences on all the page tables. They implicitly depend on any 3119 * move fences from amdgpu_vm_handle_moved above. 3120 */ 3121 ret = process_sync_pds_resv(process_info, &sync_obj); 3122 if (ret) { 3123 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n"); 3124 goto validate_map_fail; 3125 } 3126 3127 /* Wait for validate and PT updates to finish */ 3128 amdgpu_sync_wait(&sync_obj, false); 3129 3130 /* The old eviction fence may be unsignaled if restore happens 3131 * after a GPU reset or suspend/resume. Keep the old fence in that 3132 * case. Otherwise release the old eviction fence and create new 3133 * one, because fence only goes from unsignaled to signaled once 3134 * and cannot be reused. Use context and mm from the old fence. 3135 * 3136 * If an old eviction fence signals after this check, that's OK. 3137 * Anyone signaling an eviction fence must stop the queues first 3138 * and schedule another restore worker. 3139 */ 3140 if (dma_fence_is_signaled(&process_info->eviction_fence->base)) { 3141 struct amdgpu_amdkfd_fence *new_fence = 3142 amdgpu_amdkfd_fence_create( 3143 process_info->eviction_fence->base.context, 3144 process_info->eviction_fence->mm, 3145 process_info->context_id); 3146 3147 if (!new_fence) { 3148 pr_err("Failed to create eviction fence\n"); 3149 ret = -ENOMEM; 3150 goto validate_map_fail; 3151 } 3152 dma_fence_put(&process_info->eviction_fence->base); 3153 process_info->eviction_fence = new_fence; 3154 replace_eviction_fence(ef, dma_fence_get(&new_fence->base)); 3155 } else { 3156 WARN_ONCE(rcu_access_pointer(*ef) != &process_info->eviction_fence->base, 3157 "KFD eviction fence doesn't match KGD process_info"); 3158 } 3159 3160 /* Attach new eviction fence to all BOs except pinned ones */ 3161 list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) { 3162 if (mem->bo->tbo.pin_count) 3163 continue; 3164 3165 dma_resv_add_fence(mem->bo->tbo.base.resv, 3166 &process_info->eviction_fence->base, 3167 DMA_RESV_USAGE_BOOKKEEP); 3168 } 3169 /* Attach eviction fence to PD / PT BOs and DMABuf imports */ 3170 list_for_each_entry(peer_vm, &process_info->vm_list_head, 3171 vm_list_node) { 3172 struct amdgpu_bo *bo = peer_vm->root.bo; 3173 3174 dma_resv_add_fence(bo->tbo.base.resv, 3175 &process_info->eviction_fence->base, 3176 DMA_RESV_USAGE_BOOKKEEP); 3177 } 3178 3179 validate_map_fail: 3180 amdgpu_sync_free(&sync_obj); 3181 ttm_reserve_fail: 3182 drm_exec_fini(&exec); 3183 mutex_unlock(&process_info->lock); 3184 return ret; 3185 } 3186 3187 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem) 3188 { 3189 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 3190 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws; 3191 int ret; 3192 3193 if (!info || !gws) 3194 return -EINVAL; 3195 3196 *mem = kzalloc_obj(struct kgd_mem); 3197 if (!*mem) 3198 return -ENOMEM; 3199 3200 mutex_init(&(*mem)->lock); 3201 INIT_LIST_HEAD(&(*mem)->attachments); 3202 (*mem)->bo = amdgpu_bo_ref(gws_bo); 3203 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; 3204 (*mem)->process_info = process_info; 3205 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false); 3206 amdgpu_sync_create(&(*mem)->sync); 3207 3208 3209 /* Validate gws bo the first time it is added to process */ 3210 mutex_lock(&(*mem)->process_info->lock); 3211 ret = amdgpu_bo_reserve(gws_bo, false); 3212 if (unlikely(ret)) { 3213 pr_err("Reserve gws bo failed %d\n", ret); 3214 goto bo_reservation_failure; 3215 } 3216 3217 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true); 3218 if (ret) { 3219 pr_err("GWS BO validate failed %d\n", ret); 3220 goto bo_validation_failure; 3221 } 3222 /* GWS resource is shared b/t amdgpu and amdkfd 3223 * Add process eviction fence to bo so they can 3224 * evict each other. 3225 */ 3226 ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1); 3227 if (ret) 3228 goto reserve_shared_fail; 3229 dma_resv_add_fence(gws_bo->tbo.base.resv, 3230 &process_info->eviction_fence->base, 3231 DMA_RESV_USAGE_BOOKKEEP); 3232 amdgpu_bo_unreserve(gws_bo); 3233 mutex_unlock(&(*mem)->process_info->lock); 3234 3235 return ret; 3236 3237 reserve_shared_fail: 3238 bo_validation_failure: 3239 amdgpu_bo_unreserve(gws_bo); 3240 bo_reservation_failure: 3241 mutex_unlock(&(*mem)->process_info->lock); 3242 amdgpu_sync_free(&(*mem)->sync); 3243 remove_kgd_mem_from_kfd_bo_list(*mem, process_info); 3244 amdgpu_bo_unref(&gws_bo); 3245 mutex_destroy(&(*mem)->lock); 3246 kfree(*mem); 3247 *mem = NULL; 3248 return ret; 3249 } 3250 3251 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem) 3252 { 3253 int ret; 3254 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 3255 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem; 3256 struct amdgpu_bo *gws_bo = kgd_mem->bo; 3257 3258 /* Remove BO from process's validate list so restore worker won't touch 3259 * it anymore 3260 */ 3261 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info); 3262 3263 ret = amdgpu_bo_reserve(gws_bo, false); 3264 if (unlikely(ret)) { 3265 pr_err("Reserve gws bo failed %d\n", ret); 3266 //TODO add BO back to validate_list? 3267 return ret; 3268 } 3269 amdgpu_amdkfd_remove_eviction_fence(gws_bo, 3270 process_info->eviction_fence); 3271 amdgpu_bo_unreserve(gws_bo); 3272 amdgpu_sync_free(&kgd_mem->sync); 3273 amdgpu_bo_unref(&gws_bo); 3274 mutex_destroy(&kgd_mem->lock); 3275 kfree(mem); 3276 return 0; 3277 } 3278 3279 /* Returns GPU-specific tiling mode information */ 3280 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev, 3281 struct tile_config *config) 3282 { 3283 config->gb_addr_config = adev->gfx.config.gb_addr_config; 3284 config->tile_config_ptr = adev->gfx.config.tile_mode_array; 3285 config->num_tile_configs = 3286 ARRAY_SIZE(adev->gfx.config.tile_mode_array); 3287 config->macro_tile_config_ptr = 3288 adev->gfx.config.macrotile_mode_array; 3289 config->num_macro_tile_configs = 3290 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 3291 3292 /* Those values are not set from GFX9 onwards */ 3293 config->num_banks = adev->gfx.config.num_banks; 3294 config->num_ranks = adev->gfx.config.num_ranks; 3295 3296 return 0; 3297 } 3298 3299 bool amdgpu_amdkfd_bo_mapped_to_dev(void *drm_priv, struct kgd_mem *mem) 3300 { 3301 struct amdgpu_vm *vm = drm_priv_to_vm(drm_priv); 3302 struct kfd_mem_attachment *entry; 3303 3304 list_for_each_entry(entry, &mem->attachments, list) { 3305 if (entry->is_mapped && entry->bo_va->base.vm == vm) 3306 return true; 3307 } 3308 return false; 3309 } 3310 3311 #if defined(CONFIG_DEBUG_FS) 3312 3313 int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data) 3314 { 3315 3316 spin_lock(&kfd_mem_limit.mem_limit_lock); 3317 seq_printf(m, "System mem used %lldM out of %lluM\n", 3318 (kfd_mem_limit.system_mem_used >> 20), 3319 (kfd_mem_limit.max_system_mem_limit >> 20)); 3320 seq_printf(m, "TTM mem used %lldM out of %lluM\n", 3321 (kfd_mem_limit.ttm_mem_used >> 20), 3322 (kfd_mem_limit.max_ttm_mem_limit >> 20)); 3323 spin_unlock(&kfd_mem_limit.mem_limit_lock); 3324 3325 return 0; 3326 } 3327 3328 #endif 3329