1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2014-2018 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 #include <linux/dma-buf.h> 24 #include <linux/list.h> 25 #include <linux/pagemap.h> 26 #include <linux/sched/mm.h> 27 #include <linux/sched/task.h> 28 #include <drm/ttm/ttm_tt.h> 29 30 #include <drm/drm_exec.h> 31 32 #include "amdgpu_object.h" 33 #include "amdgpu_gem.h" 34 #include "amdgpu_vm.h" 35 #include "amdgpu_hmm.h" 36 #include "amdgpu_amdkfd.h" 37 #include "amdgpu_dma_buf.h" 38 #include <uapi/linux/kfd_ioctl.h> 39 #include "amdgpu_xgmi.h" 40 #include "kfd_priv.h" 41 #include "kfd_smi_events.h" 42 43 /* Userptr restore delay, just long enough to allow consecutive VM 44 * changes to accumulate 45 */ 46 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 47 #define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29) 48 49 /* Impose limit on how much memory KFD can use */ 50 static struct { 51 uint64_t max_system_mem_limit; 52 uint64_t max_ttm_mem_limit; 53 int64_t system_mem_used; 54 int64_t ttm_mem_used; 55 spinlock_t mem_limit_lock; 56 } kfd_mem_limit; 57 58 static const char * const domain_bit_to_string[] = { 59 "CPU", 60 "GTT", 61 "VRAM", 62 "GDS", 63 "GWS", 64 "OA" 65 }; 66 67 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] 68 69 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); 70 static size_t __get_available_memory(struct amdgpu_device *adev, 71 uint8_t xcp_id, int64_t vram_used); 72 73 static bool kfd_mem_is_attached(struct amdgpu_vm *avm, 74 struct kgd_mem *mem) 75 { 76 struct kfd_mem_attachment *entry; 77 78 list_for_each_entry(entry, &mem->attachments, list) 79 if (entry->bo_va->base.vm == avm) 80 return true; 81 82 return false; 83 } 84 85 /** 86 * reuse_dmamap() - Check whether adev can share the original 87 * userptr BO 88 * 89 * If both adev and bo_adev are in direct mapping or 90 * in the same iommu group, they can share the original BO. 91 * 92 * @adev: Device to which can or cannot share the original BO 93 * @bo_adev: Device to which allocated BO belongs to 94 * 95 * Return: returns true if adev can share original userptr BO, 96 * false otherwise. 97 */ 98 static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev) 99 { 100 return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) || 101 (adev->dev->iommu_group == bo_adev->dev->iommu_group); 102 } 103 104 /* Set memory usage limits. Current, limits are 105 * System (TTM + userptr) memory - 15/16th System RAM 106 * TTM memory - 3/8th System RAM 107 */ 108 void amdgpu_amdkfd_gpuvm_init_mem_limits(void) 109 { 110 struct sysinfo si; 111 uint64_t mem; 112 113 if (kfd_mem_limit.max_system_mem_limit) 114 return; 115 116 si_meminfo(&si); 117 mem = si.totalram - si.totalhigh; 118 mem *= si.mem_unit; 119 120 spin_lock_init(&kfd_mem_limit.mem_limit_lock); 121 kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6); 122 if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT) 123 kfd_mem_limit.max_system_mem_limit >>= 1; 124 else 125 kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT; 126 127 kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT; 128 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", 129 (kfd_mem_limit.max_system_mem_limit >> 20), 130 (kfd_mem_limit.max_ttm_mem_limit >> 20)); 131 } 132 133 void amdgpu_amdkfd_reserve_system_mem(uint64_t size) 134 { 135 kfd_mem_limit.system_mem_used += size; 136 } 137 138 /* Estimate page table size needed to represent a given memory size 139 * 140 * With 4KB pages, we need one 8 byte PTE for each 4KB of memory 141 * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB 142 * of memory (factor 256K, >> 18). ROCm user mode tries to optimize 143 * for 2MB pages for TLB efficiency. However, small allocations and 144 * fragmented system memory still need some 4KB pages. We choose a 145 * compromise that should work in most cases without reserving too 146 * much memory for page tables unnecessarily (factor 16K, >> 14). 147 */ 148 149 #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) 150 151 /** 152 * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size 153 * of buffer. 154 * 155 * @adev: Device to which allocated BO belongs to 156 * @size: Size of buffer, in bytes, encapsulated by B0. This should be 157 * equivalent to amdgpu_bo_size(BO) 158 * @alloc_flag: Flag used in allocating a BO as noted above 159 * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is 160 * managed as one compute node in driver for app 161 * 162 * Return: 163 * returns -ENOMEM in case of error, ZERO otherwise 164 */ 165 int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, 166 uint64_t size, u32 alloc_flag, int8_t xcp_id) 167 { 168 size_t system_mem_needed, ttm_mem_needed, vram_needed; 169 int ret = 0; 170 171 system_mem_needed = 0; 172 ttm_mem_needed = 0; 173 vram_needed = 0; 174 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 175 system_mem_needed = size; 176 ttm_mem_needed = size; 177 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 178 /* 179 * Conservatively round up the allocation requirement to 2 MB 180 * to avoid fragmentation caused by 4K allocations in the tail 181 * 2M BO chunk. 182 */ 183 vram_needed = size; 184 /* 185 * For GFX 9.4.3, get the VRAM size from XCP structs 186 */ 187 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) 188 return -EINVAL; 189 190 if (adev->apu_prefer_gtt) { 191 system_mem_needed = size; 192 ttm_mem_needed = size; 193 } 194 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 195 system_mem_needed = size; 196 } else if (!(alloc_flag & 197 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 198 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 199 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 200 return -ENOMEM; 201 } 202 203 spin_lock(&kfd_mem_limit.mem_limit_lock); 204 205 if (kfd_mem_limit.system_mem_used + system_mem_needed > 206 kfd_mem_limit.max_system_mem_limit) { 207 pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); 208 if (!no_system_mem_limit) { 209 ret = -ENOMEM; 210 goto release; 211 } 212 } 213 214 if (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > 215 kfd_mem_limit.max_ttm_mem_limit) { 216 ret = -ENOMEM; 217 goto release; 218 } 219 220 /*if is_app_apu is false and apu_prefer_gtt is true, it is an APU with 221 * carve out < gtt. In that case, VRAM allocation will go to gtt domain, skip 222 * VRAM check since ttm_mem_limit check already cover this allocation 223 */ 224 225 if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu) && 226 vram_needed > __get_available_memory(adev, xcp_id, 227 adev->kfd.vram_used[xcp_id])) { 228 ret = -ENOMEM; 229 goto release; 230 } 231 232 /* Update memory accounting by decreasing available system 233 * memory, TTM memory and GPU memory as computed above 234 */ 235 WARN_ONCE(vram_needed && !adev, 236 "adev reference can't be null when vram is used"); 237 if (adev && xcp_id >= 0) { 238 adev->kfd.vram_used[xcp_id] += vram_needed; 239 adev->kfd.vram_used_aligned[xcp_id] += 240 adev->apu_prefer_gtt ? 241 vram_needed : 242 ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); 243 } 244 kfd_mem_limit.system_mem_used += system_mem_needed; 245 kfd_mem_limit.ttm_mem_used += ttm_mem_needed; 246 247 release: 248 spin_unlock(&kfd_mem_limit.mem_limit_lock); 249 return ret; 250 } 251 252 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, 253 uint64_t size, u32 alloc_flag, int8_t xcp_id) 254 { 255 spin_lock(&kfd_mem_limit.mem_limit_lock); 256 257 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 258 kfd_mem_limit.system_mem_used -= size; 259 kfd_mem_limit.ttm_mem_used -= size; 260 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 261 WARN_ONCE(!adev, 262 "adev reference can't be null when alloc mem flags vram is set"); 263 if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) 264 goto release; 265 266 if (adev) { 267 adev->kfd.vram_used[xcp_id] -= size; 268 if (adev->apu_prefer_gtt) { 269 adev->kfd.vram_used_aligned[xcp_id] -= size; 270 kfd_mem_limit.system_mem_used -= size; 271 kfd_mem_limit.ttm_mem_used -= size; 272 } else { 273 adev->kfd.vram_used_aligned[xcp_id] -= 274 ALIGN(size, VRAM_AVAILABLITY_ALIGN); 275 } 276 } 277 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 278 kfd_mem_limit.system_mem_used -= size; 279 } else if (!(alloc_flag & 280 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 281 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 282 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 283 goto release; 284 } 285 WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0, 286 "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id); 287 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, 288 "KFD TTM memory accounting unbalanced"); 289 WARN_ONCE(kfd_mem_limit.system_mem_used < 0, 290 "KFD system memory accounting unbalanced"); 291 292 release: 293 spin_unlock(&kfd_mem_limit.mem_limit_lock); 294 } 295 296 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) 297 { 298 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 299 u32 alloc_flags = bo->kfd_bo->alloc_flags; 300 u64 size = amdgpu_bo_size(bo); 301 302 amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags, 303 bo->xcp_id); 304 305 kfree(bo->kfd_bo); 306 } 307 308 void amdgpu_amdkfd_account_vram_pin(struct amdgpu_bo *bo, int xcp_id) 309 { 310 if (!bo->kfd_bo) 311 return; 312 313 if (WARN_ONCE(xcp_id < 0, "KFD BO pinned in VRAM with no partition")) 314 return; 315 316 atomic64_add(amdgpu_bo_size(bo), 317 &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]); 318 } 319 320 void amdgpu_amdkfd_account_vram_unpin(struct amdgpu_bo *bo, int xcp_id) 321 { 322 if (!bo->kfd_bo) 323 return; 324 325 if (WARN_ONCE(xcp_id < 0, "KFD BO unpinned in VRAM with no partition")) 326 return; 327 328 atomic64_sub(amdgpu_bo_size(bo), 329 &amdgpu_ttm_adev(bo->tbo.bdev)->kfd.vram_pinned[xcp_id]); 330 } 331 332 /** 333 * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information 334 * about USERPTR or DOOREBELL or MMIO BO. 335 * 336 * @adev: Device for which dmamap BO is being created 337 * @mem: BO of peer device that is being DMA mapped. Provides parameters 338 * in building the dmamap BO 339 * @bo_out: Output parameter updated with handle of dmamap BO 340 */ 341 static int 342 create_dmamap_sg_bo(struct amdgpu_device *adev, 343 struct kgd_mem *mem, struct amdgpu_bo **bo_out) 344 { 345 struct drm_gem_object *gem_obj; 346 int ret; 347 uint64_t flags = 0; 348 349 ret = amdgpu_bo_reserve(mem->bo, false); 350 if (ret) 351 return ret; 352 353 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) 354 flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT | 355 AMDGPU_GEM_CREATE_UNCACHED); 356 357 ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1, 358 AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags, 359 ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0); 360 361 amdgpu_bo_unreserve(mem->bo); 362 363 if (ret) { 364 pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); 365 return -EINVAL; 366 } 367 368 *bo_out = gem_to_amdgpu_bo(gem_obj); 369 (*bo_out)->parent = amdgpu_bo_ref(mem->bo); 370 return ret; 371 } 372 373 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's 374 * reservation object. 375 * 376 * @bo: [IN] Remove eviction fence(s) from this BO 377 * @ef: [IN] This eviction fence is removed if it 378 * is present in the shared list. 379 * 380 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. 381 */ 382 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, 383 struct amdgpu_amdkfd_fence *ef) 384 { 385 struct dma_fence *replacement; 386 387 if (!ef) 388 return -EINVAL; 389 390 /* TODO: Instead of block before we should use the fence of the page 391 * table update and TLB flush here directly. 392 */ 393 replacement = dma_fence_get_stub(); 394 dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, 395 replacement, DMA_RESV_USAGE_BOOKKEEP); 396 dma_fence_put(replacement); 397 return 0; 398 } 399 400 /** 401 * amdgpu_amdkfd_remove_all_eviction_fences - Remove all eviction fences 402 * @bo: the BO where to remove the evictions fences from. 403 * 404 * This functions should only be used on release when all references to the BO 405 * are already dropped. We remove the eviction fence from the private copy of 406 * the dma_resv object here since that is what is used during release to 407 * determine of the BO is idle or not. 408 */ 409 void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo) 410 { 411 struct dma_resv *resv = &bo->tbo.base._resv; 412 struct dma_fence *fence, *stub; 413 struct dma_resv_iter cursor; 414 415 dma_resv_assert_held(resv); 416 417 stub = dma_fence_get_stub(); 418 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) { 419 if (!to_amdgpu_amdkfd_fence(fence)) 420 continue; 421 422 dma_resv_replace_fences(resv, fence->context, stub, 423 DMA_RESV_USAGE_BOOKKEEP); 424 } 425 dma_fence_put(stub); 426 } 427 428 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, 429 bool wait) 430 { 431 struct ttm_operation_ctx ctx = { false, false }; 432 int ret; 433 434 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), 435 "Called with userptr BO")) 436 return -EINVAL; 437 438 /* bo has been pinned, not need validate it */ 439 if (bo->tbo.pin_count) 440 return 0; 441 442 amdgpu_bo_placement_from_domain(bo, domain); 443 444 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 445 if (ret) 446 goto validate_fail; 447 if (wait) 448 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 449 450 validate_fail: 451 return ret; 452 } 453 454 int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, 455 uint32_t domain, 456 struct dma_fence *fence) 457 { 458 int ret = amdgpu_bo_reserve(bo, false); 459 460 if (ret) 461 return ret; 462 463 ret = amdgpu_amdkfd_bo_validate(bo, domain, true); 464 if (ret) 465 goto unreserve_out; 466 467 ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1); 468 if (ret) 469 goto unreserve_out; 470 471 dma_resv_add_fence(bo->tbo.base.resv, fence, 472 DMA_RESV_USAGE_BOOKKEEP); 473 474 unreserve_out: 475 amdgpu_bo_unreserve(bo); 476 477 return ret; 478 } 479 480 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) 481 { 482 return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); 483 } 484 485 /* vm_validate_pt_pd_bos - Validate page table and directory BOs 486 * 487 * Page directories are not updated here because huge page handling 488 * during page table updates can invalidate page directory entries 489 * again. Page directories are only updated after updating page 490 * tables. 491 */ 492 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm, 493 struct ww_acquire_ctx *ticket) 494 { 495 struct amdgpu_bo *pd = vm->root.bo; 496 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 497 int ret; 498 499 ret = amdgpu_vm_validate(adev, vm, ticket, 500 amdgpu_amdkfd_validate_vm_bo, NULL); 501 if (ret) { 502 pr_err("failed to validate PT BOs\n"); 503 return ret; 504 } 505 506 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); 507 508 return 0; 509 } 510 511 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) 512 { 513 struct amdgpu_bo *pd = vm->root.bo; 514 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 515 int ret; 516 517 ret = amdgpu_vm_update_pdes(adev, vm, false); 518 if (ret) 519 return ret; 520 521 return amdgpu_sync_fence(sync, vm->last_update, GFP_KERNEL); 522 } 523 524 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct amdgpu_vm *vm, 525 struct kgd_mem *mem) 526 { 527 uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE | 528 AMDGPU_VM_MTYPE_DEFAULT; 529 530 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) 531 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; 532 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) 533 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; 534 535 return mapping_flags; 536 } 537 538 /** 539 * create_sg_table() - Create an sg_table for a contiguous DMA addr range 540 * @addr: The starting address to point to 541 * @size: Size of memory area in bytes being pointed to 542 * 543 * Allocates an instance of sg_table and initializes it to point to memory 544 * area specified by input parameters. The address used to build is assumed 545 * to be DMA mapped, if needed. 546 * 547 * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table 548 * because they are physically contiguous. 549 * 550 * Return: Initialized instance of SG Table or NULL 551 */ 552 static struct sg_table *create_sg_table(uint64_t addr, uint32_t size) 553 { 554 struct sg_table *sg = kmalloc_obj(*sg); 555 556 if (!sg) 557 return NULL; 558 if (sg_alloc_table(sg, 1, GFP_KERNEL)) { 559 kfree(sg); 560 return NULL; 561 } 562 sg_dma_address(sg->sgl) = addr; 563 sg->sgl->length = size; 564 #ifdef CONFIG_NEED_SG_DMA_LENGTH 565 sg->sgl->dma_length = size; 566 #endif 567 return sg; 568 } 569 570 static int 571 kfd_mem_dmamap_userptr(struct kgd_mem *mem, 572 struct kfd_mem_attachment *attachment) 573 { 574 enum dma_data_direction direction = 575 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 576 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 577 struct ttm_operation_ctx ctx = {.interruptible = true}; 578 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 579 struct amdgpu_device *adev = attachment->adev; 580 struct ttm_tt *src_ttm = mem->bo->tbo.ttm; 581 struct ttm_tt *ttm = bo->tbo.ttm; 582 int ret; 583 584 if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) 585 return -EINVAL; 586 587 ttm->sg = kmalloc_obj(*ttm->sg); 588 if (unlikely(!ttm->sg)) 589 return -ENOMEM; 590 591 /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ 592 ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, 593 ttm->num_pages, 0, 594 (u64)ttm->num_pages << PAGE_SHIFT, 595 GFP_KERNEL); 596 if (unlikely(ret)) 597 goto free_sg; 598 599 ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); 600 if (unlikely(ret)) 601 goto release_sg; 602 603 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 604 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 605 if (ret) 606 goto unmap_sg; 607 608 return 0; 609 610 unmap_sg: 611 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 612 release_sg: 613 pr_err("DMA map userptr failed: %d\n", ret); 614 sg_free_table(ttm->sg); 615 free_sg: 616 kfree(ttm->sg); 617 ttm->sg = NULL; 618 return ret; 619 } 620 621 static int 622 kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment) 623 { 624 struct ttm_operation_ctx ctx = {.interruptible = true}; 625 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 626 627 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 628 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 629 } 630 631 /** 632 * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO 633 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device 634 * @attachment: Virtual address attachment of the BO on accessing device 635 * 636 * An access request from the device that owns DOORBELL does not require DMA mapping. 637 * This is because the request doesn't go through PCIe root complex i.e. it instead 638 * loops back. The need to DMA map arises only when accessing peer device's DOORBELL 639 * 640 * In contrast, all access requests for MMIO need to be DMA mapped without regard to 641 * device ownership. This is because access requests for MMIO go through PCIe root 642 * complex. 643 * 644 * This is accomplished in two steps: 645 * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used 646 * in updating requesting device's page table 647 * - Signal TTM to mark memory pointed to by requesting device's BO as GPU 648 * accessible. This allows an update of requesting device's page table 649 * with entries associated with DOOREBELL or MMIO memory 650 * 651 * This method is invoked in the following contexts: 652 * - Mapping of DOORBELL or MMIO BO of same or peer device 653 * - Validating an evicted DOOREBELL or MMIO BO on device seeking access 654 * 655 * Return: ZERO if successful, NON-ZERO otherwise 656 */ 657 static int 658 kfd_mem_dmamap_sg_bo(struct kgd_mem *mem, 659 struct kfd_mem_attachment *attachment) 660 { 661 struct ttm_operation_ctx ctx = {.interruptible = true}; 662 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 663 struct amdgpu_device *adev = attachment->adev; 664 struct ttm_tt *ttm = bo->tbo.ttm; 665 enum dma_data_direction dir; 666 dma_addr_t dma_addr; 667 bool mmio; 668 int ret; 669 670 /* Expect SG Table of dmapmap BO to be NULL */ 671 mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP); 672 if (unlikely(ttm->sg)) { 673 pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio); 674 return -EINVAL; 675 } 676 677 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 678 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 679 dma_addr = mem->bo->tbo.sg->sgl->dma_address; 680 pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length); 681 pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr); 682 dma_addr = dma_map_resource(adev->dev, dma_addr, 683 mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); 684 ret = dma_mapping_error(adev->dev, dma_addr); 685 if (unlikely(ret)) 686 return ret; 687 pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr); 688 689 ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length); 690 if (unlikely(!ttm->sg)) { 691 ret = -ENOMEM; 692 goto unmap_sg; 693 } 694 695 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 696 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 697 if (unlikely(ret)) 698 goto free_sg; 699 700 return ret; 701 702 free_sg: 703 sg_free_table(ttm->sg); 704 kfree(ttm->sg); 705 ttm->sg = NULL; 706 unmap_sg: 707 dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length, 708 dir, DMA_ATTR_SKIP_CPU_SYNC); 709 return ret; 710 } 711 712 static int 713 kfd_mem_dmamap_attachment(struct kgd_mem *mem, 714 struct kfd_mem_attachment *attachment) 715 { 716 switch (attachment->type) { 717 case KFD_MEM_ATT_SHARED: 718 return 0; 719 case KFD_MEM_ATT_USERPTR: 720 return kfd_mem_dmamap_userptr(mem, attachment); 721 case KFD_MEM_ATT_DMABUF: 722 return kfd_mem_dmamap_dmabuf(attachment); 723 case KFD_MEM_ATT_SG: 724 return kfd_mem_dmamap_sg_bo(mem, attachment); 725 default: 726 WARN_ON_ONCE(1); 727 } 728 return -EINVAL; 729 } 730 731 static void 732 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem, 733 struct kfd_mem_attachment *attachment) 734 { 735 enum dma_data_direction direction = 736 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 737 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 738 struct ttm_operation_ctx ctx = {.interruptible = false}; 739 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 740 struct amdgpu_device *adev = attachment->adev; 741 struct ttm_tt *ttm = bo->tbo.ttm; 742 743 if (unlikely(!ttm->sg)) 744 return; 745 746 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 747 (void)ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 748 749 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 750 sg_free_table(ttm->sg); 751 kfree(ttm->sg); 752 ttm->sg = NULL; 753 } 754 755 static void 756 kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment) 757 { 758 /* This is a no-op. We don't want to trigger eviction fences when 759 * unmapping DMABufs. Therefore the invalidation (moving to system 760 * domain) is done in kfd_mem_dmamap_dmabuf. 761 */ 762 } 763 764 /** 765 * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO 766 * @mem: SG BO of the DOORBELL or MMIO resource on the owning device 767 * @attachment: Virtual address attachment of the BO on accessing device 768 * 769 * The method performs following steps: 770 * - Signal TTM to mark memory pointed to by BO as GPU inaccessible 771 * - Free SG Table that is used to encapsulate DMA mapped memory of 772 * peer device's DOORBELL or MMIO memory 773 * 774 * This method is invoked in the following contexts: 775 * UNMapping of DOORBELL or MMIO BO on a device having access to its memory 776 * Eviction of DOOREBELL or MMIO BO on device having access to its memory 777 * 778 * Return: void 779 */ 780 static void 781 kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem, 782 struct kfd_mem_attachment *attachment) 783 { 784 struct ttm_operation_ctx ctx = {.interruptible = true}; 785 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 786 struct amdgpu_device *adev = attachment->adev; 787 struct ttm_tt *ttm = bo->tbo.ttm; 788 enum dma_data_direction dir; 789 790 if (unlikely(!ttm->sg)) { 791 pr_debug("SG Table of BO is NULL"); 792 return; 793 } 794 795 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 796 (void)ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 797 798 dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 799 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 800 dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address, 801 ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC); 802 sg_free_table(ttm->sg); 803 kfree(ttm->sg); 804 ttm->sg = NULL; 805 bo->tbo.sg = NULL; 806 } 807 808 static void 809 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem, 810 struct kfd_mem_attachment *attachment) 811 { 812 switch (attachment->type) { 813 case KFD_MEM_ATT_SHARED: 814 break; 815 case KFD_MEM_ATT_USERPTR: 816 kfd_mem_dmaunmap_userptr(mem, attachment); 817 break; 818 case KFD_MEM_ATT_DMABUF: 819 kfd_mem_dmaunmap_dmabuf(attachment); 820 break; 821 case KFD_MEM_ATT_SG: 822 kfd_mem_dmaunmap_sg_bo(mem, attachment); 823 break; 824 default: 825 WARN_ON_ONCE(1); 826 } 827 } 828 829 static int kfd_mem_export_dmabuf(struct kgd_mem *mem) 830 { 831 if (!mem->dmabuf) { 832 struct amdgpu_device *bo_adev; 833 struct dma_buf *dmabuf; 834 835 bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 836 dmabuf = drm_gem_prime_handle_to_dmabuf(&bo_adev->ddev, bo_adev->kfd.client.file, 837 mem->gem_handle, 838 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 839 DRM_RDWR : 0); 840 if (IS_ERR(dmabuf)) 841 return PTR_ERR(dmabuf); 842 mem->dmabuf = dmabuf; 843 } 844 845 return 0; 846 } 847 848 static int 849 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem, 850 struct amdgpu_bo **bo) 851 { 852 struct drm_gem_object *gobj; 853 int ret; 854 855 ret = kfd_mem_export_dmabuf(mem); 856 if (ret) 857 return ret; 858 859 gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf); 860 if (IS_ERR(gobj)) 861 return PTR_ERR(gobj); 862 863 *bo = gem_to_amdgpu_bo(gobj); 864 (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE; 865 866 return 0; 867 } 868 869 /* kfd_mem_attach - Add a BO to a VM 870 * 871 * Everything that needs to bo done only once when a BO is first added 872 * to a VM. It can later be mapped and unmapped many times without 873 * repeating these steps. 874 * 875 * 0. Create BO for DMA mapping, if needed 876 * 1. Allocate and initialize BO VA entry data structure 877 * 2. Add BO to the VM 878 * 3. Determine ASIC-specific PTE flags 879 * 4. Alloc page tables and directories if needed 880 * 4a. Validate new page tables and directories 881 */ 882 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem, 883 struct amdgpu_vm *vm, bool is_aql) 884 { 885 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 886 unsigned long bo_size = mem->bo->tbo.base.size; 887 uint64_t va = mem->va; 888 struct kfd_mem_attachment *attachment[2] = {NULL, NULL}; 889 struct amdgpu_bo *bo[2] = {NULL, NULL}; 890 struct amdgpu_bo_va *bo_va; 891 bool same_hive = false; 892 struct drm_exec exec; 893 int i, ret; 894 895 if (!va) { 896 pr_err("Invalid VA when adding BO to VM\n"); 897 return -EINVAL; 898 } 899 900 /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices 901 * 902 * The access path of MMIO and DOORBELL BOs of is always over PCIe. 903 * In contrast the access path of VRAM BOs depens upon the type of 904 * link that connects the peer device. Access over PCIe is allowed 905 * if peer device has large BAR. In contrast, access over xGMI is 906 * allowed for both small and large BAR configurations of peer device 907 */ 908 if ((adev != bo_adev && !adev->apu_prefer_gtt) && 909 ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) || 910 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) || 911 (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { 912 if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM) 913 same_hive = amdgpu_xgmi_same_hive(adev, bo_adev); 914 if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev)) 915 return -EINVAL; 916 } 917 918 for (i = 0; i <= is_aql; i++) { 919 attachment[i] = kzalloc_obj(*attachment[i]); 920 if (unlikely(!attachment[i])) { 921 ret = -ENOMEM; 922 goto unwind; 923 } 924 925 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, 926 va + bo_size, vm); 927 928 if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) || 929 (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && reuse_dmamap(adev, bo_adev)) || 930 (mem->domain == AMDGPU_GEM_DOMAIN_GTT && reuse_dmamap(adev, bo_adev)) || 931 same_hive) { 932 /* Mappings on the local GPU, or VRAM mappings in the 933 * local hive, or userptr, or GTT mapping can reuse dma map 934 * address space share the original BO 935 */ 936 attachment[i]->type = KFD_MEM_ATT_SHARED; 937 bo[i] = mem->bo; 938 drm_gem_object_get(&bo[i]->tbo.base); 939 } else if (i > 0) { 940 /* Multiple mappings on the same GPU share the BO */ 941 attachment[i]->type = KFD_MEM_ATT_SHARED; 942 bo[i] = bo[0]; 943 drm_gem_object_get(&bo[i]->tbo.base); 944 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { 945 /* Create an SG BO to DMA-map userptrs on other GPUs */ 946 attachment[i]->type = KFD_MEM_ATT_USERPTR; 947 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); 948 if (ret) 949 goto unwind; 950 /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */ 951 } else if (mem->bo->tbo.type == ttm_bo_type_sg) { 952 WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL || 953 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP), 954 "Handing invalid SG BO in ATTACH request"); 955 attachment[i]->type = KFD_MEM_ATT_SG; 956 ret = create_dmamap_sg_bo(adev, mem, &bo[i]); 957 if (ret) 958 goto unwind; 959 /* Enable acces to GTT and VRAM BOs of peer devices */ 960 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT || 961 mem->domain == AMDGPU_GEM_DOMAIN_VRAM) { 962 attachment[i]->type = KFD_MEM_ATT_DMABUF; 963 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]); 964 if (ret) 965 goto unwind; 966 pr_debug("Employ DMABUF mechanism to enable peer GPU access\n"); 967 } else { 968 WARN_ONCE(true, "Handling invalid ATTACH request"); 969 ret = -EINVAL; 970 goto unwind; 971 } 972 973 drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 974 drm_exec_until_all_locked(&exec) { 975 ret = amdgpu_vm_lock_pd(vm, &exec, 0); 976 drm_exec_retry_on_contention(&exec); 977 if (unlikely(ret)) 978 goto unwind; 979 ret = drm_exec_lock_obj(&exec, &bo[i]->tbo.base); 980 drm_exec_retry_on_contention(&exec); 981 if (unlikely(ret)) 982 goto unwind; 983 } 984 985 bo_va = amdgpu_vm_bo_find(vm, bo[i]); 986 if (!bo_va) 987 bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]); 988 else 989 ++bo_va->ref_count; 990 attachment[i]->bo_va = bo_va; 991 drm_exec_fini(&exec); 992 if (unlikely(!attachment[i]->bo_va)) { 993 ret = -ENOMEM; 994 pr_err("Failed to add BO object to VM. ret == %d\n", 995 ret); 996 goto unwind; 997 } 998 attachment[i]->va = va; 999 attachment[i]->pte_flags = get_pte_flags(adev, vm, mem); 1000 attachment[i]->adev = adev; 1001 list_add(&attachment[i]->list, &mem->attachments); 1002 1003 va += bo_size; 1004 } 1005 1006 return 0; 1007 1008 unwind: 1009 for (; i >= 0; i--) { 1010 if (!attachment[i]) 1011 continue; 1012 if (attachment[i]->bo_va) { 1013 (void)amdgpu_bo_reserve(bo[i], true); 1014 if (--attachment[i]->bo_va->ref_count == 0) 1015 amdgpu_vm_bo_del(adev, attachment[i]->bo_va); 1016 amdgpu_bo_unreserve(bo[i]); 1017 list_del(&attachment[i]->list); 1018 } 1019 if (bo[i]) 1020 drm_gem_object_put(&bo[i]->tbo.base); 1021 kfree(attachment[i]); 1022 } 1023 return ret; 1024 } 1025 1026 static void kfd_mem_detach(struct kfd_mem_attachment *attachment) 1027 { 1028 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 1029 1030 pr_debug("\t remove VA 0x%llx in entry %p\n", 1031 attachment->va, attachment); 1032 if (--attachment->bo_va->ref_count == 0) 1033 amdgpu_vm_bo_del(attachment->adev, attachment->bo_va); 1034 drm_gem_object_put(&bo->tbo.base); 1035 list_del(&attachment->list); 1036 kfree(attachment); 1037 } 1038 1039 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, 1040 struct amdkfd_process_info *process_info, 1041 bool userptr) 1042 { 1043 mutex_lock(&process_info->lock); 1044 if (userptr) 1045 list_add_tail(&mem->validate_list, 1046 &process_info->userptr_valid_list); 1047 else 1048 list_add_tail(&mem->validate_list, &process_info->kfd_bo_list); 1049 mutex_unlock(&process_info->lock); 1050 } 1051 1052 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem, 1053 struct amdkfd_process_info *process_info) 1054 { 1055 mutex_lock(&process_info->lock); 1056 list_del(&mem->validate_list); 1057 mutex_unlock(&process_info->lock); 1058 } 1059 1060 /* Initializes user pages. It registers the MMU notifier and validates 1061 * the userptr BO in the GTT domain. 1062 * 1063 * The BO must already be on the userptr_valid_list. Otherwise an 1064 * eviction and restore may happen that leaves the new BO unmapped 1065 * with the user mode queues running. 1066 * 1067 * Takes the process_info->lock to protect against concurrent restore 1068 * workers. 1069 * 1070 * Returns 0 for success, negative errno for errors. 1071 */ 1072 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr, 1073 bool criu_resume) 1074 { 1075 struct amdkfd_process_info *process_info = mem->process_info; 1076 struct amdgpu_bo *bo = mem->bo; 1077 struct ttm_operation_ctx ctx = { true, false }; 1078 struct amdgpu_hmm_range *range; 1079 int ret = 0; 1080 1081 mutex_lock(&process_info->lock); 1082 1083 ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0); 1084 if (ret) { 1085 pr_err("%s: Failed to set userptr: %d\n", __func__, ret); 1086 goto out; 1087 } 1088 1089 ret = amdgpu_hmm_register(bo, user_addr); 1090 if (ret) { 1091 pr_err("%s: Failed to register MMU notifier: %d\n", 1092 __func__, ret); 1093 goto out; 1094 } 1095 1096 if (criu_resume) { 1097 /* 1098 * During a CRIU restore operation, the userptr buffer objects 1099 * will be validated in the restore_userptr_work worker at a 1100 * later stage when it is scheduled by another ioctl called by 1101 * CRIU master process for the target pid for restore. 1102 */ 1103 mutex_lock(&process_info->notifier_lock); 1104 mem->invalid++; 1105 mutex_unlock(&process_info->notifier_lock); 1106 mutex_unlock(&process_info->lock); 1107 return 0; 1108 } 1109 1110 range = amdgpu_hmm_range_alloc(NULL); 1111 if (unlikely(!range)) { 1112 ret = -ENOMEM; 1113 goto unregister_out; 1114 } 1115 1116 ret = amdgpu_ttm_tt_get_user_pages(bo, range); 1117 if (ret) { 1118 amdgpu_hmm_range_free(range); 1119 if (ret == -EAGAIN) 1120 pr_debug("Failed to get user pages, try again\n"); 1121 else 1122 pr_err("%s: Failed to get user pages: %d\n", __func__, ret); 1123 goto unregister_out; 1124 } 1125 1126 ret = amdgpu_bo_reserve(bo, true); 1127 if (ret) { 1128 pr_err("%s: Failed to reserve BO\n", __func__); 1129 goto release_out; 1130 } 1131 1132 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, range); 1133 1134 amdgpu_bo_placement_from_domain(bo, mem->domain); 1135 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1136 if (ret) 1137 pr_err("%s: failed to validate BO\n", __func__); 1138 amdgpu_bo_unreserve(bo); 1139 1140 release_out: 1141 amdgpu_hmm_range_free(range); 1142 unregister_out: 1143 if (ret) 1144 amdgpu_hmm_unregister(bo); 1145 out: 1146 mutex_unlock(&process_info->lock); 1147 return ret; 1148 } 1149 1150 /* Reserving a BO and its page table BOs must happen atomically to 1151 * avoid deadlocks. Some operations update multiple VMs at once. Track 1152 * all the reservation info in a context structure. Optionally a sync 1153 * object can track VM updates. 1154 */ 1155 struct bo_vm_reservation_context { 1156 /* DRM execution context for the reservation */ 1157 struct drm_exec exec; 1158 /* Number of VMs reserved */ 1159 unsigned int n_vms; 1160 /* Pointer to sync object */ 1161 struct amdgpu_sync *sync; 1162 }; 1163 1164 enum bo_vm_match { 1165 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */ 1166 BO_VM_MAPPED, /* Match VMs where a BO is mapped */ 1167 BO_VM_ALL, /* Match all VMs a BO was added to */ 1168 }; 1169 1170 /** 1171 * reserve_bo_and_vm - reserve a BO and a VM unconditionally. 1172 * @mem: KFD BO structure. 1173 * @vm: the VM to reserve. 1174 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 1175 */ 1176 static int reserve_bo_and_vm(struct kgd_mem *mem, 1177 struct amdgpu_vm *vm, 1178 struct bo_vm_reservation_context *ctx) 1179 { 1180 struct amdgpu_bo *bo = mem->bo; 1181 int ret; 1182 1183 WARN_ON(!vm); 1184 1185 ctx->n_vms = 1; 1186 ctx->sync = &mem->sync; 1187 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT, 0); 1188 drm_exec_until_all_locked(&ctx->exec) { 1189 ret = amdgpu_vm_lock_pd(vm, &ctx->exec, 2); 1190 drm_exec_retry_on_contention(&ctx->exec); 1191 if (unlikely(ret)) 1192 goto error; 1193 1194 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); 1195 drm_exec_retry_on_contention(&ctx->exec); 1196 if (unlikely(ret)) 1197 goto error; 1198 } 1199 return 0; 1200 1201 error: 1202 pr_err("Failed to reserve buffers in ttm.\n"); 1203 drm_exec_fini(&ctx->exec); 1204 return ret; 1205 } 1206 1207 /** 1208 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally 1209 * @mem: KFD BO structure. 1210 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO 1211 * is used. Otherwise, a single VM associated with the BO. 1212 * @map_type: the mapping status that will be used to filter the VMs. 1213 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 1214 * 1215 * Returns 0 for success, negative for failure. 1216 */ 1217 static int reserve_bo_and_cond_vms(struct kgd_mem *mem, 1218 struct amdgpu_vm *vm, enum bo_vm_match map_type, 1219 struct bo_vm_reservation_context *ctx) 1220 { 1221 struct kfd_mem_attachment *entry; 1222 struct amdgpu_bo *bo = mem->bo; 1223 int ret; 1224 1225 ctx->sync = &mem->sync; 1226 drm_exec_init(&ctx->exec, DRM_EXEC_INTERRUPTIBLE_WAIT | 1227 DRM_EXEC_IGNORE_DUPLICATES, 0); 1228 drm_exec_until_all_locked(&ctx->exec) { 1229 ctx->n_vms = 0; 1230 list_for_each_entry(entry, &mem->attachments, list) { 1231 if ((vm && vm != entry->bo_va->base.vm) || 1232 (entry->is_mapped != map_type 1233 && map_type != BO_VM_ALL)) 1234 continue; 1235 1236 ret = amdgpu_vm_lock_pd(entry->bo_va->base.vm, 1237 &ctx->exec, 2); 1238 drm_exec_retry_on_contention(&ctx->exec); 1239 if (unlikely(ret)) 1240 goto error; 1241 ++ctx->n_vms; 1242 } 1243 1244 ret = drm_exec_prepare_obj(&ctx->exec, &bo->tbo.base, 1); 1245 drm_exec_retry_on_contention(&ctx->exec); 1246 if (unlikely(ret)) 1247 goto error; 1248 } 1249 return 0; 1250 1251 error: 1252 pr_err("Failed to reserve buffers in ttm.\n"); 1253 drm_exec_fini(&ctx->exec); 1254 return ret; 1255 } 1256 1257 /** 1258 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context 1259 * @ctx: Reservation context to unreserve 1260 * @wait: Optionally wait for a sync object representing pending VM updates 1261 * @intr: Whether the wait is interruptible 1262 * 1263 * Also frees any resources allocated in 1264 * reserve_bo_and_(cond_)vm(s). Returns the status from 1265 * amdgpu_sync_wait. 1266 */ 1267 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx, 1268 bool wait, bool intr) 1269 { 1270 int ret = 0; 1271 1272 if (wait) 1273 ret = amdgpu_sync_wait(ctx->sync, intr); 1274 1275 drm_exec_fini(&ctx->exec); 1276 ctx->sync = NULL; 1277 return ret; 1278 } 1279 1280 static int unmap_bo_from_gpuvm(struct kgd_mem *mem, 1281 struct kfd_mem_attachment *entry, 1282 struct amdgpu_sync *sync) 1283 { 1284 struct amdgpu_bo_va *bo_va = entry->bo_va; 1285 struct amdgpu_device *adev = entry->adev; 1286 struct amdgpu_vm *vm = bo_va->base.vm; 1287 1288 if (bo_va->queue_refcount) { 1289 pr_debug("bo_va->queue_refcount %d\n", bo_va->queue_refcount); 1290 return -EBUSY; 1291 } 1292 1293 (void)amdgpu_vm_bo_unmap(adev, bo_va, entry->va); 1294 1295 /* VM entity stopped if process killed, don't clear freed pt bo */ 1296 if (!amdgpu_vm_ready(vm)) 1297 return 0; 1298 1299 (void)amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); 1300 1301 (void)amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL); 1302 1303 return 0; 1304 } 1305 1306 static int update_gpuvm_pte(struct kgd_mem *mem, 1307 struct kfd_mem_attachment *entry, 1308 struct amdgpu_sync *sync) 1309 { 1310 struct amdgpu_bo_va *bo_va = entry->bo_va; 1311 struct amdgpu_device *adev = entry->adev; 1312 int ret; 1313 1314 ret = kfd_mem_dmamap_attachment(mem, entry); 1315 if (ret) 1316 return ret; 1317 1318 /* Update the page tables */ 1319 ret = amdgpu_vm_bo_update(adev, bo_va, false); 1320 if (ret) { 1321 pr_err("amdgpu_vm_bo_update failed\n"); 1322 return ret; 1323 } 1324 1325 return amdgpu_sync_fence(sync, bo_va->last_pt_update, GFP_KERNEL); 1326 } 1327 1328 static int map_bo_to_gpuvm(struct kgd_mem *mem, 1329 struct kfd_mem_attachment *entry, 1330 struct amdgpu_sync *sync, 1331 bool no_update_pte) 1332 { 1333 int ret; 1334 1335 /* Set virtual address for the allocation */ 1336 ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0, 1337 amdgpu_bo_size(entry->bo_va->base.bo), 1338 entry->pte_flags); 1339 if (ret) { 1340 pr_err("Failed to map VA 0x%llx in vm. ret %d\n", 1341 entry->va, ret); 1342 return ret; 1343 } 1344 1345 if (no_update_pte) 1346 return 0; 1347 1348 ret = update_gpuvm_pte(mem, entry, sync); 1349 if (ret) { 1350 pr_err("update_gpuvm_pte() failed\n"); 1351 goto update_gpuvm_pte_failed; 1352 } 1353 1354 return 0; 1355 1356 update_gpuvm_pte_failed: 1357 unmap_bo_from_gpuvm(mem, entry, sync); 1358 kfd_mem_dmaunmap_attachment(mem, entry); 1359 return ret; 1360 } 1361 1362 static int process_validate_vms(struct amdkfd_process_info *process_info, 1363 struct ww_acquire_ctx *ticket) 1364 { 1365 struct amdgpu_vm *peer_vm; 1366 int ret; 1367 1368 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1369 vm_list_node) { 1370 ret = vm_validate_pt_pd_bos(peer_vm, ticket); 1371 if (ret) 1372 return ret; 1373 } 1374 1375 return 0; 1376 } 1377 1378 static int process_sync_pds_resv(struct amdkfd_process_info *process_info, 1379 struct amdgpu_sync *sync) 1380 { 1381 struct amdgpu_vm *peer_vm; 1382 int ret; 1383 1384 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1385 vm_list_node) { 1386 struct amdgpu_bo *pd = peer_vm->root.bo; 1387 1388 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv, 1389 AMDGPU_SYNC_NE_OWNER, 1390 AMDGPU_FENCE_OWNER_KFD); 1391 if (ret) 1392 return ret; 1393 } 1394 1395 return 0; 1396 } 1397 1398 static int process_update_pds(struct amdkfd_process_info *process_info, 1399 struct amdgpu_sync *sync) 1400 { 1401 struct amdgpu_vm *peer_vm; 1402 int ret; 1403 1404 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1405 vm_list_node) { 1406 ret = vm_update_pds(peer_vm, sync); 1407 if (ret) 1408 return ret; 1409 } 1410 1411 return 0; 1412 } 1413 1414 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, 1415 struct dma_fence **ef) 1416 { 1417 struct amdkfd_process_info *info = NULL; 1418 struct kfd_process *process = NULL; 1419 int ret; 1420 1421 process = container_of(process_info, struct kfd_process, kgd_process_info); 1422 if (!*process_info) { 1423 info = kzalloc_obj(*info); 1424 if (!info) 1425 return -ENOMEM; 1426 1427 mutex_init(&info->lock); 1428 mutex_init(&info->notifier_lock); 1429 INIT_LIST_HEAD(&info->vm_list_head); 1430 INIT_LIST_HEAD(&info->kfd_bo_list); 1431 INIT_LIST_HEAD(&info->userptr_valid_list); 1432 INIT_LIST_HEAD(&info->userptr_inval_list); 1433 1434 info->eviction_fence = 1435 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), 1436 current->mm, 1437 process->context_id); 1438 if (!info->eviction_fence) { 1439 pr_err("Failed to create eviction fence\n"); 1440 ret = -ENOMEM; 1441 goto create_evict_fence_fail; 1442 } 1443 1444 info->pid = get_task_pid(current, PIDTYPE_TGID); 1445 INIT_DELAYED_WORK(&info->restore_userptr_work, 1446 amdgpu_amdkfd_restore_userptr_worker); 1447 1448 info->context_id = process->context_id; 1449 1450 *process_info = info; 1451 } 1452 1453 if (cmpxchg(&vm->process_info, NULL, *process_info) != NULL) { 1454 ret = -EINVAL; 1455 goto already_acquired; 1456 } 1457 1458 /* Validate page directory and attach eviction fence */ 1459 ret = amdgpu_bo_reserve(vm->root.bo, true); 1460 if (ret) 1461 goto reserve_pd_fail; 1462 ret = vm_validate_pt_pd_bos(vm, NULL); 1463 if (ret) { 1464 pr_err("validate_pt_pd_bos() failed\n"); 1465 goto validate_pd_fail; 1466 } 1467 ret = amdgpu_bo_sync_wait(vm->root.bo, 1468 AMDGPU_FENCE_OWNER_KFD, false); 1469 if (ret) 1470 goto wait_pd_fail; 1471 ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1); 1472 if (ret) 1473 goto reserve_shared_fail; 1474 dma_resv_add_fence(vm->root.bo->tbo.base.resv, 1475 &vm->process_info->eviction_fence->base, 1476 DMA_RESV_USAGE_BOOKKEEP); 1477 amdgpu_bo_unreserve(vm->root.bo); 1478 1479 /* Update process info */ 1480 mutex_lock(&vm->process_info->lock); 1481 list_add_tail(&vm->vm_list_node, 1482 &(vm->process_info->vm_list_head)); 1483 vm->process_info->n_vms++; 1484 if (ef) 1485 *ef = dma_fence_get(&vm->process_info->eviction_fence->base); 1486 mutex_unlock(&vm->process_info->lock); 1487 1488 return 0; 1489 1490 reserve_shared_fail: 1491 wait_pd_fail: 1492 validate_pd_fail: 1493 amdgpu_bo_unreserve(vm->root.bo); 1494 reserve_pd_fail: 1495 vm->process_info = NULL; 1496 already_acquired: 1497 if (info) { 1498 dma_fence_put(&info->eviction_fence->base); 1499 *process_info = NULL; 1500 put_pid(info->pid); 1501 create_evict_fence_fail: 1502 mutex_destroy(&info->lock); 1503 mutex_destroy(&info->notifier_lock); 1504 kfree(info); 1505 } 1506 return ret; 1507 } 1508 1509 /** 1510 * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria 1511 * @bo: Handle of buffer object being pinned 1512 * @domain: Domain into which BO should be pinned 1513 * 1514 * - USERPTR BOs are UNPINNABLE and will return error 1515 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1516 * PIN count incremented. It is valid to PIN a BO multiple times 1517 * 1518 * Return: ZERO if successful in pinning, Non-Zero in case of error. 1519 */ 1520 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) 1521 { 1522 int ret = 0; 1523 1524 ret = amdgpu_bo_reserve(bo, false); 1525 if (unlikely(ret)) 1526 return ret; 1527 1528 if (bo->flags & AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS) { 1529 /* 1530 * If bo is not contiguous on VRAM, move to system memory first to ensure 1531 * we can get contiguous VRAM space after evicting other BOs. 1532 */ 1533 if (!(bo->tbo.resource->placement & TTM_PL_FLAG_CONTIGUOUS)) { 1534 struct ttm_operation_ctx ctx = { true, false }; 1535 1536 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 1537 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 1538 if (unlikely(ret)) { 1539 pr_debug("validate bo 0x%p to GTT failed %d\n", &bo->tbo, ret); 1540 goto out; 1541 } 1542 } 1543 } 1544 1545 ret = amdgpu_bo_pin(bo, domain); 1546 if (ret) 1547 pr_err("Error in Pinning BO to domain: %d\n", domain); 1548 1549 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 1550 1551 out: 1552 amdgpu_bo_unreserve(bo); 1553 return ret; 1554 } 1555 1556 /** 1557 * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria 1558 * @bo: Handle of buffer object being unpinned 1559 * 1560 * - Is a illegal request for USERPTR BOs and is ignored 1561 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1562 * PIN count decremented. Calls to UNPIN must balance calls to PIN 1563 */ 1564 static void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo) 1565 { 1566 int ret = 0; 1567 1568 ret = amdgpu_bo_reserve(bo, false); 1569 if (unlikely(ret)) 1570 return; 1571 1572 amdgpu_bo_unpin(bo); 1573 1574 amdgpu_bo_unreserve(bo); 1575 } 1576 1577 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev, 1578 struct amdgpu_vm *avm, 1579 void **process_info, 1580 struct dma_fence **ef) 1581 { 1582 int ret; 1583 1584 /* Already a compute VM? */ 1585 if (avm->process_info) 1586 return -EINVAL; 1587 1588 /* Convert VM into a compute VM */ 1589 ret = amdgpu_vm_make_compute(adev, avm); 1590 if (ret) 1591 return ret; 1592 1593 /* Initialize KFD part of the VM and process info */ 1594 ret = init_kfd_vm(avm, process_info, ef); 1595 if (ret) 1596 return ret; 1597 1598 amdgpu_vm_set_task_info(avm); 1599 1600 return 0; 1601 } 1602 1603 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, 1604 struct amdgpu_vm *vm) 1605 { 1606 struct amdkfd_process_info *process_info = vm->process_info; 1607 1608 if (!process_info) 1609 return; 1610 1611 /* Update process info */ 1612 mutex_lock(&process_info->lock); 1613 process_info->n_vms--; 1614 list_del(&vm->vm_list_node); 1615 mutex_unlock(&process_info->lock); 1616 1617 vm->process_info = NULL; 1618 1619 /* Release per-process resources when last compute VM is destroyed */ 1620 if (!process_info->n_vms) { 1621 WARN_ON(!list_empty(&process_info->kfd_bo_list)); 1622 WARN_ON(!list_empty(&process_info->userptr_valid_list)); 1623 WARN_ON(!list_empty(&process_info->userptr_inval_list)); 1624 1625 dma_fence_put(&process_info->eviction_fence->base); 1626 cancel_delayed_work_sync(&process_info->restore_userptr_work); 1627 put_pid(process_info->pid); 1628 mutex_destroy(&process_info->lock); 1629 mutex_destroy(&process_info->notifier_lock); 1630 kfree(process_info); 1631 } 1632 } 1633 1634 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv) 1635 { 1636 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1637 struct amdgpu_bo *pd = avm->root.bo; 1638 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 1639 1640 if (adev->asic_type < CHIP_VEGA10) 1641 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT; 1642 return avm->pd_phys_addr; 1643 } 1644 1645 void amdgpu_amdkfd_block_mmu_notifications(void *p) 1646 { 1647 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; 1648 1649 mutex_lock(&pinfo->lock); 1650 WRITE_ONCE(pinfo->block_mmu_notifications, true); 1651 mutex_unlock(&pinfo->lock); 1652 } 1653 1654 int amdgpu_amdkfd_criu_resume(void *p) 1655 { 1656 int ret = 0; 1657 struct amdkfd_process_info *pinfo = (struct amdkfd_process_info *)p; 1658 1659 mutex_lock(&pinfo->lock); 1660 pr_debug("scheduling work\n"); 1661 mutex_lock(&pinfo->notifier_lock); 1662 pinfo->evicted_bos++; 1663 mutex_unlock(&pinfo->notifier_lock); 1664 if (!READ_ONCE(pinfo->block_mmu_notifications)) { 1665 ret = -EINVAL; 1666 goto out_unlock; 1667 } 1668 WRITE_ONCE(pinfo->block_mmu_notifications, false); 1669 queue_delayed_work(system_freezable_wq, 1670 &pinfo->restore_userptr_work, 0); 1671 1672 out_unlock: 1673 mutex_unlock(&pinfo->lock); 1674 return ret; 1675 } 1676 1677 /* Caller must hold kfd_mem_limit.mem_limit_lock. */ 1678 static size_t __get_available_memory(struct amdgpu_device *adev, 1679 uint8_t xcp_id, int64_t vram_used) 1680 { 1681 uint64_t reserved_for_pt = 1682 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); 1683 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 1684 uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0); 1685 ssize_t available; 1686 uint64_t vram_available, system_mem_available, ttm_mem_available; 1687 1688 lockdep_assert_held(&kfd_mem_limit.mem_limit_lock); 1689 1690 if (adev->apu_prefer_gtt && !adev->gmc.is_app_apu) 1691 vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id) 1692 - vram_used; 1693 else 1694 vram_available = KFD_XCP_MEMORY_SIZE(adev, xcp_id) 1695 - vram_used 1696 - atomic64_read(&adev->vram_pin_size_xcp[xcp_id]) 1697 - atomic64_read(&adev->vram_pin_size_common) 1698 + atomic64_read(&adev->kfd.vram_pinned[xcp_id]) 1699 - reserved_for_pt 1700 - reserved_for_ras; 1701 1702 if (adev->apu_prefer_gtt) { 1703 system_mem_available = no_system_mem_limit ? 1704 kfd_mem_limit.max_system_mem_limit : 1705 kfd_mem_limit.max_system_mem_limit - 1706 kfd_mem_limit.system_mem_used; 1707 1708 ttm_mem_available = kfd_mem_limit.max_ttm_mem_limit - 1709 kfd_mem_limit.ttm_mem_used; 1710 1711 available = min3(system_mem_available, ttm_mem_available, 1712 vram_available); 1713 } else { 1714 available = vram_available; 1715 } 1716 1717 if (available < 0) 1718 available = 0; 1719 1720 return available; 1721 } 1722 1723 size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev, 1724 uint8_t xcp_id) 1725 { 1726 size_t available; 1727 1728 spin_lock(&kfd_mem_limit.mem_limit_lock); 1729 available = __get_available_memory(adev, xcp_id, 1730 adev->kfd.vram_used_aligned[xcp_id]); 1731 spin_unlock(&kfd_mem_limit.mem_limit_lock); 1732 1733 return available; 1734 } 1735 1736 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( 1737 struct amdgpu_device *adev, uint64_t va, uint64_t size, 1738 void *drm_priv, struct kgd_mem **mem, 1739 uint64_t *offset, uint32_t flags, bool criu_resume) 1740 { 1741 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1742 struct amdgpu_fpriv *fpriv = container_of(avm, struct amdgpu_fpriv, vm); 1743 enum ttm_bo_type bo_type = ttm_bo_type_device; 1744 struct sg_table *sg = NULL; 1745 uint64_t user_addr = 0; 1746 struct amdgpu_bo *bo; 1747 struct drm_gem_object *gobj = NULL; 1748 u32 domain, alloc_domain; 1749 uint64_t aligned_size; 1750 int8_t xcp_id = -1; 1751 u64 alloc_flags; 1752 int ret; 1753 1754 /* 1755 * Check on which domain to allocate BO 1756 */ 1757 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 1758 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM; 1759 1760 if (adev->apu_prefer_gtt) { 1761 domain = AMDGPU_GEM_DOMAIN_GTT; 1762 alloc_domain = AMDGPU_GEM_DOMAIN_GTT; 1763 alloc_flags = 0; 1764 } else { 1765 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE | 1766 AMDGPU_GEM_CREATE_VRAM_CLEARED; 1767 alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ? 1768 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0; 1769 1770 /* For contiguous VRAM allocation */ 1771 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS) 1772 alloc_flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1773 } 1774 xcp_id = fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION ? 1775 0 : fpriv->xcp_id; 1776 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 1777 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT; 1778 alloc_flags = 0; 1779 } else { 1780 domain = AMDGPU_GEM_DOMAIN_GTT; 1781 alloc_domain = AMDGPU_GEM_DOMAIN_CPU; 1782 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE; 1783 1784 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 1785 if (!offset || !*offset) 1786 return -EINVAL; 1787 user_addr = untagged_addr(*offset); 1788 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1789 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1790 bo_type = ttm_bo_type_sg; 1791 if (size > UINT_MAX) 1792 return -EINVAL; 1793 sg = create_sg_table(*offset, size); 1794 if (!sg) 1795 return -ENOMEM; 1796 } else { 1797 return -EINVAL; 1798 } 1799 } 1800 1801 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT) 1802 alloc_flags |= AMDGPU_GEM_CREATE_COHERENT; 1803 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT) 1804 alloc_flags |= AMDGPU_GEM_CREATE_EXT_COHERENT; 1805 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED) 1806 alloc_flags |= AMDGPU_GEM_CREATE_UNCACHED; 1807 1808 *mem = kzalloc_obj(struct kgd_mem); 1809 if (!*mem) { 1810 ret = -ENOMEM; 1811 goto err; 1812 } 1813 INIT_LIST_HEAD(&(*mem)->attachments); 1814 mutex_init(&(*mem)->lock); 1815 (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM); 1816 1817 /* Workaround for AQL queue wraparound bug. Map the same 1818 * memory twice. That means we only actually allocate half 1819 * the memory. 1820 */ 1821 if ((*mem)->aql_queue) 1822 size >>= 1; 1823 aligned_size = PAGE_ALIGN(size); 1824 1825 /* reject AQL queue with size < 2 */ 1826 if (!aligned_size) { 1827 ret = -EINVAL; 1828 goto err_alignment_size; 1829 } 1830 1831 (*mem)->alloc_flags = flags; 1832 1833 amdgpu_sync_create(&(*mem)->sync); 1834 1835 ret = amdgpu_amdkfd_reserve_mem_limit(adev, aligned_size, flags, 1836 xcp_id); 1837 if (ret) { 1838 pr_debug("Insufficient memory\n"); 1839 goto err_reserve_limit; 1840 } 1841 1842 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s xcp_id %d\n", 1843 va, (*mem)->aql_queue ? size << 1 : size, 1844 domain_string(alloc_domain), xcp_id); 1845 1846 ret = amdgpu_gem_object_create(adev, aligned_size, 1, alloc_domain, alloc_flags, 1847 bo_type, NULL, &gobj, xcp_id + 1); 1848 if (ret) { 1849 pr_debug("Failed to create BO on domain %s. ret %d\n", 1850 domain_string(alloc_domain), ret); 1851 goto err_bo_create; 1852 } 1853 ret = drm_vma_node_allow(&gobj->vma_node, drm_priv); 1854 if (ret) { 1855 pr_debug("Failed to allow vma node access. ret %d\n", ret); 1856 goto err_node_allow; 1857 } 1858 ret = drm_gem_handle_create(adev->kfd.client.file, gobj, &(*mem)->gem_handle); 1859 if (ret) 1860 goto err_gem_handle_create; 1861 bo = gem_to_amdgpu_bo(gobj); 1862 if (bo_type == ttm_bo_type_sg) { 1863 bo->tbo.sg = sg; 1864 bo->tbo.ttm->sg = sg; 1865 } 1866 bo->kfd_bo = *mem; 1867 (*mem)->bo = bo; 1868 if (user_addr) 1869 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO; 1870 1871 (*mem)->va = va; 1872 (*mem)->domain = domain; 1873 (*mem)->mapped_to_gpu_memory = 0; 1874 (*mem)->process_info = avm->process_info; 1875 1876 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr); 1877 1878 if (user_addr) { 1879 pr_debug("creating userptr BO for user_addr = %llx\n", user_addr); 1880 ret = init_user_pages(*mem, user_addr, criu_resume); 1881 if (ret) 1882 goto allocate_init_user_pages_failed; 1883 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1884 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1885 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT); 1886 if (ret) { 1887 pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n"); 1888 goto err_pin_bo; 1889 } 1890 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 1891 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 1892 } else { 1893 mutex_lock(&avm->process_info->lock); 1894 if (avm->process_info->eviction_fence && 1895 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base)) 1896 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, domain, 1897 &avm->process_info->eviction_fence->base); 1898 mutex_unlock(&avm->process_info->lock); 1899 if (ret) 1900 goto err_validate_bo; 1901 } 1902 1903 if (offset) 1904 *offset = amdgpu_bo_mmap_offset(bo); 1905 1906 return 0; 1907 1908 allocate_init_user_pages_failed: 1909 err_pin_bo: 1910 err_validate_bo: 1911 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); 1912 drm_gem_handle_delete(adev->kfd.client.file, (*mem)->gem_handle); 1913 err_gem_handle_create: 1914 drm_vma_node_revoke(&gobj->vma_node, drm_priv); 1915 err_node_allow: 1916 /* Don't unreserve system mem limit twice */ 1917 goto err_reserve_limit; 1918 err_bo_create: 1919 amdgpu_amdkfd_unreserve_mem_limit(adev, aligned_size, flags, xcp_id); 1920 err_reserve_limit: 1921 amdgpu_sync_free(&(*mem)->sync); 1922 err_alignment_size: 1923 mutex_destroy(&(*mem)->lock); 1924 if (gobj) 1925 drm_gem_object_put(gobj); 1926 else 1927 kfree(*mem); 1928 err: 1929 if (sg) { 1930 sg_free_table(sg); 1931 kfree(sg); 1932 } 1933 return ret; 1934 } 1935 1936 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( 1937 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv, 1938 uint64_t *size) 1939 { 1940 struct amdkfd_process_info *process_info = mem->process_info; 1941 unsigned long bo_size = mem->bo->tbo.base.size; 1942 bool use_release_notifier = (mem->bo->kfd_bo == mem); 1943 struct kfd_mem_attachment *entry, *tmp; 1944 struct bo_vm_reservation_context ctx; 1945 unsigned int mapped_to_gpu_memory; 1946 int ret; 1947 bool is_imported = false; 1948 1949 mutex_lock(&mem->lock); 1950 1951 mapped_to_gpu_memory = mem->mapped_to_gpu_memory; 1952 is_imported = mem->is_imported; 1953 mutex_unlock(&mem->lock); 1954 /* lock is not needed after this, since mem is unused and will 1955 * be freed anyway 1956 */ 1957 1958 if (mapped_to_gpu_memory > 0) { 1959 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n", 1960 mem->va, bo_size); 1961 return -EBUSY; 1962 } 1963 1964 /* At this point the BO is guaranteed to be freed, so unpin the 1965 * MMIO/DOORBELL BOs that were pinned during allocation. 1966 */ 1967 if (mem->alloc_flags & 1968 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1969 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1970 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); 1971 } 1972 1973 /* Make sure restore workers don't access the BO any more */ 1974 mutex_lock(&process_info->lock); 1975 if (!list_empty(&mem->validate_list)) 1976 list_del_init(&mem->validate_list); 1977 mutex_unlock(&process_info->lock); 1978 1979 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); 1980 if (unlikely(ret)) 1981 return ret; 1982 1983 /* Cleanup user pages and MMU notifiers */ 1984 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 1985 amdgpu_hmm_unregister(mem->bo); 1986 amdgpu_hmm_range_free(mem->range); 1987 mem->range = NULL; 1988 } 1989 1990 amdgpu_amdkfd_remove_eviction_fence(mem->bo, 1991 process_info->eviction_fence); 1992 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, 1993 mem->va + bo_size * (1 + mem->aql_queue)); 1994 1995 /* Remove from VM internal data structures */ 1996 list_for_each_entry_safe(entry, tmp, &mem->attachments, list) { 1997 kfd_mem_dmaunmap_attachment(mem, entry); 1998 kfd_mem_detach(entry); 1999 } 2000 2001 ret = unreserve_bo_and_vms(&ctx, false, false); 2002 2003 /* Free the sync object */ 2004 amdgpu_sync_free(&mem->sync); 2005 2006 /* If the SG is not NULL, it's one we created for a doorbell or mmio 2007 * remap BO. We need to free it. 2008 */ 2009 if (mem->bo->tbo.sg) { 2010 sg_free_table(mem->bo->tbo.sg); 2011 kfree(mem->bo->tbo.sg); 2012 } 2013 2014 /* Update the size of the BO being freed if it was allocated from 2015 * VRAM and is not imported. For APP APU VRAM allocations are done 2016 * in GTT domain 2017 */ 2018 if (size) { 2019 if (!is_imported && 2020 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) 2021 *size = bo_size; 2022 else 2023 *size = 0; 2024 } 2025 2026 /* Free the BO*/ 2027 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv); 2028 drm_gem_handle_delete(adev->kfd.client.file, mem->gem_handle); 2029 if (mem->dmabuf) { 2030 dma_buf_put(mem->dmabuf); 2031 mem->dmabuf = NULL; 2032 } 2033 mutex_destroy(&mem->lock); 2034 2035 /* If this releases the last reference, it will end up calling 2036 * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why 2037 * this needs to be the last call here. 2038 */ 2039 drm_gem_object_put(&mem->bo->tbo.base); 2040 2041 /* 2042 * For kgd_mem allocated in import_obj_create() via 2043 * amdgpu_amdkfd_gpuvm_import_dmabuf_fd(), 2044 * explicitly free it here. 2045 */ 2046 if (!use_release_notifier) 2047 kfree(mem); 2048 2049 return ret; 2050 } 2051 2052 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( 2053 struct amdgpu_device *adev, struct kgd_mem *mem, 2054 void *drm_priv) 2055 { 2056 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2057 int ret; 2058 struct amdgpu_bo *bo; 2059 uint32_t domain; 2060 struct kfd_mem_attachment *entry; 2061 struct bo_vm_reservation_context ctx; 2062 unsigned long bo_size; 2063 bool is_invalid_userptr = false; 2064 2065 bo = mem->bo; 2066 if (!bo) { 2067 pr_err("Invalid BO when mapping memory to GPU\n"); 2068 return -EINVAL; 2069 } 2070 2071 /* Make sure restore is not running concurrently. Since we 2072 * don't map invalid userptr BOs, we rely on the next restore 2073 * worker to do the mapping 2074 */ 2075 mutex_lock(&mem->process_info->lock); 2076 2077 /* Lock notifier lock. If we find an invalid userptr BO, we can be 2078 * sure that the MMU notifier is no longer running 2079 * concurrently and the queues are actually stopped 2080 */ 2081 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 2082 mutex_lock(&mem->process_info->notifier_lock); 2083 is_invalid_userptr = !!mem->invalid; 2084 mutex_unlock(&mem->process_info->notifier_lock); 2085 } 2086 2087 mutex_lock(&mem->lock); 2088 2089 domain = mem->domain; 2090 bo_size = bo->tbo.base.size; 2091 2092 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", 2093 mem->va, 2094 mem->va + bo_size * (1 + mem->aql_queue), 2095 avm, domain_string(domain)); 2096 2097 if (!kfd_mem_is_attached(avm, mem)) { 2098 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue); 2099 if (ret) 2100 goto out; 2101 } 2102 2103 ret = reserve_bo_and_vm(mem, avm, &ctx); 2104 if (unlikely(ret)) 2105 goto out; 2106 2107 /* Userptr can be marked as "not invalid", but not actually be 2108 * validated yet (still in the system domain). In that case 2109 * the queues are still stopped and we can leave mapping for 2110 * the next restore worker 2111 */ 2112 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && 2113 bo->tbo.resource->mem_type == TTM_PL_SYSTEM) 2114 is_invalid_userptr = true; 2115 2116 ret = vm_validate_pt_pd_bos(avm, NULL); 2117 if (unlikely(ret)) 2118 goto out_unreserve; 2119 2120 list_for_each_entry(entry, &mem->attachments, list) { 2121 if (entry->bo_va->base.vm != avm || entry->is_mapped) 2122 continue; 2123 2124 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n", 2125 entry->va, entry->va + bo_size, entry); 2126 2127 ret = map_bo_to_gpuvm(mem, entry, ctx.sync, 2128 is_invalid_userptr); 2129 if (ret) { 2130 pr_err("Failed to map bo to gpuvm\n"); 2131 goto out_unreserve; 2132 } 2133 2134 ret = vm_update_pds(avm, ctx.sync); 2135 if (ret) { 2136 pr_err("Failed to update page directories\n"); 2137 goto out_unreserve; 2138 } 2139 2140 entry->is_mapped = true; 2141 mem->mapped_to_gpu_memory++; 2142 pr_debug("\t INC mapping count %d\n", 2143 mem->mapped_to_gpu_memory); 2144 } 2145 2146 ret = unreserve_bo_and_vms(&ctx, false, false); 2147 2148 goto out; 2149 2150 out_unreserve: 2151 unreserve_bo_and_vms(&ctx, false, false); 2152 out: 2153 mutex_unlock(&mem->process_info->lock); 2154 mutex_unlock(&mem->lock); 2155 return ret; 2156 } 2157 2158 int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv) 2159 { 2160 struct kfd_mem_attachment *entry; 2161 struct amdgpu_vm *vm; 2162 int ret; 2163 2164 vm = drm_priv_to_vm(drm_priv); 2165 2166 mutex_lock(&mem->lock); 2167 2168 ret = amdgpu_bo_reserve(mem->bo, true); 2169 if (ret) 2170 goto out; 2171 2172 list_for_each_entry(entry, &mem->attachments, list) { 2173 if (entry->bo_va->base.vm != vm) 2174 continue; 2175 if (entry->bo_va->base.bo->tbo.ttm && 2176 !entry->bo_va->base.bo->tbo.ttm->sg) 2177 continue; 2178 2179 kfd_mem_dmaunmap_attachment(mem, entry); 2180 } 2181 2182 amdgpu_bo_unreserve(mem->bo); 2183 out: 2184 mutex_unlock(&mem->lock); 2185 2186 return ret; 2187 } 2188 2189 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( 2190 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv) 2191 { 2192 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2193 unsigned long bo_size = mem->bo->tbo.base.size; 2194 struct kfd_mem_attachment *entry; 2195 struct bo_vm_reservation_context ctx; 2196 int ret; 2197 2198 mutex_lock(&mem->lock); 2199 2200 ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx); 2201 if (unlikely(ret)) 2202 goto out; 2203 /* If no VMs were reserved, it means the BO wasn't actually mapped */ 2204 if (ctx.n_vms == 0) { 2205 ret = -EINVAL; 2206 goto unreserve_out; 2207 } 2208 2209 ret = vm_validate_pt_pd_bos(avm, NULL); 2210 if (unlikely(ret)) 2211 goto unreserve_out; 2212 2213 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n", 2214 mem->va, 2215 mem->va + bo_size * (1 + mem->aql_queue), 2216 avm); 2217 2218 list_for_each_entry(entry, &mem->attachments, list) { 2219 if (entry->bo_va->base.vm != avm || !entry->is_mapped) 2220 continue; 2221 2222 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n", 2223 entry->va, entry->va + bo_size, entry); 2224 2225 ret = unmap_bo_from_gpuvm(mem, entry, ctx.sync); 2226 if (ret) 2227 goto unreserve_out; 2228 2229 entry->is_mapped = false; 2230 2231 mem->mapped_to_gpu_memory--; 2232 pr_debug("\t DEC mapping count %d\n", 2233 mem->mapped_to_gpu_memory); 2234 } 2235 2236 unreserve_out: 2237 unreserve_bo_and_vms(&ctx, false, false); 2238 out: 2239 mutex_unlock(&mem->lock); 2240 return ret; 2241 } 2242 2243 int amdgpu_amdkfd_gpuvm_sync_memory( 2244 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr) 2245 { 2246 struct amdgpu_sync sync; 2247 int ret; 2248 2249 amdgpu_sync_create(&sync); 2250 2251 mutex_lock(&mem->lock); 2252 amdgpu_sync_clone(&mem->sync, &sync); 2253 mutex_unlock(&mem->lock); 2254 2255 ret = amdgpu_sync_wait(&sync, intr); 2256 amdgpu_sync_free(&sync); 2257 return ret; 2258 } 2259 2260 /** 2261 * amdgpu_amdkfd_map_gtt_bo_to_gart - Map BO to GART and increment reference count 2262 * @bo: Buffer object to be mapped 2263 * @bo_gart: Return bo reference 2264 * 2265 * Before return, bo reference count is incremented. To release the reference and unpin/ 2266 * unmap the BO, call amdgpu_amdkfd_free_kernel_mem. 2267 */ 2268 int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart) 2269 { 2270 int ret; 2271 2272 ret = amdgpu_bo_reserve(bo, true); 2273 if (ret) { 2274 pr_err("Failed to reserve bo. ret %d\n", ret); 2275 goto err_reserve_bo_failed; 2276 } 2277 2278 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 2279 if (ret) { 2280 pr_err("Failed to pin bo. ret %d\n", ret); 2281 goto err_pin_bo_failed; 2282 } 2283 2284 ret = amdgpu_ttm_alloc_gart(&bo->tbo); 2285 if (ret) { 2286 pr_err("Failed to bind bo to GART. ret %d\n", ret); 2287 goto err_map_bo_gart_failed; 2288 } 2289 2290 amdgpu_amdkfd_remove_eviction_fence( 2291 bo, bo->vm_bo->vm->process_info->eviction_fence); 2292 2293 amdgpu_bo_unreserve(bo); 2294 2295 *bo_gart = amdgpu_bo_ref(bo); 2296 2297 return 0; 2298 2299 err_map_bo_gart_failed: 2300 amdgpu_bo_unpin(bo); 2301 err_pin_bo_failed: 2302 amdgpu_bo_unreserve(bo); 2303 err_reserve_bo_failed: 2304 2305 return ret; 2306 } 2307 2308 /** amdgpu_amdkfd_gpuvm_map_bo_to_kernel() - Map GTT or VRAM BO for kernel CPU access 2309 * 2310 * @mem: Buffer object to be mapped for CPU access 2311 * @kptr[out]: pointer in kernel CPU address space 2312 * @size[out]: size of the buffer 2313 * @domain[IN]: domain for pinning (AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_DOMAIN_VRAM, 2314 * or their combination to let the driver choose). CPU visibility is 2315 * automatically enforced by amdgpu_bo_pin() 2316 * 2317 * Pins the BO and maps it for kernel CPU access. The eviction fence is removed 2318 * from the BO, since pinned BOs cannot be evicted. The bo must remain on the 2319 * validate_list, so the GPU mapping can be restored after a page table was 2320 * evicted. 2321 * 2322 * Return: 0 on success, error code on failure 2323 */ 2324 int amdgpu_amdkfd_gpuvm_map_bo_to_kernel(struct kgd_mem *mem, void **kptr, 2325 u64 *size, u32 domain) 2326 { 2327 int ret; 2328 struct amdgpu_bo *bo = mem->bo; 2329 2330 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 2331 pr_err("userptr can't be mapped to kernel\n"); 2332 return -EINVAL; 2333 } 2334 2335 if (!(domain & (AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM))) { 2336 pr_debug("Invalid domain 0x%x for kernel mapping\n", domain); 2337 return -EINVAL; 2338 } 2339 2340 mutex_lock(&mem->process_info->lock); 2341 2342 ret = amdgpu_bo_reserve(bo, true); 2343 if (ret) { 2344 pr_err("Failed to reserve bo. ret %d\n", ret); 2345 goto bo_reserve_failed; 2346 } 2347 2348 ret = amdgpu_bo_pin(bo, domain); 2349 if (ret) { 2350 pr_err("Failed to pin bo. ret %d\n", ret); 2351 goto pin_failed; 2352 } 2353 2354 ret = amdgpu_bo_kmap(bo, kptr); 2355 if (ret) { 2356 pr_err("Failed to map bo to kernel. ret %d\n", ret); 2357 goto kmap_failed; 2358 } 2359 2360 amdgpu_amdkfd_remove_eviction_fence( 2361 bo, mem->process_info->eviction_fence); 2362 2363 if (size) 2364 *size = amdgpu_bo_size(bo); 2365 2366 amdgpu_bo_unreserve(bo); 2367 2368 mutex_unlock(&mem->process_info->lock); 2369 return 0; 2370 2371 kmap_failed: 2372 amdgpu_bo_unpin(bo); 2373 pin_failed: 2374 amdgpu_bo_unreserve(bo); 2375 bo_reserve_failed: 2376 mutex_unlock(&mem->process_info->lock); 2377 2378 return ret; 2379 } 2380 2381 /** amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel() - Unmap GTT or VRAM BO for kernel CPU access 2382 * 2383 * @mem: Buffer object to be unmapped for CPU access 2384 * 2385 * Removes the kernel CPU mapping and unpins the BO. It does not restore the 2386 * eviction fence, so this function should only be used for cleanup before the 2387 * BO is destroyed. 2388 */ 2389 void amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(struct kgd_mem *mem) 2390 { 2391 struct amdgpu_bo *bo = mem->bo; 2392 2393 (void)amdgpu_bo_reserve(bo, true); 2394 amdgpu_bo_kunmap(bo); 2395 amdgpu_bo_unpin(bo); 2396 amdgpu_bo_unreserve(bo); 2397 } 2398 2399 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev, 2400 struct kfd_vm_fault_info *mem) 2401 { 2402 if (atomic_read_acquire(&adev->gmc.vm_fault_info_updated) == 1) { 2403 *mem = *adev->gmc.vm_fault_info; 2404 atomic_set_release(&adev->gmc.vm_fault_info_updated, 0); 2405 } 2406 return 0; 2407 } 2408 2409 static int import_obj_create(struct amdgpu_device *adev, 2410 struct dma_buf *dma_buf, 2411 struct drm_gem_object *obj, 2412 uint64_t va, void *drm_priv, 2413 struct kgd_mem **mem, uint64_t *size, 2414 uint64_t *mmap_offset) 2415 { 2416 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 2417 struct amdgpu_bo *bo; 2418 int ret; 2419 2420 bo = gem_to_amdgpu_bo(obj); 2421 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | 2422 AMDGPU_GEM_DOMAIN_GTT))) 2423 /* Only VRAM and GTT BOs are supported */ 2424 return -EINVAL; 2425 2426 *mem = kzalloc_obj(struct kgd_mem); 2427 if (!*mem) 2428 return -ENOMEM; 2429 2430 ret = drm_vma_node_allow(&obj->vma_node, drm_priv); 2431 if (ret) 2432 goto err_free_mem; 2433 2434 if (size) 2435 *size = amdgpu_bo_size(bo); 2436 2437 if (mmap_offset) 2438 *mmap_offset = amdgpu_bo_mmap_offset(bo); 2439 2440 INIT_LIST_HEAD(&(*mem)->attachments); 2441 mutex_init(&(*mem)->lock); 2442 2443 (*mem)->alloc_flags = 2444 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? 2445 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT) 2446 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE 2447 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 2448 2449 get_dma_buf(dma_buf); 2450 (*mem)->dmabuf = dma_buf; 2451 (*mem)->bo = bo; 2452 (*mem)->va = va; 2453 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) && 2454 !adev->apu_prefer_gtt ? 2455 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT; 2456 2457 (*mem)->mapped_to_gpu_memory = 0; 2458 (*mem)->process_info = avm->process_info; 2459 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false); 2460 amdgpu_sync_create(&(*mem)->sync); 2461 (*mem)->is_imported = true; 2462 2463 mutex_lock(&avm->process_info->lock); 2464 if (avm->process_info->eviction_fence && 2465 !dma_fence_is_signaled(&avm->process_info->eviction_fence->base)) 2466 ret = amdgpu_amdkfd_bo_validate_and_fence(bo, (*mem)->domain, 2467 &avm->process_info->eviction_fence->base); 2468 mutex_unlock(&avm->process_info->lock); 2469 if (ret) 2470 goto err_remove_mem; 2471 2472 return 0; 2473 2474 err_remove_mem: 2475 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); 2476 drm_vma_node_revoke(&obj->vma_node, drm_priv); 2477 err_free_mem: 2478 kfree(*mem); 2479 return ret; 2480 } 2481 2482 int amdgpu_amdkfd_gpuvm_import_dmabuf_fd(struct amdgpu_device *adev, int fd, 2483 uint64_t va, void *drm_priv, 2484 struct kgd_mem **mem, uint64_t *size, 2485 uint64_t *mmap_offset) 2486 { 2487 struct drm_gem_object *obj; 2488 uint32_t handle; 2489 int ret; 2490 2491 ret = drm_gem_prime_fd_to_handle(&adev->ddev, adev->kfd.client.file, fd, 2492 &handle); 2493 if (ret) 2494 return ret; 2495 obj = drm_gem_object_lookup(adev->kfd.client.file, handle); 2496 if (!obj) { 2497 ret = -EINVAL; 2498 goto err_release_handle; 2499 } 2500 2501 ret = import_obj_create(adev, obj->dma_buf, obj, va, drm_priv, mem, size, 2502 mmap_offset); 2503 if (ret) 2504 goto err_put_obj; 2505 2506 (*mem)->gem_handle = handle; 2507 2508 return 0; 2509 2510 err_put_obj: 2511 drm_gem_object_put(obj); 2512 err_release_handle: 2513 drm_gem_handle_delete(adev->kfd.client.file, handle); 2514 return ret; 2515 } 2516 2517 int amdgpu_amdkfd_gpuvm_export_dmabuf(struct kgd_mem *mem, 2518 struct dma_buf **dma_buf) 2519 { 2520 int ret; 2521 2522 mutex_lock(&mem->lock); 2523 ret = kfd_mem_export_dmabuf(mem); 2524 if (ret) 2525 goto out; 2526 2527 get_dma_buf(mem->dmabuf); 2528 *dma_buf = mem->dmabuf; 2529 out: 2530 mutex_unlock(&mem->lock); 2531 return ret; 2532 } 2533 2534 /* Evict a userptr BO by stopping the queues if necessary 2535 * 2536 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it 2537 * cannot do any memory allocations, and cannot take any locks that 2538 * are held elsewhere while allocating memory. 2539 * 2540 * It doesn't do anything to the BO itself. The real work happens in 2541 * restore, where we get updated page addresses. This function only 2542 * ensures that GPU access to the BO is stopped. 2543 */ 2544 int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni, 2545 unsigned long cur_seq, struct kgd_mem *mem) 2546 { 2547 struct amdkfd_process_info *process_info = mem->process_info; 2548 int r = 0; 2549 2550 /* Do not process MMU notifications during CRIU restore until 2551 * KFD_CRIU_OP_RESUME IOCTL is received 2552 */ 2553 if (READ_ONCE(process_info->block_mmu_notifications)) 2554 return 0; 2555 2556 mutex_lock(&process_info->notifier_lock); 2557 mmu_interval_set_seq(mni, cur_seq); 2558 2559 mem->invalid++; 2560 if (++process_info->evicted_bos == 1) { 2561 /* First eviction, stop the queues */ 2562 r = kgd2kfd_quiesce_mm(mni->mm, 2563 KFD_QUEUE_EVICTION_TRIGGER_USERPTR); 2564 2565 if (r && r != -ESRCH) 2566 pr_err("Failed to quiesce KFD\n"); 2567 2568 if (r != -ESRCH) 2569 queue_delayed_work(system_freezable_wq, 2570 &process_info->restore_userptr_work, 2571 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2572 } 2573 mutex_unlock(&process_info->notifier_lock); 2574 2575 return r; 2576 } 2577 2578 /* Update invalid userptr BOs 2579 * 2580 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to 2581 * userptr_inval_list and updates user pages for all BOs that have 2582 * been invalidated since their last update. 2583 */ 2584 static int update_invalid_user_pages(struct amdkfd_process_info *process_info, 2585 struct mm_struct *mm) 2586 { 2587 struct kgd_mem *mem, *tmp_mem; 2588 struct amdgpu_bo *bo; 2589 struct ttm_operation_ctx ctx = { false, false }; 2590 uint32_t invalid; 2591 int ret = 0; 2592 2593 mutex_lock(&process_info->notifier_lock); 2594 2595 /* Move all invalidated BOs to the userptr_inval_list */ 2596 list_for_each_entry_safe(mem, tmp_mem, 2597 &process_info->userptr_valid_list, 2598 validate_list) 2599 if (mem->invalid) 2600 list_move_tail(&mem->validate_list, 2601 &process_info->userptr_inval_list); 2602 2603 /* Go through userptr_inval_list and update any invalid user_pages */ 2604 list_for_each_entry(mem, &process_info->userptr_inval_list, 2605 validate_list) { 2606 invalid = mem->invalid; 2607 if (!invalid) 2608 /* BO hasn't been invalidated since the last 2609 * revalidation attempt. Keep its page list. 2610 */ 2611 continue; 2612 2613 bo = mem->bo; 2614 2615 amdgpu_hmm_range_free(mem->range); 2616 mem->range = NULL; 2617 2618 /* BO reservations and getting user pages (hmm_range_fault) 2619 * must happen outside the notifier lock 2620 */ 2621 mutex_unlock(&process_info->notifier_lock); 2622 2623 /* Move the BO to system (CPU) domain if necessary to unmap 2624 * and free the SG table 2625 */ 2626 if (bo->tbo.resource->mem_type != TTM_PL_SYSTEM) { 2627 if (amdgpu_bo_reserve(bo, true)) 2628 return -EAGAIN; 2629 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 2630 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2631 amdgpu_bo_unreserve(bo); 2632 if (ret) { 2633 pr_err("%s: Failed to invalidate userptr BO\n", 2634 __func__); 2635 return -EAGAIN; 2636 } 2637 } 2638 2639 mem->range = amdgpu_hmm_range_alloc(NULL); 2640 if (unlikely(!mem->range)) 2641 return -ENOMEM; 2642 /* Get updated user pages */ 2643 ret = amdgpu_ttm_tt_get_user_pages(bo, mem->range); 2644 if (ret) { 2645 amdgpu_hmm_range_free(mem->range); 2646 mem->range = NULL; 2647 pr_debug("Failed %d to get user pages\n", ret); 2648 2649 /* Return -EFAULT bad address error as success. It will 2650 * fail later with a VM fault if the GPU tries to access 2651 * it. Better than hanging indefinitely with stalled 2652 * user mode queues. 2653 * 2654 * Return other error -EBUSY or -ENOMEM to retry restore 2655 */ 2656 if (ret != -EFAULT) 2657 return ret; 2658 2659 /* If applications unmap memory before destroying the userptr 2660 * from the KFD, trigger a segmentation fault in VM debug mode. 2661 */ 2662 if (amdgpu_ttm_adev(bo->tbo.bdev)->debug_vm_userptr) { 2663 struct kfd_process *p; 2664 2665 pr_err("Pid %d unmapped memory before destroying userptr at GPU addr 0x%llx\n", 2666 pid_nr(process_info->pid), mem->va); 2667 2668 // Send GPU VM fault to user space 2669 p = kfd_lookup_process_by_pid(process_info->pid); 2670 if (p) { 2671 kfd_signal_vm_fault_event_with_userptr(p, mem->va); 2672 kfd_unref_process(p); 2673 } 2674 } 2675 2676 ret = 0; 2677 } 2678 2679 amdgpu_ttm_tt_set_user_pages(bo->tbo.ttm, mem->range); 2680 2681 mutex_lock(&process_info->notifier_lock); 2682 2683 /* Mark the BO as valid unless it was invalidated 2684 * again concurrently. 2685 */ 2686 if (mem->invalid != invalid) { 2687 ret = -EAGAIN; 2688 goto unlock_out; 2689 } 2690 /* set mem valid if mem has hmm range associated */ 2691 if (mem->range) 2692 mem->invalid = 0; 2693 } 2694 2695 unlock_out: 2696 mutex_unlock(&process_info->notifier_lock); 2697 2698 return ret; 2699 } 2700 2701 /* Validate invalid userptr BOs 2702 * 2703 * Validates BOs on the userptr_inval_list. Also updates GPUVM page tables 2704 * with new page addresses and waits for the page table updates to complete. 2705 */ 2706 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) 2707 { 2708 struct ttm_operation_ctx ctx = { false, false }; 2709 struct amdgpu_sync sync; 2710 struct drm_exec exec; 2711 2712 struct amdgpu_vm *peer_vm; 2713 struct kgd_mem *mem, *tmp_mem; 2714 struct amdgpu_bo *bo; 2715 int ret; 2716 2717 amdgpu_sync_create(&sync); 2718 2719 drm_exec_init(&exec, 0, 0); 2720 /* Reserve all BOs and page tables for validation */ 2721 drm_exec_until_all_locked(&exec) { 2722 /* Reserve all the page directories */ 2723 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2724 vm_list_node) { 2725 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2); 2726 drm_exec_retry_on_contention(&exec); 2727 if (unlikely(ret)) 2728 goto unreserve_out; 2729 } 2730 2731 /* Reserve the userptr_inval_list entries to resv_list */ 2732 list_for_each_entry(mem, &process_info->userptr_inval_list, 2733 validate_list) { 2734 struct drm_gem_object *gobj; 2735 2736 gobj = &mem->bo->tbo.base; 2737 ret = drm_exec_prepare_obj(&exec, gobj, 1); 2738 drm_exec_retry_on_contention(&exec); 2739 if (unlikely(ret)) 2740 goto unreserve_out; 2741 } 2742 } 2743 2744 ret = process_validate_vms(process_info, NULL); 2745 if (ret) 2746 goto unreserve_out; 2747 2748 /* Validate BOs and update GPUVM page tables */ 2749 list_for_each_entry_safe(mem, tmp_mem, 2750 &process_info->userptr_inval_list, 2751 validate_list) { 2752 struct kfd_mem_attachment *attachment; 2753 2754 bo = mem->bo; 2755 2756 /* Validate the BO if we got user pages */ 2757 if (bo->tbo.ttm->pages[0]) { 2758 amdgpu_bo_placement_from_domain(bo, mem->domain); 2759 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2760 if (ret) { 2761 pr_err("%s: failed to validate BO\n", __func__); 2762 goto unreserve_out; 2763 } 2764 } 2765 2766 /* Update mapping. If the BO was not validated 2767 * (because we couldn't get user pages), this will 2768 * clear the page table entries, which will result in 2769 * VM faults if the GPU tries to access the invalid 2770 * memory. 2771 */ 2772 list_for_each_entry(attachment, &mem->attachments, list) { 2773 if (!attachment->is_mapped) 2774 continue; 2775 2776 kfd_mem_dmaunmap_attachment(mem, attachment); 2777 ret = update_gpuvm_pte(mem, attachment, &sync); 2778 if (ret) { 2779 pr_err("%s: update PTE failed\n", __func__); 2780 /* make sure this gets validated again */ 2781 mutex_lock(&process_info->notifier_lock); 2782 mem->invalid++; 2783 mutex_unlock(&process_info->notifier_lock); 2784 goto unreserve_out; 2785 } 2786 } 2787 } 2788 2789 /* Update page directories */ 2790 ret = process_update_pds(process_info, &sync); 2791 2792 unreserve_out: 2793 drm_exec_fini(&exec); 2794 amdgpu_sync_wait(&sync, false); 2795 amdgpu_sync_free(&sync); 2796 2797 return ret; 2798 } 2799 2800 /* Confirm that all user pages are valid while holding the notifier lock 2801 * 2802 * Moves valid BOs from the userptr_inval_list back to userptr_val_list. 2803 */ 2804 static int confirm_valid_user_pages_locked(struct amdkfd_process_info *process_info) 2805 { 2806 struct kgd_mem *mem, *tmp_mem; 2807 int ret = 0; 2808 2809 list_for_each_entry_safe(mem, tmp_mem, 2810 &process_info->userptr_inval_list, 2811 validate_list) { 2812 bool valid; 2813 2814 /* keep mem without hmm range at userptr_inval_list */ 2815 if (!mem->range) 2816 continue; 2817 2818 /* Only check mem with hmm range associated */ 2819 valid = amdgpu_hmm_range_valid(mem->range); 2820 amdgpu_hmm_range_free(mem->range); 2821 2822 mem->range = NULL; 2823 if (!valid) { 2824 WARN(!mem->invalid, "Invalid BO not marked invalid"); 2825 ret = -EAGAIN; 2826 continue; 2827 } 2828 2829 if (mem->invalid) { 2830 WARN(1, "Valid BO is marked invalid"); 2831 ret = -EAGAIN; 2832 continue; 2833 } 2834 2835 list_move_tail(&mem->validate_list, 2836 &process_info->userptr_valid_list); 2837 } 2838 2839 return ret; 2840 } 2841 2842 /* Worker callback to restore evicted userptr BOs 2843 * 2844 * Tries to update and validate all userptr BOs. If successful and no 2845 * concurrent evictions happened, the queues are restarted. Otherwise, 2846 * reschedule for another attempt later. 2847 */ 2848 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) 2849 { 2850 struct delayed_work *dwork = to_delayed_work(work); 2851 struct amdkfd_process_info *process_info = 2852 container_of(dwork, struct amdkfd_process_info, 2853 restore_userptr_work); 2854 struct task_struct *usertask; 2855 struct mm_struct *mm; 2856 uint32_t evicted_bos; 2857 int ret; 2858 2859 mutex_lock(&process_info->notifier_lock); 2860 evicted_bos = process_info->evicted_bos; 2861 mutex_unlock(&process_info->notifier_lock); 2862 if (!evicted_bos) 2863 return; 2864 2865 /* Reference task and mm in case of concurrent process termination */ 2866 usertask = get_pid_task(process_info->pid, PIDTYPE_PID); 2867 if (!usertask) 2868 return; 2869 mm = get_task_mm(usertask); 2870 if (!mm) { 2871 put_task_struct(usertask); 2872 return; 2873 } 2874 2875 mutex_lock(&process_info->lock); 2876 2877 ret = update_invalid_user_pages(process_info, mm); 2878 if (ret) 2879 goto unlock_out; 2880 /* userptr_inval_list can be empty if all evicted userptr BOs 2881 * have been freed. In that case there is nothing to validate 2882 * and we can just restart the queues. 2883 */ 2884 if (!list_empty(&process_info->userptr_inval_list)) { 2885 ret = validate_invalid_user_pages(process_info); 2886 if (ret) 2887 goto unlock_out; 2888 } 2889 /* Final check for concurrent evicton and atomic update. If 2890 * another eviction happens after successful update, it will 2891 * be a first eviction that calls quiesce_mm. The eviction 2892 * reference counting inside KFD will handle this case. 2893 */ 2894 mutex_lock(&process_info->notifier_lock); 2895 if (process_info->evicted_bos != evicted_bos) { 2896 ret = -EAGAIN; 2897 goto unlock_notifier_out; 2898 } 2899 2900 if (confirm_valid_user_pages_locked(process_info)) { 2901 WARN(1, "User pages unexpectedly invalid"); 2902 ret = -EINVAL; 2903 goto unlock_notifier_out; 2904 } 2905 2906 process_info->evicted_bos = evicted_bos = 0; 2907 2908 if (kgd2kfd_resume_mm(mm)) { 2909 pr_err("%s: Failed to resume KFD\n", __func__); 2910 /* No recovery from this failure. Probably the CP is 2911 * hanging. No point trying again. 2912 */ 2913 } 2914 2915 unlock_notifier_out: 2916 mutex_unlock(&process_info->notifier_lock); 2917 unlock_out: 2918 mutex_unlock(&process_info->lock); 2919 2920 /* If validation failed, reschedule another attempt, unless the 2921 * device is gone for good 2922 */ 2923 if (evicted_bos && ret != -ENODEV) { 2924 queue_delayed_work(system_freezable_wq, 2925 &process_info->restore_userptr_work, 2926 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2927 2928 kfd_smi_event_queue_restore_rescheduled(mm); 2929 } 2930 mmput(mm); 2931 put_task_struct(usertask); 2932 } 2933 2934 static void replace_eviction_fence(struct dma_fence __rcu **ef, 2935 struct dma_fence *new_ef) 2936 { 2937 struct dma_fence *old_ef = rcu_replace_pointer(*ef, new_ef, true 2938 /* protected by process_info->lock */); 2939 2940 /* If we're replacing an unsignaled eviction fence, that fence will 2941 * never be signaled, and if anyone is still waiting on that fence, 2942 * they will hang forever. This should never happen. We should only 2943 * replace the fence in restore_work that only gets scheduled after 2944 * eviction work signaled the fence. 2945 */ 2946 WARN_ONCE(!dma_fence_is_signaled(old_ef), 2947 "Replacing unsignaled eviction fence"); 2948 dma_fence_put(old_ef); 2949 } 2950 2951 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given 2952 * KFD process identified by process_info 2953 * 2954 * @process_info: amdkfd_process_info of the KFD process 2955 * 2956 * After memory eviction, restore thread calls this function. The function 2957 * should be called when the Process is still valid. BO restore involves - 2958 * 2959 * 1. Release old eviction fence and create new one 2960 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list. 2961 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of 2962 * BOs that need to be reserved. 2963 * 4. Reserve all the BOs 2964 * 5. Validate of PD and PT BOs. 2965 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence 2966 * 7. Add fence to all PD and PT BOs. 2967 * 8. Unreserve all BOs 2968 */ 2969 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence __rcu **ef) 2970 { 2971 struct amdkfd_process_info *process_info = info; 2972 struct amdgpu_vm *peer_vm; 2973 struct kgd_mem *mem; 2974 struct list_head duplicate_save; 2975 struct amdgpu_sync sync_obj; 2976 unsigned long failed_size = 0; 2977 unsigned long total_size = 0; 2978 struct drm_exec exec; 2979 int ret; 2980 2981 INIT_LIST_HEAD(&duplicate_save); 2982 2983 mutex_lock(&process_info->lock); 2984 2985 drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 0); 2986 drm_exec_until_all_locked(&exec) { 2987 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2988 vm_list_node) { 2989 ret = amdgpu_vm_lock_pd(peer_vm, &exec, 2); 2990 drm_exec_retry_on_contention(&exec); 2991 if (unlikely(ret)) { 2992 pr_err("Locking VM PD failed, ret: %d\n", ret); 2993 goto ttm_reserve_fail; 2994 } 2995 } 2996 2997 /* Reserve all BOs and page tables/directory. Add all BOs from 2998 * kfd_bo_list to ctx.list 2999 */ 3000 list_for_each_entry(mem, &process_info->kfd_bo_list, 3001 validate_list) { 3002 struct drm_gem_object *gobj; 3003 3004 gobj = &mem->bo->tbo.base; 3005 ret = drm_exec_prepare_obj(&exec, gobj, 1); 3006 drm_exec_retry_on_contention(&exec); 3007 if (unlikely(ret)) { 3008 pr_err("drm_exec_prepare_obj failed, ret: %d\n", ret); 3009 goto ttm_reserve_fail; 3010 } 3011 } 3012 } 3013 3014 amdgpu_sync_create(&sync_obj); 3015 3016 /* Validate BOs managed by KFD */ 3017 list_for_each_entry(mem, &process_info->kfd_bo_list, 3018 validate_list) { 3019 3020 struct amdgpu_bo *bo = mem->bo; 3021 uint32_t domain = mem->domain; 3022 struct dma_resv_iter cursor; 3023 struct dma_fence *fence; 3024 3025 total_size += amdgpu_bo_size(bo); 3026 3027 ret = amdgpu_amdkfd_bo_validate(bo, domain, false); 3028 if (ret) { 3029 pr_debug("Memory eviction: Validate BOs failed\n"); 3030 failed_size += amdgpu_bo_size(bo); 3031 ret = amdgpu_amdkfd_bo_validate(bo, 3032 AMDGPU_GEM_DOMAIN_GTT, false); 3033 if (ret) { 3034 pr_debug("Memory eviction: Try again\n"); 3035 goto validate_map_fail; 3036 } 3037 } 3038 dma_resv_for_each_fence(&cursor, bo->tbo.base.resv, 3039 DMA_RESV_USAGE_KERNEL, fence) { 3040 ret = amdgpu_sync_fence(&sync_obj, fence, GFP_KERNEL); 3041 if (ret) { 3042 pr_debug("Memory eviction: Sync BO fence failed. Try again\n"); 3043 goto validate_map_fail; 3044 } 3045 } 3046 } 3047 3048 if (failed_size) 3049 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size); 3050 3051 /* Validate PDs, PTs and evicted DMABuf imports last. Otherwise BO 3052 * validations above would invalidate DMABuf imports again. 3053 */ 3054 ret = process_validate_vms(process_info, drm_exec_ticket(&exec)); 3055 if (ret) { 3056 pr_debug("Validating VMs failed, ret: %d\n", ret); 3057 goto validate_map_fail; 3058 } 3059 3060 /* Update mappings managed by KFD. */ 3061 list_for_each_entry(mem, &process_info->kfd_bo_list, 3062 validate_list) { 3063 struct kfd_mem_attachment *attachment; 3064 3065 list_for_each_entry(attachment, &mem->attachments, list) { 3066 if (!attachment->is_mapped) 3067 continue; 3068 3069 kfd_mem_dmaunmap_attachment(mem, attachment); 3070 ret = update_gpuvm_pte(mem, attachment, &sync_obj); 3071 if (ret) { 3072 pr_debug("Memory eviction: update PTE failed. Try again\n"); 3073 goto validate_map_fail; 3074 } 3075 } 3076 } 3077 3078 /* Update mappings not managed by KFD */ 3079 list_for_each_entry(peer_vm, &process_info->vm_list_head, 3080 vm_list_node) { 3081 struct amdgpu_device *adev = amdgpu_ttm_adev( 3082 peer_vm->root.bo->tbo.bdev); 3083 3084 struct amdgpu_fpriv *fpriv = 3085 container_of(peer_vm, struct amdgpu_fpriv, vm); 3086 3087 ret = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 3088 if (ret) { 3089 dev_dbg(adev->dev, 3090 "Memory eviction: handle PRT moved failed, pid %8d. Try again.\n", 3091 pid_nr(process_info->pid)); 3092 goto validate_map_fail; 3093 } 3094 3095 ret = amdgpu_vm_handle_moved(adev, peer_vm, drm_exec_ticket(&exec)); 3096 if (ret) { 3097 dev_dbg(adev->dev, 3098 "Memory eviction: handle moved failed, pid %8d. Try again.\n", 3099 pid_nr(process_info->pid)); 3100 goto validate_map_fail; 3101 } 3102 } 3103 3104 /* Update page directories */ 3105 ret = process_update_pds(process_info, &sync_obj); 3106 if (ret) { 3107 pr_debug("Memory eviction: update PDs failed. Try again\n"); 3108 goto validate_map_fail; 3109 } 3110 3111 /* Sync with fences on all the page tables. They implicitly depend on any 3112 * move fences from amdgpu_vm_handle_moved above. 3113 */ 3114 ret = process_sync_pds_resv(process_info, &sync_obj); 3115 if (ret) { 3116 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n"); 3117 goto validate_map_fail; 3118 } 3119 3120 /* Wait for validate and PT updates to finish */ 3121 amdgpu_sync_wait(&sync_obj, false); 3122 3123 /* The old eviction fence may be unsignaled if restore happens 3124 * after a GPU reset or suspend/resume. Keep the old fence in that 3125 * case. Otherwise release the old eviction fence and create new 3126 * one, because fence only goes from unsignaled to signaled once 3127 * and cannot be reused. Use context and mm from the old fence. 3128 * 3129 * If an old eviction fence signals after this check, that's OK. 3130 * Anyone signaling an eviction fence must stop the queues first 3131 * and schedule another restore worker. 3132 */ 3133 if (dma_fence_is_signaled(&process_info->eviction_fence->base)) { 3134 struct amdgpu_amdkfd_fence *new_fence = 3135 amdgpu_amdkfd_fence_create( 3136 process_info->eviction_fence->base.context, 3137 process_info->eviction_fence->mm, 3138 process_info->context_id); 3139 3140 if (!new_fence) { 3141 pr_err("Failed to create eviction fence\n"); 3142 ret = -ENOMEM; 3143 goto validate_map_fail; 3144 } 3145 dma_fence_put(&process_info->eviction_fence->base); 3146 process_info->eviction_fence = new_fence; 3147 replace_eviction_fence(ef, dma_fence_get(&new_fence->base)); 3148 } else { 3149 WARN_ONCE(rcu_access_pointer(*ef) != &process_info->eviction_fence->base, 3150 "KFD eviction fence doesn't match KGD process_info"); 3151 } 3152 3153 /* Attach new eviction fence to all BOs except pinned ones */ 3154 list_for_each_entry(mem, &process_info->kfd_bo_list, validate_list) { 3155 if (mem->bo->tbo.pin_count) 3156 continue; 3157 3158 dma_resv_add_fence(mem->bo->tbo.base.resv, 3159 &process_info->eviction_fence->base, 3160 DMA_RESV_USAGE_BOOKKEEP); 3161 } 3162 /* Attach eviction fence to PD / PT BOs and DMABuf imports */ 3163 list_for_each_entry(peer_vm, &process_info->vm_list_head, 3164 vm_list_node) { 3165 struct amdgpu_bo *bo = peer_vm->root.bo; 3166 3167 dma_resv_add_fence(bo->tbo.base.resv, 3168 &process_info->eviction_fence->base, 3169 DMA_RESV_USAGE_BOOKKEEP); 3170 } 3171 3172 validate_map_fail: 3173 amdgpu_sync_free(&sync_obj); 3174 ttm_reserve_fail: 3175 drm_exec_fini(&exec); 3176 mutex_unlock(&process_info->lock); 3177 return ret; 3178 } 3179 3180 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem) 3181 { 3182 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 3183 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws; 3184 int ret; 3185 3186 if (!info || !gws) 3187 return -EINVAL; 3188 3189 *mem = kzalloc_obj(struct kgd_mem); 3190 if (!*mem) 3191 return -ENOMEM; 3192 3193 mutex_init(&(*mem)->lock); 3194 INIT_LIST_HEAD(&(*mem)->attachments); 3195 (*mem)->bo = amdgpu_bo_ref(gws_bo); 3196 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; 3197 (*mem)->process_info = process_info; 3198 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false); 3199 amdgpu_sync_create(&(*mem)->sync); 3200 3201 3202 /* Validate gws bo the first time it is added to process */ 3203 mutex_lock(&(*mem)->process_info->lock); 3204 ret = amdgpu_bo_reserve(gws_bo, false); 3205 if (unlikely(ret)) { 3206 pr_err("Reserve gws bo failed %d\n", ret); 3207 goto bo_reservation_failure; 3208 } 3209 3210 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true); 3211 if (ret) { 3212 pr_err("GWS BO validate failed %d\n", ret); 3213 goto bo_validation_failure; 3214 } 3215 /* GWS resource is shared b/t amdgpu and amdkfd 3216 * Add process eviction fence to bo so they can 3217 * evict each other. 3218 */ 3219 ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1); 3220 if (ret) 3221 goto reserve_shared_fail; 3222 dma_resv_add_fence(gws_bo->tbo.base.resv, 3223 &process_info->eviction_fence->base, 3224 DMA_RESV_USAGE_BOOKKEEP); 3225 amdgpu_bo_unreserve(gws_bo); 3226 mutex_unlock(&(*mem)->process_info->lock); 3227 3228 return ret; 3229 3230 reserve_shared_fail: 3231 bo_validation_failure: 3232 amdgpu_bo_unreserve(gws_bo); 3233 bo_reservation_failure: 3234 mutex_unlock(&(*mem)->process_info->lock); 3235 amdgpu_sync_free(&(*mem)->sync); 3236 remove_kgd_mem_from_kfd_bo_list(*mem, process_info); 3237 amdgpu_bo_unref(&gws_bo); 3238 mutex_destroy(&(*mem)->lock); 3239 kfree(*mem); 3240 *mem = NULL; 3241 return ret; 3242 } 3243 3244 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem) 3245 { 3246 int ret; 3247 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 3248 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem; 3249 struct amdgpu_bo *gws_bo = kgd_mem->bo; 3250 3251 /* Remove BO from process's validate list so restore worker won't touch 3252 * it anymore 3253 */ 3254 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info); 3255 3256 ret = amdgpu_bo_reserve(gws_bo, false); 3257 if (unlikely(ret)) { 3258 pr_err("Reserve gws bo failed %d\n", ret); 3259 //TODO add BO back to validate_list? 3260 return ret; 3261 } 3262 amdgpu_amdkfd_remove_eviction_fence(gws_bo, 3263 process_info->eviction_fence); 3264 amdgpu_bo_unreserve(gws_bo); 3265 amdgpu_sync_free(&kgd_mem->sync); 3266 amdgpu_bo_unref(&gws_bo); 3267 mutex_destroy(&kgd_mem->lock); 3268 kfree(mem); 3269 return 0; 3270 } 3271 3272 /* Returns GPU-specific tiling mode information */ 3273 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev, 3274 struct tile_config *config) 3275 { 3276 config->gb_addr_config = adev->gfx.config.gb_addr_config; 3277 config->tile_config_ptr = adev->gfx.config.tile_mode_array; 3278 config->num_tile_configs = 3279 ARRAY_SIZE(adev->gfx.config.tile_mode_array); 3280 config->macro_tile_config_ptr = 3281 adev->gfx.config.macrotile_mode_array; 3282 config->num_macro_tile_configs = 3283 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 3284 3285 /* Those values are not set from GFX9 onwards */ 3286 config->num_banks = adev->gfx.config.num_banks; 3287 config->num_ranks = adev->gfx.config.num_ranks; 3288 3289 return 0; 3290 } 3291 3292 bool amdgpu_amdkfd_bo_mapped_to_dev(void *drm_priv, struct kgd_mem *mem) 3293 { 3294 struct amdgpu_vm *vm = drm_priv_to_vm(drm_priv); 3295 struct kfd_mem_attachment *entry; 3296 3297 list_for_each_entry(entry, &mem->attachments, list) { 3298 if (entry->is_mapped && entry->bo_va->base.vm == vm) 3299 return true; 3300 } 3301 return false; 3302 } 3303 3304 #if defined(CONFIG_DEBUG_FS) 3305 3306 int kfd_debugfs_kfd_mem_limits(struct seq_file *m, void *data) 3307 { 3308 3309 spin_lock(&kfd_mem_limit.mem_limit_lock); 3310 seq_printf(m, "System mem used %lldM out of %lluM\n", 3311 (kfd_mem_limit.system_mem_used >> 20), 3312 (kfd_mem_limit.max_system_mem_limit >> 20)); 3313 seq_printf(m, "TTM mem used %lldM out of %lluM\n", 3314 (kfd_mem_limit.ttm_mem_used >> 20), 3315 (kfd_mem_limit.max_ttm_mem_limit >> 20)); 3316 spin_unlock(&kfd_mem_limit.mem_limit_lock); 3317 3318 return 0; 3319 } 3320 3321 #endif 3322