1 /* 2 * Copyright 2014-2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 #include <linux/dma-buf.h> 23 #include <linux/list.h> 24 #include <linux/pagemap.h> 25 #include <linux/sched/mm.h> 26 #include <linux/sched/task.h> 27 28 #include "amdgpu_object.h" 29 #include "amdgpu_gem.h" 30 #include "amdgpu_vm.h" 31 #include "amdgpu_amdkfd.h" 32 #include "amdgpu_dma_buf.h" 33 #include <uapi/linux/kfd_ioctl.h> 34 #include "amdgpu_xgmi.h" 35 36 /* Userptr restore delay, just long enough to allow consecutive VM 37 * changes to accumulate 38 */ 39 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 40 41 /* Impose limit on how much memory KFD can use */ 42 static struct { 43 uint64_t max_system_mem_limit; 44 uint64_t max_ttm_mem_limit; 45 int64_t system_mem_used; 46 int64_t ttm_mem_used; 47 spinlock_t mem_limit_lock; 48 } kfd_mem_limit; 49 50 static const char * const domain_bit_to_string[] = { 51 "CPU", 52 "GTT", 53 "VRAM", 54 "GDS", 55 "GWS", 56 "OA" 57 }; 58 59 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] 60 61 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); 62 63 static bool kfd_mem_is_attached(struct amdgpu_vm *avm, 64 struct kgd_mem *mem) 65 { 66 struct kfd_mem_attachment *entry; 67 68 list_for_each_entry(entry, &mem->attachments, list) 69 if (entry->bo_va->base.vm == avm) 70 return true; 71 72 return false; 73 } 74 75 /* Set memory usage limits. Current, limits are 76 * System (TTM + userptr) memory - 15/16th System RAM 77 * TTM memory - 3/8th System RAM 78 */ 79 void amdgpu_amdkfd_gpuvm_init_mem_limits(void) 80 { 81 struct sysinfo si; 82 uint64_t mem; 83 84 si_meminfo(&si); 85 mem = si.freeram - si.freehigh; 86 mem *= si.mem_unit; 87 88 spin_lock_init(&kfd_mem_limit.mem_limit_lock); 89 kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4); 90 kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); 91 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", 92 (kfd_mem_limit.max_system_mem_limit >> 20), 93 (kfd_mem_limit.max_ttm_mem_limit >> 20)); 94 } 95 96 void amdgpu_amdkfd_reserve_system_mem(uint64_t size) 97 { 98 kfd_mem_limit.system_mem_used += size; 99 } 100 101 /* Estimate page table size needed to represent a given memory size 102 * 103 * With 4KB pages, we need one 8 byte PTE for each 4KB of memory 104 * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB 105 * of memory (factor 256K, >> 18). ROCm user mode tries to optimize 106 * for 2MB pages for TLB efficiency. However, small allocations and 107 * fragmented system memory still need some 4KB pages. We choose a 108 * compromise that should work in most cases without reserving too 109 * much memory for page tables unnecessarily (factor 16K, >> 14). 110 */ 111 #define ESTIMATE_PT_SIZE(mem_size) ((mem_size) >> 14) 112 113 static size_t amdgpu_amdkfd_acc_size(uint64_t size) 114 { 115 size >>= PAGE_SHIFT; 116 size *= sizeof(dma_addr_t) + sizeof(void *); 117 118 return __roundup_pow_of_two(sizeof(struct amdgpu_bo)) + 119 __roundup_pow_of_two(sizeof(struct ttm_tt)) + 120 PAGE_ALIGN(size); 121 } 122 123 /** 124 * @amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size 125 * of buffer including any reserved for control structures 126 * 127 * @adev: Device to which allocated BO belongs to 128 * @size: Size of buffer, in bytes, encapsulated by B0. This should be 129 * equivalent to amdgpu_bo_size(BO) 130 * @alloc_flag: Flag used in allocating a BO as noted above 131 * 132 * Return: returns -ENOMEM in case of error, ZERO otherwise 133 */ 134 static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, 135 uint64_t size, u32 alloc_flag) 136 { 137 uint64_t reserved_for_pt = 138 ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); 139 size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed; 140 int ret = 0; 141 142 acc_size = amdgpu_amdkfd_acc_size(size); 143 144 vram_needed = 0; 145 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 146 system_mem_needed = acc_size + size; 147 ttm_mem_needed = acc_size + size; 148 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 149 system_mem_needed = acc_size; 150 ttm_mem_needed = acc_size; 151 vram_needed = size; 152 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 153 system_mem_needed = acc_size + size; 154 ttm_mem_needed = acc_size; 155 } else if (alloc_flag & 156 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 157 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 158 system_mem_needed = acc_size; 159 ttm_mem_needed = acc_size; 160 } else { 161 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 162 return -ENOMEM; 163 } 164 165 spin_lock(&kfd_mem_limit.mem_limit_lock); 166 167 if (kfd_mem_limit.system_mem_used + system_mem_needed > 168 kfd_mem_limit.max_system_mem_limit) 169 pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); 170 171 if ((kfd_mem_limit.system_mem_used + system_mem_needed > 172 kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || 173 (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > 174 kfd_mem_limit.max_ttm_mem_limit) || 175 (adev->kfd.vram_used + vram_needed > 176 adev->gmc.real_vram_size - reserved_for_pt)) { 177 ret = -ENOMEM; 178 goto release; 179 } 180 181 /* Update memory accounting by decreasing available system 182 * memory, TTM memory and GPU memory as computed above 183 */ 184 adev->kfd.vram_used += vram_needed; 185 kfd_mem_limit.system_mem_used += system_mem_needed; 186 kfd_mem_limit.ttm_mem_used += ttm_mem_needed; 187 188 release: 189 spin_unlock(&kfd_mem_limit.mem_limit_lock); 190 return ret; 191 } 192 193 static void unreserve_mem_limit(struct amdgpu_device *adev, 194 uint64_t size, u32 alloc_flag) 195 { 196 size_t acc_size; 197 198 acc_size = amdgpu_amdkfd_acc_size(size); 199 200 spin_lock(&kfd_mem_limit.mem_limit_lock); 201 202 if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 203 kfd_mem_limit.system_mem_used -= (acc_size + size); 204 kfd_mem_limit.ttm_mem_used -= (acc_size + size); 205 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 206 kfd_mem_limit.system_mem_used -= acc_size; 207 kfd_mem_limit.ttm_mem_used -= acc_size; 208 adev->kfd.vram_used -= size; 209 } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 210 kfd_mem_limit.system_mem_used -= (acc_size + size); 211 kfd_mem_limit.ttm_mem_used -= acc_size; 212 } else if (alloc_flag & 213 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 214 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 215 kfd_mem_limit.system_mem_used -= acc_size; 216 kfd_mem_limit.ttm_mem_used -= acc_size; 217 } else { 218 pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); 219 goto release; 220 } 221 222 WARN_ONCE(adev->kfd.vram_used < 0, 223 "KFD VRAM memory accounting unbalanced"); 224 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, 225 "KFD TTM memory accounting unbalanced"); 226 WARN_ONCE(kfd_mem_limit.system_mem_used < 0, 227 "KFD system memory accounting unbalanced"); 228 229 release: 230 spin_unlock(&kfd_mem_limit.mem_limit_lock); 231 } 232 233 void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) 234 { 235 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 236 u32 alloc_flags = bo->kfd_bo->alloc_flags; 237 u64 size = amdgpu_bo_size(bo); 238 239 unreserve_mem_limit(adev, size, alloc_flags); 240 241 kfree(bo->kfd_bo); 242 } 243 244 /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's 245 * reservation object. 246 * 247 * @bo: [IN] Remove eviction fence(s) from this BO 248 * @ef: [IN] This eviction fence is removed if it 249 * is present in the shared list. 250 * 251 * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. 252 */ 253 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, 254 struct amdgpu_amdkfd_fence *ef) 255 { 256 struct dma_resv *resv = bo->tbo.base.resv; 257 struct dma_resv_list *old, *new; 258 unsigned int i, j, k; 259 260 if (!ef) 261 return -EINVAL; 262 263 old = dma_resv_shared_list(resv); 264 if (!old) 265 return 0; 266 267 new = kmalloc(struct_size(new, shared, old->shared_max), GFP_KERNEL); 268 if (!new) 269 return -ENOMEM; 270 271 /* Go through all the shared fences in the resevation object and sort 272 * the interesting ones to the end of the list. 273 */ 274 for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) { 275 struct dma_fence *f; 276 277 f = rcu_dereference_protected(old->shared[i], 278 dma_resv_held(resv)); 279 280 if (f->context == ef->base.context) 281 RCU_INIT_POINTER(new->shared[--j], f); 282 else 283 RCU_INIT_POINTER(new->shared[k++], f); 284 } 285 new->shared_max = old->shared_max; 286 new->shared_count = k; 287 288 /* Install the new fence list, seqcount provides the barriers */ 289 write_seqcount_begin(&resv->seq); 290 RCU_INIT_POINTER(resv->fence, new); 291 write_seqcount_end(&resv->seq); 292 293 /* Drop the references to the removed fences or move them to ef_list */ 294 for (i = j; i < old->shared_count; ++i) { 295 struct dma_fence *f; 296 297 f = rcu_dereference_protected(new->shared[i], 298 dma_resv_held(resv)); 299 dma_fence_put(f); 300 } 301 kfree_rcu(old, rcu); 302 303 return 0; 304 } 305 306 int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) 307 { 308 struct amdgpu_bo *root = bo; 309 struct amdgpu_vm_bo_base *vm_bo; 310 struct amdgpu_vm *vm; 311 struct amdkfd_process_info *info; 312 struct amdgpu_amdkfd_fence *ef; 313 int ret; 314 315 /* we can always get vm_bo from root PD bo.*/ 316 while (root->parent) 317 root = root->parent; 318 319 vm_bo = root->vm_bo; 320 if (!vm_bo) 321 return 0; 322 323 vm = vm_bo->vm; 324 if (!vm) 325 return 0; 326 327 info = vm->process_info; 328 if (!info || !info->eviction_fence) 329 return 0; 330 331 ef = container_of(dma_fence_get(&info->eviction_fence->base), 332 struct amdgpu_amdkfd_fence, base); 333 334 BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); 335 ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); 336 dma_resv_unlock(bo->tbo.base.resv); 337 338 dma_fence_put(&ef->base); 339 return ret; 340 } 341 342 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, 343 bool wait) 344 { 345 struct ttm_operation_ctx ctx = { false, false }; 346 int ret; 347 348 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), 349 "Called with userptr BO")) 350 return -EINVAL; 351 352 amdgpu_bo_placement_from_domain(bo, domain); 353 354 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 355 if (ret) 356 goto validate_fail; 357 if (wait) 358 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 359 360 validate_fail: 361 return ret; 362 } 363 364 static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) 365 { 366 return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); 367 } 368 369 /* vm_validate_pt_pd_bos - Validate page table and directory BOs 370 * 371 * Page directories are not updated here because huge page handling 372 * during page table updates can invalidate page directory entries 373 * again. Page directories are only updated after updating page 374 * tables. 375 */ 376 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm) 377 { 378 struct amdgpu_bo *pd = vm->root.bo; 379 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 380 int ret; 381 382 ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate_vm_bo, NULL); 383 if (ret) { 384 pr_err("failed to validate PT BOs\n"); 385 return ret; 386 } 387 388 ret = amdgpu_amdkfd_validate_vm_bo(NULL, pd); 389 if (ret) { 390 pr_err("failed to validate PD\n"); 391 return ret; 392 } 393 394 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); 395 396 if (vm->use_cpu_for_update) { 397 ret = amdgpu_bo_kmap(pd, NULL); 398 if (ret) { 399 pr_err("failed to kmap PD, ret=%d\n", ret); 400 return ret; 401 } 402 } 403 404 return 0; 405 } 406 407 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) 408 { 409 struct amdgpu_bo *pd = vm->root.bo; 410 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 411 int ret; 412 413 ret = amdgpu_vm_update_pdes(adev, vm, false); 414 if (ret) 415 return ret; 416 417 return amdgpu_sync_fence(sync, vm->last_update); 418 } 419 420 static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) 421 { 422 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 423 bool coherent = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; 424 bool uncached = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; 425 uint32_t mapping_flags; 426 uint64_t pte_flags; 427 bool snoop = false; 428 429 mapping_flags = AMDGPU_VM_PAGE_READABLE; 430 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) 431 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; 432 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) 433 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; 434 435 switch (adev->asic_type) { 436 case CHIP_ARCTURUS: 437 if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 438 if (bo_adev == adev) 439 mapping_flags |= coherent ? 440 AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; 441 else 442 mapping_flags |= coherent ? 443 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 444 } else { 445 mapping_flags |= coherent ? 446 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 447 } 448 break; 449 case CHIP_ALDEBARAN: 450 if (coherent && uncached) { 451 if (adev->gmc.xgmi.connected_to_cpu || 452 !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) 453 snoop = true; 454 mapping_flags |= AMDGPU_VM_MTYPE_UC; 455 } else if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 456 if (bo_adev == adev) { 457 mapping_flags |= coherent ? 458 AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; 459 if (adev->gmc.xgmi.connected_to_cpu) 460 snoop = true; 461 } else { 462 mapping_flags |= coherent ? 463 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 464 if (amdgpu_xgmi_same_hive(adev, bo_adev)) 465 snoop = true; 466 } 467 } else { 468 snoop = true; 469 mapping_flags |= coherent ? 470 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 471 } 472 break; 473 default: 474 mapping_flags |= coherent ? 475 AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; 476 } 477 478 pte_flags = amdgpu_gem_va_map_flags(adev, mapping_flags); 479 pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; 480 481 return pte_flags; 482 } 483 484 static int 485 kfd_mem_dmamap_userptr(struct kgd_mem *mem, 486 struct kfd_mem_attachment *attachment) 487 { 488 enum dma_data_direction direction = 489 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 490 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 491 struct ttm_operation_ctx ctx = {.interruptible = true}; 492 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 493 struct amdgpu_device *adev = attachment->adev; 494 struct ttm_tt *src_ttm = mem->bo->tbo.ttm; 495 struct ttm_tt *ttm = bo->tbo.ttm; 496 int ret; 497 498 ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL); 499 if (unlikely(!ttm->sg)) 500 return -ENOMEM; 501 502 if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) 503 return -EINVAL; 504 505 /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ 506 ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, 507 ttm->num_pages, 0, 508 (u64)ttm->num_pages << PAGE_SHIFT, 509 GFP_KERNEL); 510 if (unlikely(ret)) 511 goto free_sg; 512 513 ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); 514 if (unlikely(ret)) 515 goto release_sg; 516 517 drm_prime_sg_to_dma_addr_array(ttm->sg, ttm->dma_address, 518 ttm->num_pages); 519 520 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 521 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 522 if (ret) 523 goto unmap_sg; 524 525 return 0; 526 527 unmap_sg: 528 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 529 release_sg: 530 pr_err("DMA map userptr failed: %d\n", ret); 531 sg_free_table(ttm->sg); 532 free_sg: 533 kfree(ttm->sg); 534 ttm->sg = NULL; 535 return ret; 536 } 537 538 static int 539 kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment) 540 { 541 struct ttm_operation_ctx ctx = {.interruptible = true}; 542 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 543 544 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 545 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 546 } 547 548 static int 549 kfd_mem_dmamap_attachment(struct kgd_mem *mem, 550 struct kfd_mem_attachment *attachment) 551 { 552 switch (attachment->type) { 553 case KFD_MEM_ATT_SHARED: 554 return 0; 555 case KFD_MEM_ATT_USERPTR: 556 return kfd_mem_dmamap_userptr(mem, attachment); 557 case KFD_MEM_ATT_DMABUF: 558 return kfd_mem_dmamap_dmabuf(attachment); 559 default: 560 WARN_ON_ONCE(1); 561 } 562 return -EINVAL; 563 } 564 565 static void 566 kfd_mem_dmaunmap_userptr(struct kgd_mem *mem, 567 struct kfd_mem_attachment *attachment) 568 { 569 enum dma_data_direction direction = 570 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 571 DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 572 struct ttm_operation_ctx ctx = {.interruptible = false}; 573 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 574 struct amdgpu_device *adev = attachment->adev; 575 struct ttm_tt *ttm = bo->tbo.ttm; 576 577 if (unlikely(!ttm->sg)) 578 return; 579 580 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 581 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 582 583 dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); 584 sg_free_table(ttm->sg); 585 kfree(ttm->sg); 586 ttm->sg = NULL; 587 } 588 589 static void 590 kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment) 591 { 592 struct ttm_operation_ctx ctx = {.interruptible = true}; 593 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 594 595 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 596 ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 597 } 598 599 static void 600 kfd_mem_dmaunmap_attachment(struct kgd_mem *mem, 601 struct kfd_mem_attachment *attachment) 602 { 603 switch (attachment->type) { 604 case KFD_MEM_ATT_SHARED: 605 break; 606 case KFD_MEM_ATT_USERPTR: 607 kfd_mem_dmaunmap_userptr(mem, attachment); 608 break; 609 case KFD_MEM_ATT_DMABUF: 610 kfd_mem_dmaunmap_dmabuf(attachment); 611 break; 612 default: 613 WARN_ON_ONCE(1); 614 } 615 } 616 617 static int 618 kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem, 619 struct amdgpu_bo **bo) 620 { 621 unsigned long bo_size = mem->bo->tbo.base.size; 622 struct drm_gem_object *gobj; 623 int ret; 624 625 ret = amdgpu_bo_reserve(mem->bo, false); 626 if (ret) 627 return ret; 628 629 ret = amdgpu_gem_object_create(adev, bo_size, 1, 630 AMDGPU_GEM_DOMAIN_CPU, 631 AMDGPU_GEM_CREATE_PREEMPTIBLE, 632 ttm_bo_type_sg, mem->bo->tbo.base.resv, 633 &gobj); 634 amdgpu_bo_unreserve(mem->bo); 635 if (ret) 636 return ret; 637 638 *bo = gem_to_amdgpu_bo(gobj); 639 (*bo)->parent = amdgpu_bo_ref(mem->bo); 640 641 return 0; 642 } 643 644 static int 645 kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem, 646 struct amdgpu_bo **bo) 647 { 648 struct drm_gem_object *gobj; 649 int ret; 650 651 if (!mem->dmabuf) { 652 mem->dmabuf = amdgpu_gem_prime_export(&mem->bo->tbo.base, 653 mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? 654 DRM_RDWR : 0); 655 if (IS_ERR(mem->dmabuf)) { 656 ret = PTR_ERR(mem->dmabuf); 657 mem->dmabuf = NULL; 658 return ret; 659 } 660 } 661 662 gobj = amdgpu_gem_prime_import(adev_to_drm(adev), mem->dmabuf); 663 if (IS_ERR(gobj)) 664 return PTR_ERR(gobj); 665 666 /* Import takes an extra reference on the dmabuf. Drop it now to 667 * avoid leaking it. We only need the one reference in 668 * kgd_mem->dmabuf. 669 */ 670 dma_buf_put(mem->dmabuf); 671 672 *bo = gem_to_amdgpu_bo(gobj); 673 (*bo)->flags |= AMDGPU_GEM_CREATE_PREEMPTIBLE; 674 (*bo)->parent = amdgpu_bo_ref(mem->bo); 675 676 return 0; 677 } 678 679 /* kfd_mem_attach - Add a BO to a VM 680 * 681 * Everything that needs to bo done only once when a BO is first added 682 * to a VM. It can later be mapped and unmapped many times without 683 * repeating these steps. 684 * 685 * 0. Create BO for DMA mapping, if needed 686 * 1. Allocate and initialize BO VA entry data structure 687 * 2. Add BO to the VM 688 * 3. Determine ASIC-specific PTE flags 689 * 4. Alloc page tables and directories if needed 690 * 4a. Validate new page tables and directories 691 */ 692 static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem, 693 struct amdgpu_vm *vm, bool is_aql) 694 { 695 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); 696 unsigned long bo_size = mem->bo->tbo.base.size; 697 uint64_t va = mem->va; 698 struct kfd_mem_attachment *attachment[2] = {NULL, NULL}; 699 struct amdgpu_bo *bo[2] = {NULL, NULL}; 700 int i, ret; 701 702 if (!va) { 703 pr_err("Invalid VA when adding BO to VM\n"); 704 return -EINVAL; 705 } 706 707 for (i = 0; i <= is_aql; i++) { 708 attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL); 709 if (unlikely(!attachment[i])) { 710 ret = -ENOMEM; 711 goto unwind; 712 } 713 714 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va, 715 va + bo_size, vm); 716 717 if (adev == bo_adev || (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && 718 amdgpu_xgmi_same_hive(adev, bo_adev))) { 719 /* Mappings on the local GPU and VRAM mappings in the 720 * local hive share the original BO 721 */ 722 attachment[i]->type = KFD_MEM_ATT_SHARED; 723 bo[i] = mem->bo; 724 drm_gem_object_get(&bo[i]->tbo.base); 725 } else if (i > 0) { 726 /* Multiple mappings on the same GPU share the BO */ 727 attachment[i]->type = KFD_MEM_ATT_SHARED; 728 bo[i] = bo[0]; 729 drm_gem_object_get(&bo[i]->tbo.base); 730 } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) { 731 /* Create an SG BO to DMA-map userptrs on other GPUs */ 732 attachment[i]->type = KFD_MEM_ATT_USERPTR; 733 ret = kfd_mem_attach_userptr(adev, mem, &bo[i]); 734 if (ret) 735 goto unwind; 736 } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT && 737 mem->bo->tbo.type != ttm_bo_type_sg) { 738 /* GTT BOs use DMA-mapping ability of dynamic-attach 739 * DMA bufs. TODO: The same should work for VRAM on 740 * large-BAR GPUs. 741 */ 742 attachment[i]->type = KFD_MEM_ATT_DMABUF; 743 ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]); 744 if (ret) 745 goto unwind; 746 } else { 747 /* FIXME: Need to DMA-map other BO types: 748 * large-BAR VRAM, doorbells, MMIO remap 749 */ 750 attachment[i]->type = KFD_MEM_ATT_SHARED; 751 bo[i] = mem->bo; 752 drm_gem_object_get(&bo[i]->tbo.base); 753 } 754 755 /* Add BO to VM internal data structures */ 756 ret = amdgpu_bo_reserve(bo[i], false); 757 if (ret) { 758 pr_debug("Unable to reserve BO during memory attach"); 759 goto unwind; 760 } 761 attachment[i]->bo_va = amdgpu_vm_bo_add(adev, vm, bo[i]); 762 amdgpu_bo_unreserve(bo[i]); 763 if (unlikely(!attachment[i]->bo_va)) { 764 ret = -ENOMEM; 765 pr_err("Failed to add BO object to VM. ret == %d\n", 766 ret); 767 goto unwind; 768 } 769 attachment[i]->va = va; 770 attachment[i]->pte_flags = get_pte_flags(adev, mem); 771 attachment[i]->adev = adev; 772 list_add(&attachment[i]->list, &mem->attachments); 773 774 va += bo_size; 775 } 776 777 return 0; 778 779 unwind: 780 for (; i >= 0; i--) { 781 if (!attachment[i]) 782 continue; 783 if (attachment[i]->bo_va) { 784 amdgpu_bo_reserve(bo[i], true); 785 amdgpu_vm_bo_rmv(adev, attachment[i]->bo_va); 786 amdgpu_bo_unreserve(bo[i]); 787 list_del(&attachment[i]->list); 788 } 789 if (bo[i]) 790 drm_gem_object_put(&bo[i]->tbo.base); 791 kfree(attachment[i]); 792 } 793 return ret; 794 } 795 796 static void kfd_mem_detach(struct kfd_mem_attachment *attachment) 797 { 798 struct amdgpu_bo *bo = attachment->bo_va->base.bo; 799 800 pr_debug("\t remove VA 0x%llx in entry %p\n", 801 attachment->va, attachment); 802 amdgpu_vm_bo_rmv(attachment->adev, attachment->bo_va); 803 drm_gem_object_put(&bo->tbo.base); 804 list_del(&attachment->list); 805 kfree(attachment); 806 } 807 808 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem, 809 struct amdkfd_process_info *process_info, 810 bool userptr) 811 { 812 struct ttm_validate_buffer *entry = &mem->validate_list; 813 struct amdgpu_bo *bo = mem->bo; 814 815 INIT_LIST_HEAD(&entry->head); 816 entry->num_shared = 1; 817 entry->bo = &bo->tbo; 818 mutex_lock(&process_info->lock); 819 if (userptr) 820 list_add_tail(&entry->head, &process_info->userptr_valid_list); 821 else 822 list_add_tail(&entry->head, &process_info->kfd_bo_list); 823 mutex_unlock(&process_info->lock); 824 } 825 826 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem, 827 struct amdkfd_process_info *process_info) 828 { 829 struct ttm_validate_buffer *bo_list_entry; 830 831 bo_list_entry = &mem->validate_list; 832 mutex_lock(&process_info->lock); 833 list_del(&bo_list_entry->head); 834 mutex_unlock(&process_info->lock); 835 } 836 837 /* Initializes user pages. It registers the MMU notifier and validates 838 * the userptr BO in the GTT domain. 839 * 840 * The BO must already be on the userptr_valid_list. Otherwise an 841 * eviction and restore may happen that leaves the new BO unmapped 842 * with the user mode queues running. 843 * 844 * Takes the process_info->lock to protect against concurrent restore 845 * workers. 846 * 847 * Returns 0 for success, negative errno for errors. 848 */ 849 static int init_user_pages(struct kgd_mem *mem, uint64_t user_addr) 850 { 851 struct amdkfd_process_info *process_info = mem->process_info; 852 struct amdgpu_bo *bo = mem->bo; 853 struct ttm_operation_ctx ctx = { true, false }; 854 int ret = 0; 855 856 mutex_lock(&process_info->lock); 857 858 ret = amdgpu_ttm_tt_set_userptr(&bo->tbo, user_addr, 0); 859 if (ret) { 860 pr_err("%s: Failed to set userptr: %d\n", __func__, ret); 861 goto out; 862 } 863 864 ret = amdgpu_mn_register(bo, user_addr); 865 if (ret) { 866 pr_err("%s: Failed to register MMU notifier: %d\n", 867 __func__, ret); 868 goto out; 869 } 870 871 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); 872 if (ret) { 873 pr_err("%s: Failed to get user pages: %d\n", __func__, ret); 874 goto unregister_out; 875 } 876 877 ret = amdgpu_bo_reserve(bo, true); 878 if (ret) { 879 pr_err("%s: Failed to reserve BO\n", __func__); 880 goto release_out; 881 } 882 amdgpu_bo_placement_from_domain(bo, mem->domain); 883 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 884 if (ret) 885 pr_err("%s: failed to validate BO\n", __func__); 886 amdgpu_bo_unreserve(bo); 887 888 release_out: 889 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); 890 unregister_out: 891 if (ret) 892 amdgpu_mn_unregister(bo); 893 out: 894 mutex_unlock(&process_info->lock); 895 return ret; 896 } 897 898 /* Reserving a BO and its page table BOs must happen atomically to 899 * avoid deadlocks. Some operations update multiple VMs at once. Track 900 * all the reservation info in a context structure. Optionally a sync 901 * object can track VM updates. 902 */ 903 struct bo_vm_reservation_context { 904 struct amdgpu_bo_list_entry kfd_bo; /* BO list entry for the KFD BO */ 905 unsigned int n_vms; /* Number of VMs reserved */ 906 struct amdgpu_bo_list_entry *vm_pd; /* Array of VM BO list entries */ 907 struct ww_acquire_ctx ticket; /* Reservation ticket */ 908 struct list_head list, duplicates; /* BO lists */ 909 struct amdgpu_sync *sync; /* Pointer to sync object */ 910 bool reserved; /* Whether BOs are reserved */ 911 }; 912 913 enum bo_vm_match { 914 BO_VM_NOT_MAPPED = 0, /* Match VMs where a BO is not mapped */ 915 BO_VM_MAPPED, /* Match VMs where a BO is mapped */ 916 BO_VM_ALL, /* Match all VMs a BO was added to */ 917 }; 918 919 /** 920 * reserve_bo_and_vm - reserve a BO and a VM unconditionally. 921 * @mem: KFD BO structure. 922 * @vm: the VM to reserve. 923 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 924 */ 925 static int reserve_bo_and_vm(struct kgd_mem *mem, 926 struct amdgpu_vm *vm, 927 struct bo_vm_reservation_context *ctx) 928 { 929 struct amdgpu_bo *bo = mem->bo; 930 int ret; 931 932 WARN_ON(!vm); 933 934 ctx->reserved = false; 935 ctx->n_vms = 1; 936 ctx->sync = &mem->sync; 937 938 INIT_LIST_HEAD(&ctx->list); 939 INIT_LIST_HEAD(&ctx->duplicates); 940 941 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), GFP_KERNEL); 942 if (!ctx->vm_pd) 943 return -ENOMEM; 944 945 ctx->kfd_bo.priority = 0; 946 ctx->kfd_bo.tv.bo = &bo->tbo; 947 ctx->kfd_bo.tv.num_shared = 1; 948 list_add(&ctx->kfd_bo.tv.head, &ctx->list); 949 950 amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]); 951 952 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list, 953 false, &ctx->duplicates); 954 if (ret) { 955 pr_err("Failed to reserve buffers in ttm.\n"); 956 kfree(ctx->vm_pd); 957 ctx->vm_pd = NULL; 958 return ret; 959 } 960 961 ctx->reserved = true; 962 return 0; 963 } 964 965 /** 966 * reserve_bo_and_cond_vms - reserve a BO and some VMs conditionally 967 * @mem: KFD BO structure. 968 * @vm: the VM to reserve. If NULL, then all VMs associated with the BO 969 * is used. Otherwise, a single VM associated with the BO. 970 * @map_type: the mapping status that will be used to filter the VMs. 971 * @ctx: the struct that will be used in unreserve_bo_and_vms(). 972 * 973 * Returns 0 for success, negative for failure. 974 */ 975 static int reserve_bo_and_cond_vms(struct kgd_mem *mem, 976 struct amdgpu_vm *vm, enum bo_vm_match map_type, 977 struct bo_vm_reservation_context *ctx) 978 { 979 struct amdgpu_bo *bo = mem->bo; 980 struct kfd_mem_attachment *entry; 981 unsigned int i; 982 int ret; 983 984 ctx->reserved = false; 985 ctx->n_vms = 0; 986 ctx->vm_pd = NULL; 987 ctx->sync = &mem->sync; 988 989 INIT_LIST_HEAD(&ctx->list); 990 INIT_LIST_HEAD(&ctx->duplicates); 991 992 list_for_each_entry(entry, &mem->attachments, list) { 993 if ((vm && vm != entry->bo_va->base.vm) || 994 (entry->is_mapped != map_type 995 && map_type != BO_VM_ALL)) 996 continue; 997 998 ctx->n_vms++; 999 } 1000 1001 if (ctx->n_vms != 0) { 1002 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), 1003 GFP_KERNEL); 1004 if (!ctx->vm_pd) 1005 return -ENOMEM; 1006 } 1007 1008 ctx->kfd_bo.priority = 0; 1009 ctx->kfd_bo.tv.bo = &bo->tbo; 1010 ctx->kfd_bo.tv.num_shared = 1; 1011 list_add(&ctx->kfd_bo.tv.head, &ctx->list); 1012 1013 i = 0; 1014 list_for_each_entry(entry, &mem->attachments, list) { 1015 if ((vm && vm != entry->bo_va->base.vm) || 1016 (entry->is_mapped != map_type 1017 && map_type != BO_VM_ALL)) 1018 continue; 1019 1020 amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list, 1021 &ctx->vm_pd[i]); 1022 i++; 1023 } 1024 1025 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list, 1026 false, &ctx->duplicates); 1027 if (ret) { 1028 pr_err("Failed to reserve buffers in ttm.\n"); 1029 kfree(ctx->vm_pd); 1030 ctx->vm_pd = NULL; 1031 return ret; 1032 } 1033 1034 ctx->reserved = true; 1035 return 0; 1036 } 1037 1038 /** 1039 * unreserve_bo_and_vms - Unreserve BO and VMs from a reservation context 1040 * @ctx: Reservation context to unreserve 1041 * @wait: Optionally wait for a sync object representing pending VM updates 1042 * @intr: Whether the wait is interruptible 1043 * 1044 * Also frees any resources allocated in 1045 * reserve_bo_and_(cond_)vm(s). Returns the status from 1046 * amdgpu_sync_wait. 1047 */ 1048 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx, 1049 bool wait, bool intr) 1050 { 1051 int ret = 0; 1052 1053 if (wait) 1054 ret = amdgpu_sync_wait(ctx->sync, intr); 1055 1056 if (ctx->reserved) 1057 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list); 1058 kfree(ctx->vm_pd); 1059 1060 ctx->sync = NULL; 1061 1062 ctx->reserved = false; 1063 ctx->vm_pd = NULL; 1064 1065 return ret; 1066 } 1067 1068 static void unmap_bo_from_gpuvm(struct kgd_mem *mem, 1069 struct kfd_mem_attachment *entry, 1070 struct amdgpu_sync *sync) 1071 { 1072 struct amdgpu_bo_va *bo_va = entry->bo_va; 1073 struct amdgpu_device *adev = entry->adev; 1074 struct amdgpu_vm *vm = bo_va->base.vm; 1075 1076 amdgpu_vm_bo_unmap(adev, bo_va, entry->va); 1077 1078 amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update); 1079 1080 amdgpu_sync_fence(sync, bo_va->last_pt_update); 1081 1082 kfd_mem_dmaunmap_attachment(mem, entry); 1083 } 1084 1085 static int update_gpuvm_pte(struct kgd_mem *mem, 1086 struct kfd_mem_attachment *entry, 1087 struct amdgpu_sync *sync, 1088 bool *table_freed) 1089 { 1090 struct amdgpu_bo_va *bo_va = entry->bo_va; 1091 struct amdgpu_device *adev = entry->adev; 1092 int ret; 1093 1094 ret = kfd_mem_dmamap_attachment(mem, entry); 1095 if (ret) 1096 return ret; 1097 1098 /* Update the page tables */ 1099 ret = amdgpu_vm_bo_update(adev, bo_va, false, table_freed); 1100 if (ret) { 1101 pr_err("amdgpu_vm_bo_update failed\n"); 1102 return ret; 1103 } 1104 1105 return amdgpu_sync_fence(sync, bo_va->last_pt_update); 1106 } 1107 1108 static int map_bo_to_gpuvm(struct kgd_mem *mem, 1109 struct kfd_mem_attachment *entry, 1110 struct amdgpu_sync *sync, 1111 bool no_update_pte, 1112 bool *table_freed) 1113 { 1114 int ret; 1115 1116 /* Set virtual address for the allocation */ 1117 ret = amdgpu_vm_bo_map(entry->adev, entry->bo_va, entry->va, 0, 1118 amdgpu_bo_size(entry->bo_va->base.bo), 1119 entry->pte_flags); 1120 if (ret) { 1121 pr_err("Failed to map VA 0x%llx in vm. ret %d\n", 1122 entry->va, ret); 1123 return ret; 1124 } 1125 1126 if (no_update_pte) 1127 return 0; 1128 1129 ret = update_gpuvm_pte(mem, entry, sync, table_freed); 1130 if (ret) { 1131 pr_err("update_gpuvm_pte() failed\n"); 1132 goto update_gpuvm_pte_failed; 1133 } 1134 1135 return 0; 1136 1137 update_gpuvm_pte_failed: 1138 unmap_bo_from_gpuvm(mem, entry, sync); 1139 return ret; 1140 } 1141 1142 static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size) 1143 { 1144 struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL); 1145 1146 if (!sg) 1147 return NULL; 1148 if (sg_alloc_table(sg, 1, GFP_KERNEL)) { 1149 kfree(sg); 1150 return NULL; 1151 } 1152 sg->sgl->dma_address = addr; 1153 sg->sgl->length = size; 1154 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1155 sg->sgl->dma_length = size; 1156 #endif 1157 return sg; 1158 } 1159 1160 static int process_validate_vms(struct amdkfd_process_info *process_info) 1161 { 1162 struct amdgpu_vm *peer_vm; 1163 int ret; 1164 1165 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1166 vm_list_node) { 1167 ret = vm_validate_pt_pd_bos(peer_vm); 1168 if (ret) 1169 return ret; 1170 } 1171 1172 return 0; 1173 } 1174 1175 static int process_sync_pds_resv(struct amdkfd_process_info *process_info, 1176 struct amdgpu_sync *sync) 1177 { 1178 struct amdgpu_vm *peer_vm; 1179 int ret; 1180 1181 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1182 vm_list_node) { 1183 struct amdgpu_bo *pd = peer_vm->root.bo; 1184 1185 ret = amdgpu_sync_resv(NULL, sync, pd->tbo.base.resv, 1186 AMDGPU_SYNC_NE_OWNER, 1187 AMDGPU_FENCE_OWNER_KFD); 1188 if (ret) 1189 return ret; 1190 } 1191 1192 return 0; 1193 } 1194 1195 static int process_update_pds(struct amdkfd_process_info *process_info, 1196 struct amdgpu_sync *sync) 1197 { 1198 struct amdgpu_vm *peer_vm; 1199 int ret; 1200 1201 list_for_each_entry(peer_vm, &process_info->vm_list_head, 1202 vm_list_node) { 1203 ret = vm_update_pds(peer_vm, sync); 1204 if (ret) 1205 return ret; 1206 } 1207 1208 return 0; 1209 } 1210 1211 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info, 1212 struct dma_fence **ef) 1213 { 1214 struct amdkfd_process_info *info = NULL; 1215 int ret; 1216 1217 if (!*process_info) { 1218 info = kzalloc(sizeof(*info), GFP_KERNEL); 1219 if (!info) 1220 return -ENOMEM; 1221 1222 mutex_init(&info->lock); 1223 INIT_LIST_HEAD(&info->vm_list_head); 1224 INIT_LIST_HEAD(&info->kfd_bo_list); 1225 INIT_LIST_HEAD(&info->userptr_valid_list); 1226 INIT_LIST_HEAD(&info->userptr_inval_list); 1227 1228 info->eviction_fence = 1229 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1), 1230 current->mm, 1231 NULL); 1232 if (!info->eviction_fence) { 1233 pr_err("Failed to create eviction fence\n"); 1234 ret = -ENOMEM; 1235 goto create_evict_fence_fail; 1236 } 1237 1238 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID); 1239 atomic_set(&info->evicted_bos, 0); 1240 INIT_DELAYED_WORK(&info->restore_userptr_work, 1241 amdgpu_amdkfd_restore_userptr_worker); 1242 1243 *process_info = info; 1244 *ef = dma_fence_get(&info->eviction_fence->base); 1245 } 1246 1247 vm->process_info = *process_info; 1248 1249 /* Validate page directory and attach eviction fence */ 1250 ret = amdgpu_bo_reserve(vm->root.bo, true); 1251 if (ret) 1252 goto reserve_pd_fail; 1253 ret = vm_validate_pt_pd_bos(vm); 1254 if (ret) { 1255 pr_err("validate_pt_pd_bos() failed\n"); 1256 goto validate_pd_fail; 1257 } 1258 ret = amdgpu_bo_sync_wait(vm->root.bo, 1259 AMDGPU_FENCE_OWNER_KFD, false); 1260 if (ret) 1261 goto wait_pd_fail; 1262 ret = dma_resv_reserve_shared(vm->root.bo->tbo.base.resv, 1); 1263 if (ret) 1264 goto reserve_shared_fail; 1265 amdgpu_bo_fence(vm->root.bo, 1266 &vm->process_info->eviction_fence->base, true); 1267 amdgpu_bo_unreserve(vm->root.bo); 1268 1269 /* Update process info */ 1270 mutex_lock(&vm->process_info->lock); 1271 list_add_tail(&vm->vm_list_node, 1272 &(vm->process_info->vm_list_head)); 1273 vm->process_info->n_vms++; 1274 mutex_unlock(&vm->process_info->lock); 1275 1276 return 0; 1277 1278 reserve_shared_fail: 1279 wait_pd_fail: 1280 validate_pd_fail: 1281 amdgpu_bo_unreserve(vm->root.bo); 1282 reserve_pd_fail: 1283 vm->process_info = NULL; 1284 if (info) { 1285 /* Two fence references: one in info and one in *ef */ 1286 dma_fence_put(&info->eviction_fence->base); 1287 dma_fence_put(*ef); 1288 *ef = NULL; 1289 *process_info = NULL; 1290 put_pid(info->pid); 1291 create_evict_fence_fail: 1292 mutex_destroy(&info->lock); 1293 kfree(info); 1294 } 1295 return ret; 1296 } 1297 1298 /** 1299 * amdgpu_amdkfd_gpuvm_pin_bo() - Pins a BO using following criteria 1300 * @bo: Handle of buffer object being pinned 1301 * @domain: Domain into which BO should be pinned 1302 * 1303 * - USERPTR BOs are UNPINNABLE and will return error 1304 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1305 * PIN count incremented. It is valid to PIN a BO multiple times 1306 * 1307 * Return: ZERO if successful in pinning, Non-Zero in case of error. 1308 */ 1309 static int amdgpu_amdkfd_gpuvm_pin_bo(struct amdgpu_bo *bo, u32 domain) 1310 { 1311 int ret = 0; 1312 1313 ret = amdgpu_bo_reserve(bo, false); 1314 if (unlikely(ret)) 1315 return ret; 1316 1317 ret = amdgpu_bo_pin_restricted(bo, domain, 0, 0); 1318 if (ret) 1319 pr_err("Error in Pinning BO to domain: %d\n", domain); 1320 1321 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); 1322 amdgpu_bo_unreserve(bo); 1323 1324 return ret; 1325 } 1326 1327 /** 1328 * amdgpu_amdkfd_gpuvm_unpin_bo() - Unpins BO using following criteria 1329 * @bo: Handle of buffer object being unpinned 1330 * 1331 * - Is a illegal request for USERPTR BOs and is ignored 1332 * - All other BO types (GTT, VRAM, MMIO and DOORBELL) will have their 1333 * PIN count decremented. Calls to UNPIN must balance calls to PIN 1334 */ 1335 void amdgpu_amdkfd_gpuvm_unpin_bo(struct amdgpu_bo *bo) 1336 { 1337 int ret = 0; 1338 1339 ret = amdgpu_bo_reserve(bo, false); 1340 if (unlikely(ret)) 1341 return; 1342 1343 amdgpu_bo_unpin(bo); 1344 amdgpu_bo_unreserve(bo); 1345 } 1346 1347 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct amdgpu_device *adev, 1348 struct file *filp, u32 pasid, 1349 void **process_info, 1350 struct dma_fence **ef) 1351 { 1352 struct amdgpu_fpriv *drv_priv; 1353 struct amdgpu_vm *avm; 1354 int ret; 1355 1356 ret = amdgpu_file_to_fpriv(filp, &drv_priv); 1357 if (ret) 1358 return ret; 1359 avm = &drv_priv->vm; 1360 1361 /* Already a compute VM? */ 1362 if (avm->process_info) 1363 return -EINVAL; 1364 1365 /* Free the original amdgpu allocated pasid, 1366 * will be replaced with kfd allocated pasid. 1367 */ 1368 if (avm->pasid) { 1369 amdgpu_pasid_free(avm->pasid); 1370 amdgpu_vm_set_pasid(adev, avm, 0); 1371 } 1372 1373 /* Convert VM into a compute VM */ 1374 ret = amdgpu_vm_make_compute(adev, avm); 1375 if (ret) 1376 return ret; 1377 1378 ret = amdgpu_vm_set_pasid(adev, avm, pasid); 1379 if (ret) 1380 return ret; 1381 /* Initialize KFD part of the VM and process info */ 1382 ret = init_kfd_vm(avm, process_info, ef); 1383 if (ret) 1384 return ret; 1385 1386 amdgpu_vm_set_task_info(avm); 1387 1388 return 0; 1389 } 1390 1391 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, 1392 struct amdgpu_vm *vm) 1393 { 1394 struct amdkfd_process_info *process_info = vm->process_info; 1395 struct amdgpu_bo *pd = vm->root.bo; 1396 1397 if (!process_info) 1398 return; 1399 1400 /* Release eviction fence from PD */ 1401 amdgpu_bo_reserve(pd, false); 1402 amdgpu_bo_fence(pd, NULL, false); 1403 amdgpu_bo_unreserve(pd); 1404 1405 /* Update process info */ 1406 mutex_lock(&process_info->lock); 1407 process_info->n_vms--; 1408 list_del(&vm->vm_list_node); 1409 mutex_unlock(&process_info->lock); 1410 1411 vm->process_info = NULL; 1412 1413 /* Release per-process resources when last compute VM is destroyed */ 1414 if (!process_info->n_vms) { 1415 WARN_ON(!list_empty(&process_info->kfd_bo_list)); 1416 WARN_ON(!list_empty(&process_info->userptr_valid_list)); 1417 WARN_ON(!list_empty(&process_info->userptr_inval_list)); 1418 1419 dma_fence_put(&process_info->eviction_fence->base); 1420 cancel_delayed_work_sync(&process_info->restore_userptr_work); 1421 put_pid(process_info->pid); 1422 mutex_destroy(&process_info->lock); 1423 kfree(process_info); 1424 } 1425 } 1426 1427 void amdgpu_amdkfd_gpuvm_release_process_vm(struct amdgpu_device *adev, 1428 void *drm_priv) 1429 { 1430 struct amdgpu_vm *avm; 1431 1432 if (WARN_ON(!adev || !drm_priv)) 1433 return; 1434 1435 avm = drm_priv_to_vm(drm_priv); 1436 1437 pr_debug("Releasing process vm %p\n", avm); 1438 1439 /* The original pasid of amdgpu vm has already been 1440 * released during making a amdgpu vm to a compute vm 1441 * The current pasid is managed by kfd and will be 1442 * released on kfd process destroy. Set amdgpu pasid 1443 * to 0 to avoid duplicate release. 1444 */ 1445 amdgpu_vm_release_compute(adev, avm); 1446 } 1447 1448 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv) 1449 { 1450 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1451 struct amdgpu_bo *pd = avm->root.bo; 1452 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); 1453 1454 if (adev->asic_type < CHIP_VEGA10) 1455 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT; 1456 return avm->pd_phys_addr; 1457 } 1458 1459 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( 1460 struct amdgpu_device *adev, uint64_t va, uint64_t size, 1461 void *drm_priv, struct kgd_mem **mem, 1462 uint64_t *offset, uint32_t flags) 1463 { 1464 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1465 enum ttm_bo_type bo_type = ttm_bo_type_device; 1466 struct sg_table *sg = NULL; 1467 uint64_t user_addr = 0; 1468 struct amdgpu_bo *bo; 1469 struct drm_gem_object *gobj; 1470 u32 domain, alloc_domain; 1471 u64 alloc_flags; 1472 int ret; 1473 1474 /* 1475 * Check on which domain to allocate BO 1476 */ 1477 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { 1478 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM; 1479 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE; 1480 alloc_flags |= (flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) ? 1481 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED : 0; 1482 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { 1483 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT; 1484 alloc_flags = 0; 1485 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { 1486 domain = AMDGPU_GEM_DOMAIN_GTT; 1487 alloc_domain = AMDGPU_GEM_DOMAIN_CPU; 1488 alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE; 1489 if (!offset || !*offset) 1490 return -EINVAL; 1491 user_addr = untagged_addr(*offset); 1492 } else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1493 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1494 domain = AMDGPU_GEM_DOMAIN_GTT; 1495 alloc_domain = AMDGPU_GEM_DOMAIN_CPU; 1496 bo_type = ttm_bo_type_sg; 1497 alloc_flags = 0; 1498 if (size > UINT_MAX) 1499 return -EINVAL; 1500 sg = create_doorbell_sg(*offset, size); 1501 if (!sg) 1502 return -ENOMEM; 1503 } else { 1504 return -EINVAL; 1505 } 1506 1507 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); 1508 if (!*mem) { 1509 ret = -ENOMEM; 1510 goto err; 1511 } 1512 INIT_LIST_HEAD(&(*mem)->attachments); 1513 mutex_init(&(*mem)->lock); 1514 (*mem)->aql_queue = !!(flags & KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM); 1515 1516 /* Workaround for AQL queue wraparound bug. Map the same 1517 * memory twice. That means we only actually allocate half 1518 * the memory. 1519 */ 1520 if ((*mem)->aql_queue) 1521 size = size >> 1; 1522 1523 (*mem)->alloc_flags = flags; 1524 1525 amdgpu_sync_create(&(*mem)->sync); 1526 1527 ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, flags); 1528 if (ret) { 1529 pr_debug("Insufficient memory\n"); 1530 goto err_reserve_limit; 1531 } 1532 1533 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n", 1534 va, size, domain_string(alloc_domain)); 1535 1536 ret = amdgpu_gem_object_create(adev, size, 1, alloc_domain, alloc_flags, 1537 bo_type, NULL, &gobj); 1538 if (ret) { 1539 pr_debug("Failed to create BO on domain %s. ret %d\n", 1540 domain_string(alloc_domain), ret); 1541 goto err_bo_create; 1542 } 1543 ret = drm_vma_node_allow(&gobj->vma_node, drm_priv); 1544 if (ret) { 1545 pr_debug("Failed to allow vma node access. ret %d\n", ret); 1546 goto err_node_allow; 1547 } 1548 bo = gem_to_amdgpu_bo(gobj); 1549 if (bo_type == ttm_bo_type_sg) { 1550 bo->tbo.sg = sg; 1551 bo->tbo.ttm->sg = sg; 1552 } 1553 bo->kfd_bo = *mem; 1554 (*mem)->bo = bo; 1555 if (user_addr) 1556 bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO; 1557 1558 (*mem)->va = va; 1559 (*mem)->domain = domain; 1560 (*mem)->mapped_to_gpu_memory = 0; 1561 (*mem)->process_info = avm->process_info; 1562 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr); 1563 1564 if (user_addr) { 1565 ret = init_user_pages(*mem, user_addr); 1566 if (ret) 1567 goto allocate_init_user_pages_failed; 1568 } 1569 1570 if (offset) 1571 *offset = amdgpu_bo_mmap_offset(bo); 1572 1573 if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1574 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1575 ret = amdgpu_amdkfd_gpuvm_pin_bo(bo, AMDGPU_GEM_DOMAIN_GTT); 1576 if (ret) { 1577 pr_err("Pinning MMIO/DOORBELL BO during ALLOC FAILED\n"); 1578 goto err_pin_bo; 1579 } 1580 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 1581 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 1582 } 1583 1584 return 0; 1585 1586 allocate_init_user_pages_failed: 1587 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info); 1588 err_pin_bo: 1589 drm_vma_node_revoke(&gobj->vma_node, drm_priv); 1590 err_node_allow: 1591 drm_gem_object_put(gobj); 1592 /* Don't unreserve system mem limit twice */ 1593 goto err_reserve_limit; 1594 err_bo_create: 1595 unreserve_mem_limit(adev, size, flags); 1596 err_reserve_limit: 1597 mutex_destroy(&(*mem)->lock); 1598 kfree(*mem); 1599 err: 1600 if (sg) { 1601 sg_free_table(sg); 1602 kfree(sg); 1603 } 1604 return ret; 1605 } 1606 1607 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( 1608 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv, 1609 uint64_t *size) 1610 { 1611 struct amdkfd_process_info *process_info = mem->process_info; 1612 unsigned long bo_size = mem->bo->tbo.base.size; 1613 struct kfd_mem_attachment *entry, *tmp; 1614 struct bo_vm_reservation_context ctx; 1615 struct ttm_validate_buffer *bo_list_entry; 1616 unsigned int mapped_to_gpu_memory; 1617 int ret; 1618 bool is_imported = false; 1619 1620 mutex_lock(&mem->lock); 1621 1622 /* Unpin MMIO/DOORBELL BO's that were pinnned during allocation */ 1623 if (mem->alloc_flags & 1624 (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | 1625 KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) { 1626 amdgpu_amdkfd_gpuvm_unpin_bo(mem->bo); 1627 } 1628 1629 mapped_to_gpu_memory = mem->mapped_to_gpu_memory; 1630 is_imported = mem->is_imported; 1631 mutex_unlock(&mem->lock); 1632 /* lock is not needed after this, since mem is unused and will 1633 * be freed anyway 1634 */ 1635 1636 if (mapped_to_gpu_memory > 0) { 1637 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n", 1638 mem->va, bo_size); 1639 return -EBUSY; 1640 } 1641 1642 /* Make sure restore workers don't access the BO any more */ 1643 bo_list_entry = &mem->validate_list; 1644 mutex_lock(&process_info->lock); 1645 list_del(&bo_list_entry->head); 1646 mutex_unlock(&process_info->lock); 1647 1648 /* No more MMU notifiers */ 1649 amdgpu_mn_unregister(mem->bo); 1650 1651 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx); 1652 if (unlikely(ret)) 1653 return ret; 1654 1655 /* The eviction fence should be removed by the last unmap. 1656 * TODO: Log an error condition if the bo still has the eviction fence 1657 * attached 1658 */ 1659 amdgpu_amdkfd_remove_eviction_fence(mem->bo, 1660 process_info->eviction_fence); 1661 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va, 1662 mem->va + bo_size * (1 + mem->aql_queue)); 1663 1664 /* Remove from VM internal data structures */ 1665 list_for_each_entry_safe(entry, tmp, &mem->attachments, list) 1666 kfd_mem_detach(entry); 1667 1668 ret = unreserve_bo_and_vms(&ctx, false, false); 1669 1670 /* Free the sync object */ 1671 amdgpu_sync_free(&mem->sync); 1672 1673 /* If the SG is not NULL, it's one we created for a doorbell or mmio 1674 * remap BO. We need to free it. 1675 */ 1676 if (mem->bo->tbo.sg) { 1677 sg_free_table(mem->bo->tbo.sg); 1678 kfree(mem->bo->tbo.sg); 1679 } 1680 1681 /* Update the size of the BO being freed if it was allocated from 1682 * VRAM and is not imported. 1683 */ 1684 if (size) { 1685 if ((mem->bo->preferred_domains == AMDGPU_GEM_DOMAIN_VRAM) && 1686 (!is_imported)) 1687 *size = bo_size; 1688 else 1689 *size = 0; 1690 } 1691 1692 /* Free the BO*/ 1693 drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv); 1694 if (mem->dmabuf) 1695 dma_buf_put(mem->dmabuf); 1696 mutex_destroy(&mem->lock); 1697 1698 /* If this releases the last reference, it will end up calling 1699 * amdgpu_amdkfd_release_notify and kfree the mem struct. That's why 1700 * this needs to be the last call here. 1701 */ 1702 drm_gem_object_put(&mem->bo->tbo.base); 1703 1704 return ret; 1705 } 1706 1707 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( 1708 struct amdgpu_device *adev, struct kgd_mem *mem, 1709 void *drm_priv, bool *table_freed) 1710 { 1711 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1712 int ret; 1713 struct amdgpu_bo *bo; 1714 uint32_t domain; 1715 struct kfd_mem_attachment *entry; 1716 struct bo_vm_reservation_context ctx; 1717 unsigned long bo_size; 1718 bool is_invalid_userptr = false; 1719 1720 bo = mem->bo; 1721 if (!bo) { 1722 pr_err("Invalid BO when mapping memory to GPU\n"); 1723 return -EINVAL; 1724 } 1725 1726 /* Make sure restore is not running concurrently. Since we 1727 * don't map invalid userptr BOs, we rely on the next restore 1728 * worker to do the mapping 1729 */ 1730 mutex_lock(&mem->process_info->lock); 1731 1732 /* Lock mmap-sem. If we find an invalid userptr BO, we can be 1733 * sure that the MMU notifier is no longer running 1734 * concurrently and the queues are actually stopped 1735 */ 1736 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 1737 mmap_write_lock(current->mm); 1738 is_invalid_userptr = atomic_read(&mem->invalid); 1739 mmap_write_unlock(current->mm); 1740 } 1741 1742 mutex_lock(&mem->lock); 1743 1744 domain = mem->domain; 1745 bo_size = bo->tbo.base.size; 1746 1747 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n", 1748 mem->va, 1749 mem->va + bo_size * (1 + mem->aql_queue), 1750 avm, domain_string(domain)); 1751 1752 if (!kfd_mem_is_attached(avm, mem)) { 1753 ret = kfd_mem_attach(adev, mem, avm, mem->aql_queue); 1754 if (ret) 1755 goto out; 1756 } 1757 1758 ret = reserve_bo_and_vm(mem, avm, &ctx); 1759 if (unlikely(ret)) 1760 goto out; 1761 1762 /* Userptr can be marked as "not invalid", but not actually be 1763 * validated yet (still in the system domain). In that case 1764 * the queues are still stopped and we can leave mapping for 1765 * the next restore worker 1766 */ 1767 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && 1768 bo->tbo.resource->mem_type == TTM_PL_SYSTEM) 1769 is_invalid_userptr = true; 1770 1771 ret = vm_validate_pt_pd_bos(avm); 1772 if (unlikely(ret)) 1773 goto out_unreserve; 1774 1775 if (mem->mapped_to_gpu_memory == 0 && 1776 !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 1777 /* Validate BO only once. The eviction fence gets added to BO 1778 * the first time it is mapped. Validate will wait for all 1779 * background evictions to complete. 1780 */ 1781 ret = amdgpu_amdkfd_bo_validate(bo, domain, true); 1782 if (ret) { 1783 pr_debug("Validate failed\n"); 1784 goto out_unreserve; 1785 } 1786 } 1787 1788 list_for_each_entry(entry, &mem->attachments, list) { 1789 if (entry->bo_va->base.vm != avm || entry->is_mapped) 1790 continue; 1791 1792 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n", 1793 entry->va, entry->va + bo_size, entry); 1794 1795 ret = map_bo_to_gpuvm(mem, entry, ctx.sync, 1796 is_invalid_userptr, table_freed); 1797 if (ret) { 1798 pr_err("Failed to map bo to gpuvm\n"); 1799 goto out_unreserve; 1800 } 1801 1802 ret = vm_update_pds(avm, ctx.sync); 1803 if (ret) { 1804 pr_err("Failed to update page directories\n"); 1805 goto out_unreserve; 1806 } 1807 1808 entry->is_mapped = true; 1809 mem->mapped_to_gpu_memory++; 1810 pr_debug("\t INC mapping count %d\n", 1811 mem->mapped_to_gpu_memory); 1812 } 1813 1814 if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count) 1815 amdgpu_bo_fence(bo, 1816 &avm->process_info->eviction_fence->base, 1817 true); 1818 ret = unreserve_bo_and_vms(&ctx, false, false); 1819 1820 /* Only apply no TLB flush on Aldebaran to 1821 * workaround regressions on other Asics. 1822 */ 1823 if (table_freed && (adev->asic_type != CHIP_ALDEBARAN)) 1824 *table_freed = true; 1825 1826 goto out; 1827 1828 out_unreserve: 1829 unreserve_bo_and_vms(&ctx, false, false); 1830 out: 1831 mutex_unlock(&mem->process_info->lock); 1832 mutex_unlock(&mem->lock); 1833 return ret; 1834 } 1835 1836 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( 1837 struct amdgpu_device *adev, struct kgd_mem *mem, void *drm_priv) 1838 { 1839 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1840 struct amdkfd_process_info *process_info = avm->process_info; 1841 unsigned long bo_size = mem->bo->tbo.base.size; 1842 struct kfd_mem_attachment *entry; 1843 struct bo_vm_reservation_context ctx; 1844 int ret; 1845 1846 mutex_lock(&mem->lock); 1847 1848 ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx); 1849 if (unlikely(ret)) 1850 goto out; 1851 /* If no VMs were reserved, it means the BO wasn't actually mapped */ 1852 if (ctx.n_vms == 0) { 1853 ret = -EINVAL; 1854 goto unreserve_out; 1855 } 1856 1857 ret = vm_validate_pt_pd_bos(avm); 1858 if (unlikely(ret)) 1859 goto unreserve_out; 1860 1861 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n", 1862 mem->va, 1863 mem->va + bo_size * (1 + mem->aql_queue), 1864 avm); 1865 1866 list_for_each_entry(entry, &mem->attachments, list) { 1867 if (entry->bo_va->base.vm != avm || !entry->is_mapped) 1868 continue; 1869 1870 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n", 1871 entry->va, entry->va + bo_size, entry); 1872 1873 unmap_bo_from_gpuvm(mem, entry, ctx.sync); 1874 entry->is_mapped = false; 1875 1876 mem->mapped_to_gpu_memory--; 1877 pr_debug("\t DEC mapping count %d\n", 1878 mem->mapped_to_gpu_memory); 1879 } 1880 1881 /* If BO is unmapped from all VMs, unfence it. It can be evicted if 1882 * required. 1883 */ 1884 if (mem->mapped_to_gpu_memory == 0 && 1885 !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && 1886 !mem->bo->tbo.pin_count) 1887 amdgpu_amdkfd_remove_eviction_fence(mem->bo, 1888 process_info->eviction_fence); 1889 1890 unreserve_out: 1891 unreserve_bo_and_vms(&ctx, false, false); 1892 out: 1893 mutex_unlock(&mem->lock); 1894 return ret; 1895 } 1896 1897 int amdgpu_amdkfd_gpuvm_sync_memory( 1898 struct amdgpu_device *adev, struct kgd_mem *mem, bool intr) 1899 { 1900 struct amdgpu_sync sync; 1901 int ret; 1902 1903 amdgpu_sync_create(&sync); 1904 1905 mutex_lock(&mem->lock); 1906 amdgpu_sync_clone(&mem->sync, &sync); 1907 mutex_unlock(&mem->lock); 1908 1909 ret = amdgpu_sync_wait(&sync, intr); 1910 amdgpu_sync_free(&sync); 1911 return ret; 1912 } 1913 1914 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct amdgpu_device *adev, 1915 struct kgd_mem *mem, void **kptr, uint64_t *size) 1916 { 1917 int ret; 1918 struct amdgpu_bo *bo = mem->bo; 1919 1920 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) { 1921 pr_err("userptr can't be mapped to kernel\n"); 1922 return -EINVAL; 1923 } 1924 1925 /* delete kgd_mem from kfd_bo_list to avoid re-validating 1926 * this BO in BO's restoring after eviction. 1927 */ 1928 mutex_lock(&mem->process_info->lock); 1929 1930 ret = amdgpu_bo_reserve(bo, true); 1931 if (ret) { 1932 pr_err("Failed to reserve bo. ret %d\n", ret); 1933 goto bo_reserve_failed; 1934 } 1935 1936 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); 1937 if (ret) { 1938 pr_err("Failed to pin bo. ret %d\n", ret); 1939 goto pin_failed; 1940 } 1941 1942 ret = amdgpu_bo_kmap(bo, kptr); 1943 if (ret) { 1944 pr_err("Failed to map bo to kernel. ret %d\n", ret); 1945 goto kmap_failed; 1946 } 1947 1948 amdgpu_amdkfd_remove_eviction_fence( 1949 bo, mem->process_info->eviction_fence); 1950 list_del_init(&mem->validate_list.head); 1951 1952 if (size) 1953 *size = amdgpu_bo_size(bo); 1954 1955 amdgpu_bo_unreserve(bo); 1956 1957 mutex_unlock(&mem->process_info->lock); 1958 return 0; 1959 1960 kmap_failed: 1961 amdgpu_bo_unpin(bo); 1962 pin_failed: 1963 amdgpu_bo_unreserve(bo); 1964 bo_reserve_failed: 1965 mutex_unlock(&mem->process_info->lock); 1966 1967 return ret; 1968 } 1969 1970 void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct amdgpu_device *adev, 1971 struct kgd_mem *mem) 1972 { 1973 struct amdgpu_bo *bo = mem->bo; 1974 1975 amdgpu_bo_reserve(bo, true); 1976 amdgpu_bo_kunmap(bo); 1977 amdgpu_bo_unpin(bo); 1978 amdgpu_bo_unreserve(bo); 1979 } 1980 1981 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev, 1982 struct kfd_vm_fault_info *mem) 1983 { 1984 if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) { 1985 *mem = *adev->gmc.vm_fault_info; 1986 mb(); 1987 atomic_set(&adev->gmc.vm_fault_info_updated, 0); 1988 } 1989 return 0; 1990 } 1991 1992 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct amdgpu_device *adev, 1993 struct dma_buf *dma_buf, 1994 uint64_t va, void *drm_priv, 1995 struct kgd_mem **mem, uint64_t *size, 1996 uint64_t *mmap_offset) 1997 { 1998 struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv); 1999 struct drm_gem_object *obj; 2000 struct amdgpu_bo *bo; 2001 int ret; 2002 2003 if (dma_buf->ops != &amdgpu_dmabuf_ops) 2004 /* Can't handle non-graphics buffers */ 2005 return -EINVAL; 2006 2007 obj = dma_buf->priv; 2008 if (drm_to_adev(obj->dev) != adev) 2009 /* Can't handle buffers from other devices */ 2010 return -EINVAL; 2011 2012 bo = gem_to_amdgpu_bo(obj); 2013 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM | 2014 AMDGPU_GEM_DOMAIN_GTT))) 2015 /* Only VRAM and GTT BOs are supported */ 2016 return -EINVAL; 2017 2018 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); 2019 if (!*mem) 2020 return -ENOMEM; 2021 2022 ret = drm_vma_node_allow(&obj->vma_node, drm_priv); 2023 if (ret) { 2024 kfree(mem); 2025 return ret; 2026 } 2027 2028 if (size) 2029 *size = amdgpu_bo_size(bo); 2030 2031 if (mmap_offset) 2032 *mmap_offset = amdgpu_bo_mmap_offset(bo); 2033 2034 INIT_LIST_HEAD(&(*mem)->attachments); 2035 mutex_init(&(*mem)->lock); 2036 2037 (*mem)->alloc_flags = 2038 ((bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? 2039 KFD_IOC_ALLOC_MEM_FLAGS_VRAM : KFD_IOC_ALLOC_MEM_FLAGS_GTT) 2040 | KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE 2041 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 2042 2043 drm_gem_object_get(&bo->tbo.base); 2044 (*mem)->bo = bo; 2045 (*mem)->va = va; 2046 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ? 2047 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT; 2048 (*mem)->mapped_to_gpu_memory = 0; 2049 (*mem)->process_info = avm->process_info; 2050 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false); 2051 amdgpu_sync_create(&(*mem)->sync); 2052 (*mem)->is_imported = true; 2053 2054 return 0; 2055 } 2056 2057 /* Evict a userptr BO by stopping the queues if necessary 2058 * 2059 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it 2060 * cannot do any memory allocations, and cannot take any locks that 2061 * are held elsewhere while allocating memory. Therefore this is as 2062 * simple as possible, using atomic counters. 2063 * 2064 * It doesn't do anything to the BO itself. The real work happens in 2065 * restore, where we get updated page addresses. This function only 2066 * ensures that GPU access to the BO is stopped. 2067 */ 2068 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, 2069 struct mm_struct *mm) 2070 { 2071 struct amdkfd_process_info *process_info = mem->process_info; 2072 int evicted_bos; 2073 int r = 0; 2074 2075 atomic_inc(&mem->invalid); 2076 evicted_bos = atomic_inc_return(&process_info->evicted_bos); 2077 if (evicted_bos == 1) { 2078 /* First eviction, stop the queues */ 2079 r = kgd2kfd_quiesce_mm(mm); 2080 if (r) 2081 pr_err("Failed to quiesce KFD\n"); 2082 schedule_delayed_work(&process_info->restore_userptr_work, 2083 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2084 } 2085 2086 return r; 2087 } 2088 2089 /* Update invalid userptr BOs 2090 * 2091 * Moves invalidated (evicted) userptr BOs from userptr_valid_list to 2092 * userptr_inval_list and updates user pages for all BOs that have 2093 * been invalidated since their last update. 2094 */ 2095 static int update_invalid_user_pages(struct amdkfd_process_info *process_info, 2096 struct mm_struct *mm) 2097 { 2098 struct kgd_mem *mem, *tmp_mem; 2099 struct amdgpu_bo *bo; 2100 struct ttm_operation_ctx ctx = { false, false }; 2101 int invalid, ret; 2102 2103 /* Move all invalidated BOs to the userptr_inval_list and 2104 * release their user pages by migration to the CPU domain 2105 */ 2106 list_for_each_entry_safe(mem, tmp_mem, 2107 &process_info->userptr_valid_list, 2108 validate_list.head) { 2109 if (!atomic_read(&mem->invalid)) 2110 continue; /* BO is still valid */ 2111 2112 bo = mem->bo; 2113 2114 if (amdgpu_bo_reserve(bo, true)) 2115 return -EAGAIN; 2116 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU); 2117 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2118 amdgpu_bo_unreserve(bo); 2119 if (ret) { 2120 pr_err("%s: Failed to invalidate userptr BO\n", 2121 __func__); 2122 return -EAGAIN; 2123 } 2124 2125 list_move_tail(&mem->validate_list.head, 2126 &process_info->userptr_inval_list); 2127 } 2128 2129 if (list_empty(&process_info->userptr_inval_list)) 2130 return 0; /* All evicted userptr BOs were freed */ 2131 2132 /* Go through userptr_inval_list and update any invalid user_pages */ 2133 list_for_each_entry(mem, &process_info->userptr_inval_list, 2134 validate_list.head) { 2135 invalid = atomic_read(&mem->invalid); 2136 if (!invalid) 2137 /* BO hasn't been invalidated since the last 2138 * revalidation attempt. Keep its BO list. 2139 */ 2140 continue; 2141 2142 bo = mem->bo; 2143 2144 /* Get updated user pages */ 2145 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); 2146 if (ret) { 2147 pr_debug("Failed %d to get user pages\n", ret); 2148 2149 /* Return -EFAULT bad address error as success. It will 2150 * fail later with a VM fault if the GPU tries to access 2151 * it. Better than hanging indefinitely with stalled 2152 * user mode queues. 2153 * 2154 * Return other error -EBUSY or -ENOMEM to retry restore 2155 */ 2156 if (ret != -EFAULT) 2157 return ret; 2158 } else { 2159 2160 /* 2161 * FIXME: Cannot ignore the return code, must hold 2162 * notifier_lock 2163 */ 2164 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); 2165 } 2166 2167 /* Mark the BO as valid unless it was invalidated 2168 * again concurrently. 2169 */ 2170 if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid) 2171 return -EAGAIN; 2172 } 2173 2174 return 0; 2175 } 2176 2177 /* Validate invalid userptr BOs 2178 * 2179 * Validates BOs on the userptr_inval_list, and moves them back to the 2180 * userptr_valid_list. Also updates GPUVM page tables with new page 2181 * addresses and waits for the page table updates to complete. 2182 */ 2183 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info) 2184 { 2185 struct amdgpu_bo_list_entry *pd_bo_list_entries; 2186 struct list_head resv_list, duplicates; 2187 struct ww_acquire_ctx ticket; 2188 struct amdgpu_sync sync; 2189 2190 struct amdgpu_vm *peer_vm; 2191 struct kgd_mem *mem, *tmp_mem; 2192 struct amdgpu_bo *bo; 2193 struct ttm_operation_ctx ctx = { false, false }; 2194 int i, ret; 2195 2196 pd_bo_list_entries = kcalloc(process_info->n_vms, 2197 sizeof(struct amdgpu_bo_list_entry), 2198 GFP_KERNEL); 2199 if (!pd_bo_list_entries) { 2200 pr_err("%s: Failed to allocate PD BO list entries\n", __func__); 2201 ret = -ENOMEM; 2202 goto out_no_mem; 2203 } 2204 2205 INIT_LIST_HEAD(&resv_list); 2206 INIT_LIST_HEAD(&duplicates); 2207 2208 /* Get all the page directory BOs that need to be reserved */ 2209 i = 0; 2210 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2211 vm_list_node) 2212 amdgpu_vm_get_pd_bo(peer_vm, &resv_list, 2213 &pd_bo_list_entries[i++]); 2214 /* Add the userptr_inval_list entries to resv_list */ 2215 list_for_each_entry(mem, &process_info->userptr_inval_list, 2216 validate_list.head) { 2217 list_add_tail(&mem->resv_list.head, &resv_list); 2218 mem->resv_list.bo = mem->validate_list.bo; 2219 mem->resv_list.num_shared = mem->validate_list.num_shared; 2220 } 2221 2222 /* Reserve all BOs and page tables for validation */ 2223 ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates); 2224 WARN(!list_empty(&duplicates), "Duplicates should be empty"); 2225 if (ret) 2226 goto out_free; 2227 2228 amdgpu_sync_create(&sync); 2229 2230 ret = process_validate_vms(process_info); 2231 if (ret) 2232 goto unreserve_out; 2233 2234 /* Validate BOs and update GPUVM page tables */ 2235 list_for_each_entry_safe(mem, tmp_mem, 2236 &process_info->userptr_inval_list, 2237 validate_list.head) { 2238 struct kfd_mem_attachment *attachment; 2239 2240 bo = mem->bo; 2241 2242 /* Validate the BO if we got user pages */ 2243 if (bo->tbo.ttm->pages[0]) { 2244 amdgpu_bo_placement_from_domain(bo, mem->domain); 2245 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 2246 if (ret) { 2247 pr_err("%s: failed to validate BO\n", __func__); 2248 goto unreserve_out; 2249 } 2250 } 2251 2252 list_move_tail(&mem->validate_list.head, 2253 &process_info->userptr_valid_list); 2254 2255 /* Update mapping. If the BO was not validated 2256 * (because we couldn't get user pages), this will 2257 * clear the page table entries, which will result in 2258 * VM faults if the GPU tries to access the invalid 2259 * memory. 2260 */ 2261 list_for_each_entry(attachment, &mem->attachments, list) { 2262 if (!attachment->is_mapped) 2263 continue; 2264 2265 kfd_mem_dmaunmap_attachment(mem, attachment); 2266 ret = update_gpuvm_pte(mem, attachment, &sync, NULL); 2267 if (ret) { 2268 pr_err("%s: update PTE failed\n", __func__); 2269 /* make sure this gets validated again */ 2270 atomic_inc(&mem->invalid); 2271 goto unreserve_out; 2272 } 2273 } 2274 } 2275 2276 /* Update page directories */ 2277 ret = process_update_pds(process_info, &sync); 2278 2279 unreserve_out: 2280 ttm_eu_backoff_reservation(&ticket, &resv_list); 2281 amdgpu_sync_wait(&sync, false); 2282 amdgpu_sync_free(&sync); 2283 out_free: 2284 kfree(pd_bo_list_entries); 2285 out_no_mem: 2286 2287 return ret; 2288 } 2289 2290 /* Worker callback to restore evicted userptr BOs 2291 * 2292 * Tries to update and validate all userptr BOs. If successful and no 2293 * concurrent evictions happened, the queues are restarted. Otherwise, 2294 * reschedule for another attempt later. 2295 */ 2296 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work) 2297 { 2298 struct delayed_work *dwork = to_delayed_work(work); 2299 struct amdkfd_process_info *process_info = 2300 container_of(dwork, struct amdkfd_process_info, 2301 restore_userptr_work); 2302 struct task_struct *usertask; 2303 struct mm_struct *mm; 2304 int evicted_bos; 2305 2306 evicted_bos = atomic_read(&process_info->evicted_bos); 2307 if (!evicted_bos) 2308 return; 2309 2310 /* Reference task and mm in case of concurrent process termination */ 2311 usertask = get_pid_task(process_info->pid, PIDTYPE_PID); 2312 if (!usertask) 2313 return; 2314 mm = get_task_mm(usertask); 2315 if (!mm) { 2316 put_task_struct(usertask); 2317 return; 2318 } 2319 2320 mutex_lock(&process_info->lock); 2321 2322 if (update_invalid_user_pages(process_info, mm)) 2323 goto unlock_out; 2324 /* userptr_inval_list can be empty if all evicted userptr BOs 2325 * have been freed. In that case there is nothing to validate 2326 * and we can just restart the queues. 2327 */ 2328 if (!list_empty(&process_info->userptr_inval_list)) { 2329 if (atomic_read(&process_info->evicted_bos) != evicted_bos) 2330 goto unlock_out; /* Concurrent eviction, try again */ 2331 2332 if (validate_invalid_user_pages(process_info)) 2333 goto unlock_out; 2334 } 2335 /* Final check for concurrent evicton and atomic update. If 2336 * another eviction happens after successful update, it will 2337 * be a first eviction that calls quiesce_mm. The eviction 2338 * reference counting inside KFD will handle this case. 2339 */ 2340 if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) != 2341 evicted_bos) 2342 goto unlock_out; 2343 evicted_bos = 0; 2344 if (kgd2kfd_resume_mm(mm)) { 2345 pr_err("%s: Failed to resume KFD\n", __func__); 2346 /* No recovery from this failure. Probably the CP is 2347 * hanging. No point trying again. 2348 */ 2349 } 2350 2351 unlock_out: 2352 mutex_unlock(&process_info->lock); 2353 mmput(mm); 2354 put_task_struct(usertask); 2355 2356 /* If validation failed, reschedule another attempt */ 2357 if (evicted_bos) 2358 schedule_delayed_work(&process_info->restore_userptr_work, 2359 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS)); 2360 } 2361 2362 /** amdgpu_amdkfd_gpuvm_restore_process_bos - Restore all BOs for the given 2363 * KFD process identified by process_info 2364 * 2365 * @process_info: amdkfd_process_info of the KFD process 2366 * 2367 * After memory eviction, restore thread calls this function. The function 2368 * should be called when the Process is still valid. BO restore involves - 2369 * 2370 * 1. Release old eviction fence and create new one 2371 * 2. Get two copies of PD BO list from all the VMs. Keep one copy as pd_list. 2372 * 3 Use the second PD list and kfd_bo_list to create a list (ctx.list) of 2373 * BOs that need to be reserved. 2374 * 4. Reserve all the BOs 2375 * 5. Validate of PD and PT BOs. 2376 * 6. Validate all KFD BOs using kfd_bo_list and Map them and add new fence 2377 * 7. Add fence to all PD and PT BOs. 2378 * 8. Unreserve all BOs 2379 */ 2380 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef) 2381 { 2382 struct amdgpu_bo_list_entry *pd_bo_list; 2383 struct amdkfd_process_info *process_info = info; 2384 struct amdgpu_vm *peer_vm; 2385 struct kgd_mem *mem; 2386 struct bo_vm_reservation_context ctx; 2387 struct amdgpu_amdkfd_fence *new_fence; 2388 int ret = 0, i; 2389 struct list_head duplicate_save; 2390 struct amdgpu_sync sync_obj; 2391 unsigned long failed_size = 0; 2392 unsigned long total_size = 0; 2393 2394 INIT_LIST_HEAD(&duplicate_save); 2395 INIT_LIST_HEAD(&ctx.list); 2396 INIT_LIST_HEAD(&ctx.duplicates); 2397 2398 pd_bo_list = kcalloc(process_info->n_vms, 2399 sizeof(struct amdgpu_bo_list_entry), 2400 GFP_KERNEL); 2401 if (!pd_bo_list) 2402 return -ENOMEM; 2403 2404 i = 0; 2405 mutex_lock(&process_info->lock); 2406 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2407 vm_list_node) 2408 amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]); 2409 2410 /* Reserve all BOs and page tables/directory. Add all BOs from 2411 * kfd_bo_list to ctx.list 2412 */ 2413 list_for_each_entry(mem, &process_info->kfd_bo_list, 2414 validate_list.head) { 2415 2416 list_add_tail(&mem->resv_list.head, &ctx.list); 2417 mem->resv_list.bo = mem->validate_list.bo; 2418 mem->resv_list.num_shared = mem->validate_list.num_shared; 2419 } 2420 2421 ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list, 2422 false, &duplicate_save); 2423 if (ret) { 2424 pr_debug("Memory eviction: TTM Reserve Failed. Try again\n"); 2425 goto ttm_reserve_fail; 2426 } 2427 2428 amdgpu_sync_create(&sync_obj); 2429 2430 /* Validate PDs and PTs */ 2431 ret = process_validate_vms(process_info); 2432 if (ret) 2433 goto validate_map_fail; 2434 2435 ret = process_sync_pds_resv(process_info, &sync_obj); 2436 if (ret) { 2437 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n"); 2438 goto validate_map_fail; 2439 } 2440 2441 /* Validate BOs and map them to GPUVM (update VM page tables). */ 2442 list_for_each_entry(mem, &process_info->kfd_bo_list, 2443 validate_list.head) { 2444 2445 struct amdgpu_bo *bo = mem->bo; 2446 uint32_t domain = mem->domain; 2447 struct kfd_mem_attachment *attachment; 2448 2449 total_size += amdgpu_bo_size(bo); 2450 2451 ret = amdgpu_amdkfd_bo_validate(bo, domain, false); 2452 if (ret) { 2453 pr_debug("Memory eviction: Validate BOs failed\n"); 2454 failed_size += amdgpu_bo_size(bo); 2455 ret = amdgpu_amdkfd_bo_validate(bo, 2456 AMDGPU_GEM_DOMAIN_GTT, false); 2457 if (ret) { 2458 pr_debug("Memory eviction: Try again\n"); 2459 goto validate_map_fail; 2460 } 2461 } 2462 ret = amdgpu_sync_fence(&sync_obj, bo->tbo.moving); 2463 if (ret) { 2464 pr_debug("Memory eviction: Sync BO fence failed. Try again\n"); 2465 goto validate_map_fail; 2466 } 2467 list_for_each_entry(attachment, &mem->attachments, list) { 2468 if (!attachment->is_mapped) 2469 continue; 2470 2471 kfd_mem_dmaunmap_attachment(mem, attachment); 2472 ret = update_gpuvm_pte(mem, attachment, &sync_obj, NULL); 2473 if (ret) { 2474 pr_debug("Memory eviction: update PTE failed. Try again\n"); 2475 goto validate_map_fail; 2476 } 2477 } 2478 } 2479 2480 if (failed_size) 2481 pr_debug("0x%lx/0x%lx in system\n", failed_size, total_size); 2482 2483 /* Update page directories */ 2484 ret = process_update_pds(process_info, &sync_obj); 2485 if (ret) { 2486 pr_debug("Memory eviction: update PDs failed. Try again\n"); 2487 goto validate_map_fail; 2488 } 2489 2490 /* Wait for validate and PT updates to finish */ 2491 amdgpu_sync_wait(&sync_obj, false); 2492 2493 /* Release old eviction fence and create new one, because fence only 2494 * goes from unsignaled to signaled, fence cannot be reused. 2495 * Use context and mm from the old fence. 2496 */ 2497 new_fence = amdgpu_amdkfd_fence_create( 2498 process_info->eviction_fence->base.context, 2499 process_info->eviction_fence->mm, 2500 NULL); 2501 if (!new_fence) { 2502 pr_err("Failed to create eviction fence\n"); 2503 ret = -ENOMEM; 2504 goto validate_map_fail; 2505 } 2506 dma_fence_put(&process_info->eviction_fence->base); 2507 process_info->eviction_fence = new_fence; 2508 *ef = dma_fence_get(&new_fence->base); 2509 2510 /* Attach new eviction fence to all BOs */ 2511 list_for_each_entry(mem, &process_info->kfd_bo_list, 2512 validate_list.head) 2513 amdgpu_bo_fence(mem->bo, 2514 &process_info->eviction_fence->base, true); 2515 2516 /* Attach eviction fence to PD / PT BOs */ 2517 list_for_each_entry(peer_vm, &process_info->vm_list_head, 2518 vm_list_node) { 2519 struct amdgpu_bo *bo = peer_vm->root.bo; 2520 2521 amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true); 2522 } 2523 2524 validate_map_fail: 2525 ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list); 2526 amdgpu_sync_free(&sync_obj); 2527 ttm_reserve_fail: 2528 mutex_unlock(&process_info->lock); 2529 kfree(pd_bo_list); 2530 return ret; 2531 } 2532 2533 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem) 2534 { 2535 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 2536 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws; 2537 int ret; 2538 2539 if (!info || !gws) 2540 return -EINVAL; 2541 2542 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL); 2543 if (!*mem) 2544 return -ENOMEM; 2545 2546 mutex_init(&(*mem)->lock); 2547 INIT_LIST_HEAD(&(*mem)->attachments); 2548 (*mem)->bo = amdgpu_bo_ref(gws_bo); 2549 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS; 2550 (*mem)->process_info = process_info; 2551 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false); 2552 amdgpu_sync_create(&(*mem)->sync); 2553 2554 2555 /* Validate gws bo the first time it is added to process */ 2556 mutex_lock(&(*mem)->process_info->lock); 2557 ret = amdgpu_bo_reserve(gws_bo, false); 2558 if (unlikely(ret)) { 2559 pr_err("Reserve gws bo failed %d\n", ret); 2560 goto bo_reservation_failure; 2561 } 2562 2563 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true); 2564 if (ret) { 2565 pr_err("GWS BO validate failed %d\n", ret); 2566 goto bo_validation_failure; 2567 } 2568 /* GWS resource is shared b/t amdgpu and amdkfd 2569 * Add process eviction fence to bo so they can 2570 * evict each other. 2571 */ 2572 ret = dma_resv_reserve_shared(gws_bo->tbo.base.resv, 1); 2573 if (ret) 2574 goto reserve_shared_fail; 2575 amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true); 2576 amdgpu_bo_unreserve(gws_bo); 2577 mutex_unlock(&(*mem)->process_info->lock); 2578 2579 return ret; 2580 2581 reserve_shared_fail: 2582 bo_validation_failure: 2583 amdgpu_bo_unreserve(gws_bo); 2584 bo_reservation_failure: 2585 mutex_unlock(&(*mem)->process_info->lock); 2586 amdgpu_sync_free(&(*mem)->sync); 2587 remove_kgd_mem_from_kfd_bo_list(*mem, process_info); 2588 amdgpu_bo_unref(&gws_bo); 2589 mutex_destroy(&(*mem)->lock); 2590 kfree(*mem); 2591 *mem = NULL; 2592 return ret; 2593 } 2594 2595 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem) 2596 { 2597 int ret; 2598 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info; 2599 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem; 2600 struct amdgpu_bo *gws_bo = kgd_mem->bo; 2601 2602 /* Remove BO from process's validate list so restore worker won't touch 2603 * it anymore 2604 */ 2605 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info); 2606 2607 ret = amdgpu_bo_reserve(gws_bo, false); 2608 if (unlikely(ret)) { 2609 pr_err("Reserve gws bo failed %d\n", ret); 2610 //TODO add BO back to validate_list? 2611 return ret; 2612 } 2613 amdgpu_amdkfd_remove_eviction_fence(gws_bo, 2614 process_info->eviction_fence); 2615 amdgpu_bo_unreserve(gws_bo); 2616 amdgpu_sync_free(&kgd_mem->sync); 2617 amdgpu_bo_unref(&gws_bo); 2618 mutex_destroy(&kgd_mem->lock); 2619 kfree(mem); 2620 return 0; 2621 } 2622 2623 /* Returns GPU-specific tiling mode information */ 2624 int amdgpu_amdkfd_get_tile_config(struct amdgpu_device *adev, 2625 struct tile_config *config) 2626 { 2627 config->gb_addr_config = adev->gfx.config.gb_addr_config; 2628 config->tile_config_ptr = adev->gfx.config.tile_mode_array; 2629 config->num_tile_configs = 2630 ARRAY_SIZE(adev->gfx.config.tile_mode_array); 2631 config->macro_tile_config_ptr = 2632 adev->gfx.config.macrotile_mode_array; 2633 config->num_macro_tile_configs = 2634 ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 2635 2636 /* Those values are not set from GFX9 onwards */ 2637 config->num_banks = adev->gfx.config.num_banks; 2638 config->num_ranks = adev->gfx.config.num_ranks; 2639 2640 return 0; 2641 } 2642