1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/dma-fence-array.h> 29 #include <linux/interval_tree_generic.h> 30 #include <linux/idr.h> 31 #include <linux/dma-buf.h> 32 33 #include <drm/amdgpu_drm.h> 34 #include "amdgpu.h" 35 #include "amdgpu_trace.h" 36 #include "amdgpu_amdkfd.h" 37 #include "amdgpu_gmc.h" 38 #include "amdgpu_xgmi.h" 39 #include "amdgpu_dma_buf.h" 40 41 /** 42 * DOC: GPUVM 43 * 44 * GPUVM is similar to the legacy gart on older asics, however 45 * rather than there being a single global gart table 46 * for the entire GPU, there are multiple VM page tables active 47 * at any given time. The VM page tables can contain a mix 48 * vram pages and system memory pages and system memory pages 49 * can be mapped as snooped (cached system pages) or unsnooped 50 * (uncached system pages). 51 * Each VM has an ID associated with it and there is a page table 52 * associated with each VMID. When execting a command buffer, 53 * the kernel tells the the ring what VMID to use for that command 54 * buffer. VMIDs are allocated dynamically as commands are submitted. 55 * The userspace drivers maintain their own address space and the kernel 56 * sets up their pages tables accordingly when they submit their 57 * command buffers and a VMID is assigned. 58 * Cayman/Trinity support up to 8 active VMs at any given time; 59 * SI supports 16. 60 */ 61 62 #define START(node) ((node)->start) 63 #define LAST(node) ((node)->last) 64 65 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last, 66 START, LAST, static, amdgpu_vm_it) 67 68 #undef START 69 #undef LAST 70 71 /** 72 * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback 73 */ 74 struct amdgpu_prt_cb { 75 76 /** 77 * @adev: amdgpu device 78 */ 79 struct amdgpu_device *adev; 80 81 /** 82 * @cb: callback 83 */ 84 struct dma_fence_cb cb; 85 }; 86 87 /* 88 * vm eviction_lock can be taken in MMU notifiers. Make sure no reclaim-FS 89 * happens while holding this lock anywhere to prevent deadlocks when 90 * an MMU notifier runs in reclaim-FS context. 91 */ 92 static inline void amdgpu_vm_eviction_lock(struct amdgpu_vm *vm) 93 { 94 mutex_lock(&vm->eviction_lock); 95 vm->saved_flags = memalloc_nofs_save(); 96 } 97 98 static inline int amdgpu_vm_eviction_trylock(struct amdgpu_vm *vm) 99 { 100 if (mutex_trylock(&vm->eviction_lock)) { 101 vm->saved_flags = memalloc_nofs_save(); 102 return 1; 103 } 104 return 0; 105 } 106 107 static inline void amdgpu_vm_eviction_unlock(struct amdgpu_vm *vm) 108 { 109 memalloc_nofs_restore(vm->saved_flags); 110 mutex_unlock(&vm->eviction_lock); 111 } 112 113 /** 114 * amdgpu_vm_level_shift - return the addr shift for each level 115 * 116 * @adev: amdgpu_device pointer 117 * @level: VMPT level 118 * 119 * Returns: 120 * The number of bits the pfn needs to be right shifted for a level. 121 */ 122 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev, 123 unsigned level) 124 { 125 switch (level) { 126 case AMDGPU_VM_PDB2: 127 case AMDGPU_VM_PDB1: 128 case AMDGPU_VM_PDB0: 129 return 9 * (AMDGPU_VM_PDB0 - level) + 130 adev->vm_manager.block_size; 131 case AMDGPU_VM_PTB: 132 return 0; 133 default: 134 return ~0; 135 } 136 } 137 138 /** 139 * amdgpu_vm_num_entries - return the number of entries in a PD/PT 140 * 141 * @adev: amdgpu_device pointer 142 * @level: VMPT level 143 * 144 * Returns: 145 * The number of entries in a page directory or page table. 146 */ 147 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev, 148 unsigned level) 149 { 150 unsigned shift = amdgpu_vm_level_shift(adev, 151 adev->vm_manager.root_level); 152 153 if (level == adev->vm_manager.root_level) 154 /* For the root directory */ 155 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) 156 >> shift; 157 else if (level != AMDGPU_VM_PTB) 158 /* Everything in between */ 159 return 512; 160 else 161 /* For the page tables on the leaves */ 162 return AMDGPU_VM_PTE_COUNT(adev); 163 } 164 165 /** 166 * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD 167 * 168 * @adev: amdgpu_device pointer 169 * 170 * Returns: 171 * The number of entries in the root page directory which needs the ATS setting. 172 */ 173 static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev) 174 { 175 unsigned shift; 176 177 shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level); 178 return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT); 179 } 180 181 /** 182 * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT 183 * 184 * @adev: amdgpu_device pointer 185 * @level: VMPT level 186 * 187 * Returns: 188 * The mask to extract the entry number of a PD/PT from an address. 189 */ 190 static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev, 191 unsigned int level) 192 { 193 if (level <= adev->vm_manager.root_level) 194 return 0xffffffff; 195 else if (level != AMDGPU_VM_PTB) 196 return 0x1ff; 197 else 198 return AMDGPU_VM_PTE_COUNT(adev) - 1; 199 } 200 201 /** 202 * amdgpu_vm_bo_size - returns the size of the BOs in bytes 203 * 204 * @adev: amdgpu_device pointer 205 * @level: VMPT level 206 * 207 * Returns: 208 * The size of the BO for a page directory or page table in bytes. 209 */ 210 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level) 211 { 212 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8); 213 } 214 215 /** 216 * amdgpu_vm_bo_evicted - vm_bo is evicted 217 * 218 * @vm_bo: vm_bo which is evicted 219 * 220 * State for PDs/PTs and per VM BOs which are not at the location they should 221 * be. 222 */ 223 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo) 224 { 225 struct amdgpu_vm *vm = vm_bo->vm; 226 struct amdgpu_bo *bo = vm_bo->bo; 227 228 vm_bo->moved = true; 229 if (bo->tbo.type == ttm_bo_type_kernel) 230 list_move(&vm_bo->vm_status, &vm->evicted); 231 else 232 list_move_tail(&vm_bo->vm_status, &vm->evicted); 233 } 234 /** 235 * amdgpu_vm_bo_moved - vm_bo is moved 236 * 237 * @vm_bo: vm_bo which is moved 238 * 239 * State for per VM BOs which are moved, but that change is not yet reflected 240 * in the page tables. 241 */ 242 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo) 243 { 244 list_move(&vm_bo->vm_status, &vm_bo->vm->moved); 245 } 246 247 /** 248 * amdgpu_vm_bo_idle - vm_bo is idle 249 * 250 * @vm_bo: vm_bo which is now idle 251 * 252 * State for PDs/PTs and per VM BOs which have gone through the state machine 253 * and are now idle. 254 */ 255 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo) 256 { 257 list_move(&vm_bo->vm_status, &vm_bo->vm->idle); 258 vm_bo->moved = false; 259 } 260 261 /** 262 * amdgpu_vm_bo_invalidated - vm_bo is invalidated 263 * 264 * @vm_bo: vm_bo which is now invalidated 265 * 266 * State for normal BOs which are invalidated and that change not yet reflected 267 * in the PTs. 268 */ 269 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo) 270 { 271 spin_lock(&vm_bo->vm->invalidated_lock); 272 list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated); 273 spin_unlock(&vm_bo->vm->invalidated_lock); 274 } 275 276 /** 277 * amdgpu_vm_bo_relocated - vm_bo is reloacted 278 * 279 * @vm_bo: vm_bo which is relocated 280 * 281 * State for PDs/PTs which needs to update their parent PD. 282 * For the root PD, just move to idle state. 283 */ 284 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo) 285 { 286 if (vm_bo->bo->parent) 287 list_move(&vm_bo->vm_status, &vm_bo->vm->relocated); 288 else 289 amdgpu_vm_bo_idle(vm_bo); 290 } 291 292 /** 293 * amdgpu_vm_bo_done - vm_bo is done 294 * 295 * @vm_bo: vm_bo which is now done 296 * 297 * State for normal BOs which are invalidated and that change has been updated 298 * in the PTs. 299 */ 300 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo) 301 { 302 spin_lock(&vm_bo->vm->invalidated_lock); 303 list_move(&vm_bo->vm_status, &vm_bo->vm->done); 304 spin_unlock(&vm_bo->vm->invalidated_lock); 305 } 306 307 /** 308 * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm 309 * 310 * @base: base structure for tracking BO usage in a VM 311 * @vm: vm to which bo is to be added 312 * @bo: amdgpu buffer object 313 * 314 * Initialize a bo_va_base structure and add it to the appropriate lists 315 * 316 */ 317 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base, 318 struct amdgpu_vm *vm, 319 struct amdgpu_bo *bo) 320 { 321 base->vm = vm; 322 base->bo = bo; 323 base->next = NULL; 324 INIT_LIST_HEAD(&base->vm_status); 325 326 if (!bo) 327 return; 328 base->next = bo->vm_bo; 329 bo->vm_bo = base; 330 331 if (bo->tbo.base.resv != vm->root.base.bo->tbo.base.resv) 332 return; 333 334 vm->bulk_moveable = false; 335 if (bo->tbo.type == ttm_bo_type_kernel && bo->parent) 336 amdgpu_vm_bo_relocated(base); 337 else 338 amdgpu_vm_bo_idle(base); 339 340 if (bo->preferred_domains & 341 amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type)) 342 return; 343 344 /* 345 * we checked all the prerequisites, but it looks like this per vm bo 346 * is currently evicted. add the bo to the evicted list to make sure it 347 * is validated on next vm use to avoid fault. 348 * */ 349 amdgpu_vm_bo_evicted(base); 350 } 351 352 /** 353 * amdgpu_vm_pt_parent - get the parent page directory 354 * 355 * @pt: child page table 356 * 357 * Helper to get the parent entry for the child page table. NULL if we are at 358 * the root page directory. 359 */ 360 static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt) 361 { 362 struct amdgpu_bo *parent = pt->base.bo->parent; 363 364 if (!parent) 365 return NULL; 366 367 return container_of(parent->vm_bo, struct amdgpu_vm_pt, base); 368 } 369 370 /* 371 * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt 372 */ 373 struct amdgpu_vm_pt_cursor { 374 uint64_t pfn; 375 struct amdgpu_vm_pt *parent; 376 struct amdgpu_vm_pt *entry; 377 unsigned level; 378 }; 379 380 /** 381 * amdgpu_vm_pt_start - start PD/PT walk 382 * 383 * @adev: amdgpu_device pointer 384 * @vm: amdgpu_vm structure 385 * @start: start address of the walk 386 * @cursor: state to initialize 387 * 388 * Initialize a amdgpu_vm_pt_cursor to start a walk. 389 */ 390 static void amdgpu_vm_pt_start(struct amdgpu_device *adev, 391 struct amdgpu_vm *vm, uint64_t start, 392 struct amdgpu_vm_pt_cursor *cursor) 393 { 394 cursor->pfn = start; 395 cursor->parent = NULL; 396 cursor->entry = &vm->root; 397 cursor->level = adev->vm_manager.root_level; 398 } 399 400 /** 401 * amdgpu_vm_pt_descendant - go to child node 402 * 403 * @adev: amdgpu_device pointer 404 * @cursor: current state 405 * 406 * Walk to the child node of the current node. 407 * Returns: 408 * True if the walk was possible, false otherwise. 409 */ 410 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, 411 struct amdgpu_vm_pt_cursor *cursor) 412 { 413 unsigned mask, shift, idx; 414 415 if (!cursor->entry->entries) 416 return false; 417 418 BUG_ON(!cursor->entry->base.bo); 419 mask = amdgpu_vm_entries_mask(adev, cursor->level); 420 shift = amdgpu_vm_level_shift(adev, cursor->level); 421 422 ++cursor->level; 423 idx = (cursor->pfn >> shift) & mask; 424 cursor->parent = cursor->entry; 425 cursor->entry = &cursor->entry->entries[idx]; 426 return true; 427 } 428 429 /** 430 * amdgpu_vm_pt_sibling - go to sibling node 431 * 432 * @adev: amdgpu_device pointer 433 * @cursor: current state 434 * 435 * Walk to the sibling node of the current node. 436 * Returns: 437 * True if the walk was possible, false otherwise. 438 */ 439 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, 440 struct amdgpu_vm_pt_cursor *cursor) 441 { 442 unsigned shift, num_entries; 443 444 /* Root doesn't have a sibling */ 445 if (!cursor->parent) 446 return false; 447 448 /* Go to our parents and see if we got a sibling */ 449 shift = amdgpu_vm_level_shift(adev, cursor->level - 1); 450 num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1); 451 452 if (cursor->entry == &cursor->parent->entries[num_entries - 1]) 453 return false; 454 455 cursor->pfn += 1ULL << shift; 456 cursor->pfn &= ~((1ULL << shift) - 1); 457 ++cursor->entry; 458 return true; 459 } 460 461 /** 462 * amdgpu_vm_pt_ancestor - go to parent node 463 * 464 * @cursor: current state 465 * 466 * Walk to the parent node of the current node. 467 * Returns: 468 * True if the walk was possible, false otherwise. 469 */ 470 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) 471 { 472 if (!cursor->parent) 473 return false; 474 475 --cursor->level; 476 cursor->entry = cursor->parent; 477 cursor->parent = amdgpu_vm_pt_parent(cursor->parent); 478 return true; 479 } 480 481 /** 482 * amdgpu_vm_pt_next - get next PD/PT in hieratchy 483 * 484 * @adev: amdgpu_device pointer 485 * @cursor: current state 486 * 487 * Walk the PD/PT tree to the next node. 488 */ 489 static void amdgpu_vm_pt_next(struct amdgpu_device *adev, 490 struct amdgpu_vm_pt_cursor *cursor) 491 { 492 /* First try a newborn child */ 493 if (amdgpu_vm_pt_descendant(adev, cursor)) 494 return; 495 496 /* If that didn't worked try to find a sibling */ 497 while (!amdgpu_vm_pt_sibling(adev, cursor)) { 498 /* No sibling, go to our parents and grandparents */ 499 if (!amdgpu_vm_pt_ancestor(cursor)) { 500 cursor->pfn = ~0ll; 501 return; 502 } 503 } 504 } 505 506 /** 507 * amdgpu_vm_pt_first_dfs - start a deep first search 508 * 509 * @adev: amdgpu_device structure 510 * @vm: amdgpu_vm structure 511 * @start: optional cursor to start with 512 * @cursor: state to initialize 513 * 514 * Starts a deep first traversal of the PD/PT tree. 515 */ 516 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, 517 struct amdgpu_vm *vm, 518 struct amdgpu_vm_pt_cursor *start, 519 struct amdgpu_vm_pt_cursor *cursor) 520 { 521 if (start) 522 *cursor = *start; 523 else 524 amdgpu_vm_pt_start(adev, vm, 0, cursor); 525 while (amdgpu_vm_pt_descendant(adev, cursor)); 526 } 527 528 /** 529 * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue 530 * 531 * @start: starting point for the search 532 * @entry: current entry 533 * 534 * Returns: 535 * True when the search should continue, false otherwise. 536 */ 537 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, 538 struct amdgpu_vm_pt *entry) 539 { 540 return entry && (!start || entry != start->entry); 541 } 542 543 /** 544 * amdgpu_vm_pt_next_dfs - get the next node for a deep first search 545 * 546 * @adev: amdgpu_device structure 547 * @cursor: current state 548 * 549 * Move the cursor to the next node in a deep first search. 550 */ 551 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, 552 struct amdgpu_vm_pt_cursor *cursor) 553 { 554 if (!cursor->entry) 555 return; 556 557 if (!cursor->parent) 558 cursor->entry = NULL; 559 else if (amdgpu_vm_pt_sibling(adev, cursor)) 560 while (amdgpu_vm_pt_descendant(adev, cursor)); 561 else 562 amdgpu_vm_pt_ancestor(cursor); 563 } 564 565 /* 566 * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs 567 */ 568 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ 569 for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ 570 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ 571 amdgpu_vm_pt_continue_dfs((start), (entry)); \ 572 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) 573 574 /** 575 * amdgpu_vm_get_pd_bo - add the VM PD to a validation list 576 * 577 * @vm: vm providing the BOs 578 * @validated: head of validation list 579 * @entry: entry to add 580 * 581 * Add the page directory to the list of BOs to 582 * validate for command submission. 583 */ 584 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, 585 struct list_head *validated, 586 struct amdgpu_bo_list_entry *entry) 587 { 588 entry->priority = 0; 589 entry->tv.bo = &vm->root.base.bo->tbo; 590 /* Two for VM updates, one for TTM and one for the CS job */ 591 entry->tv.num_shared = 4; 592 entry->user_pages = NULL; 593 list_add(&entry->tv.head, validated); 594 } 595 596 /** 597 * amdgpu_vm_del_from_lru_notify - update bulk_moveable flag 598 * 599 * @bo: BO which was removed from the LRU 600 * 601 * Make sure the bulk_moveable flag is updated when a BO is removed from the 602 * LRU. 603 */ 604 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo) 605 { 606 struct amdgpu_bo *abo; 607 struct amdgpu_vm_bo_base *bo_base; 608 609 if (!amdgpu_bo_is_amdgpu_bo(bo)) 610 return; 611 612 if (bo->pin_count) 613 return; 614 615 abo = ttm_to_amdgpu_bo(bo); 616 if (!abo->parent) 617 return; 618 for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) { 619 struct amdgpu_vm *vm = bo_base->vm; 620 621 if (abo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 622 vm->bulk_moveable = false; 623 } 624 625 } 626 /** 627 * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU 628 * 629 * @adev: amdgpu device pointer 630 * @vm: vm providing the BOs 631 * 632 * Move all BOs to the end of LRU and remember their positions to put them 633 * together. 634 */ 635 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev, 636 struct amdgpu_vm *vm) 637 { 638 struct amdgpu_vm_bo_base *bo_base; 639 640 if (vm->bulk_moveable) { 641 spin_lock(&ttm_bo_glob.lru_lock); 642 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move); 643 spin_unlock(&ttm_bo_glob.lru_lock); 644 return; 645 } 646 647 memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move)); 648 649 spin_lock(&ttm_bo_glob.lru_lock); 650 list_for_each_entry(bo_base, &vm->idle, vm_status) { 651 struct amdgpu_bo *bo = bo_base->bo; 652 653 if (!bo->parent) 654 continue; 655 656 ttm_bo_move_to_lru_tail(&bo->tbo, &bo->tbo.mem, 657 &vm->lru_bulk_move); 658 if (bo->shadow) 659 ttm_bo_move_to_lru_tail(&bo->shadow->tbo, 660 &bo->shadow->tbo.mem, 661 &vm->lru_bulk_move); 662 } 663 spin_unlock(&ttm_bo_glob.lru_lock); 664 665 vm->bulk_moveable = true; 666 } 667 668 /** 669 * amdgpu_vm_validate_pt_bos - validate the page table BOs 670 * 671 * @adev: amdgpu device pointer 672 * @vm: vm providing the BOs 673 * @validate: callback to do the validation 674 * @param: parameter for the validation callback 675 * 676 * Validate the page table BOs on command submission if neccessary. 677 * 678 * Returns: 679 * Validation result. 680 */ 681 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm, 682 int (*validate)(void *p, struct amdgpu_bo *bo), 683 void *param) 684 { 685 struct amdgpu_vm_bo_base *bo_base, *tmp; 686 int r; 687 688 vm->bulk_moveable &= list_empty(&vm->evicted); 689 690 list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) { 691 struct amdgpu_bo *bo = bo_base->bo; 692 693 r = validate(param, bo); 694 if (r) 695 return r; 696 697 if (bo->tbo.type != ttm_bo_type_kernel) { 698 amdgpu_vm_bo_moved(bo_base); 699 } else { 700 vm->update_funcs->map_table(bo); 701 amdgpu_vm_bo_relocated(bo_base); 702 } 703 } 704 705 amdgpu_vm_eviction_lock(vm); 706 vm->evicting = false; 707 amdgpu_vm_eviction_unlock(vm); 708 709 return 0; 710 } 711 712 /** 713 * amdgpu_vm_ready - check VM is ready for updates 714 * 715 * @vm: VM to check 716 * 717 * Check if all VM PDs/PTs are ready for updates 718 * 719 * Returns: 720 * True if eviction list is empty. 721 */ 722 bool amdgpu_vm_ready(struct amdgpu_vm *vm) 723 { 724 return list_empty(&vm->evicted); 725 } 726 727 /** 728 * amdgpu_vm_clear_bo - initially clear the PDs/PTs 729 * 730 * @adev: amdgpu_device pointer 731 * @vm: VM to clear BO from 732 * @bo: BO to clear 733 * @immediate: use an immediate update 734 * 735 * Root PD needs to be reserved when calling this. 736 * 737 * Returns: 738 * 0 on success, errno otherwise. 739 */ 740 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, 741 struct amdgpu_vm *vm, 742 struct amdgpu_bo *bo, 743 bool immediate) 744 { 745 struct ttm_operation_ctx ctx = { true, false }; 746 unsigned level = adev->vm_manager.root_level; 747 struct amdgpu_vm_update_params params; 748 struct amdgpu_bo *ancestor = bo; 749 unsigned entries, ats_entries; 750 uint64_t addr; 751 int r; 752 753 /* Figure out our place in the hierarchy */ 754 if (ancestor->parent) { 755 ++level; 756 while (ancestor->parent->parent) { 757 ++level; 758 ancestor = ancestor->parent; 759 } 760 } 761 762 entries = amdgpu_bo_size(bo) / 8; 763 if (!vm->pte_support_ats) { 764 ats_entries = 0; 765 766 } else if (!bo->parent) { 767 ats_entries = amdgpu_vm_num_ats_entries(adev); 768 ats_entries = min(ats_entries, entries); 769 entries -= ats_entries; 770 771 } else { 772 struct amdgpu_vm_pt *pt; 773 774 pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base); 775 ats_entries = amdgpu_vm_num_ats_entries(adev); 776 if ((pt - vm->root.entries) >= ats_entries) { 777 ats_entries = 0; 778 } else { 779 ats_entries = entries; 780 entries = 0; 781 } 782 } 783 784 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 785 if (r) 786 return r; 787 788 if (bo->shadow) { 789 r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement, 790 &ctx); 791 if (r) 792 return r; 793 } 794 795 r = vm->update_funcs->map_table(bo); 796 if (r) 797 return r; 798 799 memset(¶ms, 0, sizeof(params)); 800 params.adev = adev; 801 params.vm = vm; 802 params.immediate = immediate; 803 804 r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); 805 if (r) 806 return r; 807 808 addr = 0; 809 if (ats_entries) { 810 uint64_t value = 0, flags; 811 812 flags = AMDGPU_PTE_DEFAULT_ATC; 813 if (level != AMDGPU_VM_PTB) { 814 /* Handle leaf PDEs as PTEs */ 815 flags |= AMDGPU_PDE_PTE; 816 amdgpu_gmc_get_vm_pde(adev, level, &value, &flags); 817 } 818 819 r = vm->update_funcs->update(¶ms, bo, addr, 0, ats_entries, 820 value, flags); 821 if (r) 822 return r; 823 824 addr += ats_entries * 8; 825 } 826 827 if (entries) { 828 uint64_t value = 0, flags = 0; 829 830 if (adev->asic_type >= CHIP_VEGA10) { 831 if (level != AMDGPU_VM_PTB) { 832 /* Handle leaf PDEs as PTEs */ 833 flags |= AMDGPU_PDE_PTE; 834 amdgpu_gmc_get_vm_pde(adev, level, 835 &value, &flags); 836 } else { 837 /* Workaround for fault priority problem on GMC9 */ 838 flags = AMDGPU_PTE_EXECUTABLE; 839 } 840 } 841 842 r = vm->update_funcs->update(¶ms, bo, addr, 0, entries, 843 value, flags); 844 if (r) 845 return r; 846 } 847 848 return vm->update_funcs->commit(¶ms, NULL); 849 } 850 851 /** 852 * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation 853 * 854 * @adev: amdgpu_device pointer 855 * @vm: requesting vm 856 * @level: the page table level 857 * @immediate: use a immediate update 858 * @bp: resulting BO allocation parameters 859 */ 860 static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm, 861 int level, bool immediate, 862 struct amdgpu_bo_param *bp) 863 { 864 memset(bp, 0, sizeof(*bp)); 865 866 bp->size = amdgpu_vm_bo_size(adev, level); 867 bp->byte_align = AMDGPU_GPU_PAGE_SIZE; 868 bp->domain = AMDGPU_GEM_DOMAIN_VRAM; 869 bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain); 870 bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | 871 AMDGPU_GEM_CREATE_CPU_GTT_USWC; 872 if (vm->use_cpu_for_update) 873 bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 874 else if (!vm->root.base.bo || vm->root.base.bo->shadow) 875 bp->flags |= AMDGPU_GEM_CREATE_SHADOW; 876 bp->type = ttm_bo_type_kernel; 877 bp->no_wait_gpu = immediate; 878 if (vm->root.base.bo) 879 bp->resv = vm->root.base.bo->tbo.base.resv; 880 } 881 882 /** 883 * amdgpu_vm_alloc_pts - Allocate a specific page table 884 * 885 * @adev: amdgpu_device pointer 886 * @vm: VM to allocate page tables for 887 * @cursor: Which page table to allocate 888 * @immediate: use an immediate update 889 * 890 * Make sure a specific page table or directory is allocated. 891 * 892 * Returns: 893 * 1 if page table needed to be allocated, 0 if page table was already 894 * allocated, negative errno if an error occurred. 895 */ 896 static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, 897 struct amdgpu_vm *vm, 898 struct amdgpu_vm_pt_cursor *cursor, 899 bool immediate) 900 { 901 struct amdgpu_vm_pt *entry = cursor->entry; 902 struct amdgpu_bo_param bp; 903 struct amdgpu_bo *pt; 904 int r; 905 906 if (cursor->level < AMDGPU_VM_PTB && !entry->entries) { 907 unsigned num_entries; 908 909 num_entries = amdgpu_vm_num_entries(adev, cursor->level); 910 entry->entries = kvmalloc_array(num_entries, 911 sizeof(*entry->entries), 912 GFP_KERNEL | __GFP_ZERO); 913 if (!entry->entries) 914 return -ENOMEM; 915 } 916 917 if (entry->base.bo) 918 return 0; 919 920 amdgpu_vm_bo_param(adev, vm, cursor->level, immediate, &bp); 921 922 r = amdgpu_bo_create(adev, &bp, &pt); 923 if (r) 924 return r; 925 926 /* Keep a reference to the root directory to avoid 927 * freeing them up in the wrong order. 928 */ 929 pt->parent = amdgpu_bo_ref(cursor->parent->base.bo); 930 amdgpu_vm_bo_base_init(&entry->base, vm, pt); 931 932 r = amdgpu_vm_clear_bo(adev, vm, pt, immediate); 933 if (r) 934 goto error_free_pt; 935 936 return 0; 937 938 error_free_pt: 939 amdgpu_bo_unref(&pt->shadow); 940 amdgpu_bo_unref(&pt); 941 return r; 942 } 943 944 /** 945 * amdgpu_vm_free_table - fre one PD/PT 946 * 947 * @entry: PDE to free 948 */ 949 static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry) 950 { 951 if (entry->base.bo) { 952 entry->base.bo->vm_bo = NULL; 953 list_del(&entry->base.vm_status); 954 amdgpu_bo_unref(&entry->base.bo->shadow); 955 amdgpu_bo_unref(&entry->base.bo); 956 } 957 kvfree(entry->entries); 958 entry->entries = NULL; 959 } 960 961 /** 962 * amdgpu_vm_free_pts - free PD/PT levels 963 * 964 * @adev: amdgpu device structure 965 * @vm: amdgpu vm structure 966 * @start: optional cursor where to start freeing PDs/PTs 967 * 968 * Free the page directory or page table level and all sub levels. 969 */ 970 static void amdgpu_vm_free_pts(struct amdgpu_device *adev, 971 struct amdgpu_vm *vm, 972 struct amdgpu_vm_pt_cursor *start) 973 { 974 struct amdgpu_vm_pt_cursor cursor; 975 struct amdgpu_vm_pt *entry; 976 977 vm->bulk_moveable = false; 978 979 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) 980 amdgpu_vm_free_table(entry); 981 982 if (start) 983 amdgpu_vm_free_table(start->entry); 984 } 985 986 /** 987 * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug 988 * 989 * @adev: amdgpu_device pointer 990 */ 991 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev) 992 { 993 const struct amdgpu_ip_block *ip_block; 994 bool has_compute_vm_bug; 995 struct amdgpu_ring *ring; 996 int i; 997 998 has_compute_vm_bug = false; 999 1000 ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 1001 if (ip_block) { 1002 /* Compute has a VM bug for GFX version < 7. 1003 Compute has a VM bug for GFX 8 MEC firmware version < 673.*/ 1004 if (ip_block->version->major <= 7) 1005 has_compute_vm_bug = true; 1006 else if (ip_block->version->major == 8) 1007 if (adev->gfx.mec_fw_version < 673) 1008 has_compute_vm_bug = true; 1009 } 1010 1011 for (i = 0; i < adev->num_rings; i++) { 1012 ring = adev->rings[i]; 1013 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) 1014 /* only compute rings */ 1015 ring->has_compute_vm_bug = has_compute_vm_bug; 1016 else 1017 ring->has_compute_vm_bug = false; 1018 } 1019 } 1020 1021 /** 1022 * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job. 1023 * 1024 * @ring: ring on which the job will be submitted 1025 * @job: job to submit 1026 * 1027 * Returns: 1028 * True if sync is needed. 1029 */ 1030 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring, 1031 struct amdgpu_job *job) 1032 { 1033 struct amdgpu_device *adev = ring->adev; 1034 unsigned vmhub = ring->funcs->vmhub; 1035 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 1036 struct amdgpu_vmid *id; 1037 bool gds_switch_needed; 1038 bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug; 1039 1040 if (job->vmid == 0) 1041 return false; 1042 id = &id_mgr->ids[job->vmid]; 1043 gds_switch_needed = ring->funcs->emit_gds_switch && ( 1044 id->gds_base != job->gds_base || 1045 id->gds_size != job->gds_size || 1046 id->gws_base != job->gws_base || 1047 id->gws_size != job->gws_size || 1048 id->oa_base != job->oa_base || 1049 id->oa_size != job->oa_size); 1050 1051 if (amdgpu_vmid_had_gpu_reset(adev, id)) 1052 return true; 1053 1054 return vm_flush_needed || gds_switch_needed; 1055 } 1056 1057 /** 1058 * amdgpu_vm_flush - hardware flush the vm 1059 * 1060 * @ring: ring to use for flush 1061 * @job: related job 1062 * @need_pipe_sync: is pipe sync needed 1063 * 1064 * Emit a VM flush when it is necessary. 1065 * 1066 * Returns: 1067 * 0 on success, errno otherwise. 1068 */ 1069 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, 1070 bool need_pipe_sync) 1071 { 1072 struct amdgpu_device *adev = ring->adev; 1073 unsigned vmhub = ring->funcs->vmhub; 1074 struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub]; 1075 struct amdgpu_vmid *id = &id_mgr->ids[job->vmid]; 1076 bool gds_switch_needed = ring->funcs->emit_gds_switch && ( 1077 id->gds_base != job->gds_base || 1078 id->gds_size != job->gds_size || 1079 id->gws_base != job->gws_base || 1080 id->gws_size != job->gws_size || 1081 id->oa_base != job->oa_base || 1082 id->oa_size != job->oa_size); 1083 bool vm_flush_needed = job->vm_needs_flush; 1084 struct dma_fence *fence = NULL; 1085 bool pasid_mapping_needed = false; 1086 unsigned patch_offset = 0; 1087 bool update_spm_vmid_needed = (job->vm && (job->vm->reserved_vmid[vmhub] != NULL)); 1088 int r; 1089 1090 if (update_spm_vmid_needed && adev->gfx.rlc.funcs->update_spm_vmid) 1091 adev->gfx.rlc.funcs->update_spm_vmid(adev, job->vmid); 1092 1093 if (amdgpu_vmid_had_gpu_reset(adev, id)) { 1094 gds_switch_needed = true; 1095 vm_flush_needed = true; 1096 pasid_mapping_needed = true; 1097 } 1098 1099 mutex_lock(&id_mgr->lock); 1100 if (id->pasid != job->pasid || !id->pasid_mapping || 1101 !dma_fence_is_signaled(id->pasid_mapping)) 1102 pasid_mapping_needed = true; 1103 mutex_unlock(&id_mgr->lock); 1104 1105 gds_switch_needed &= !!ring->funcs->emit_gds_switch; 1106 vm_flush_needed &= !!ring->funcs->emit_vm_flush && 1107 job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET; 1108 pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping && 1109 ring->funcs->emit_wreg; 1110 1111 if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync) 1112 return 0; 1113 1114 if (ring->funcs->init_cond_exec) 1115 patch_offset = amdgpu_ring_init_cond_exec(ring); 1116 1117 if (need_pipe_sync) 1118 amdgpu_ring_emit_pipeline_sync(ring); 1119 1120 if (vm_flush_needed) { 1121 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr); 1122 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr); 1123 } 1124 1125 if (pasid_mapping_needed) 1126 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid); 1127 1128 if (vm_flush_needed || pasid_mapping_needed) { 1129 r = amdgpu_fence_emit(ring, &fence, 0); 1130 if (r) 1131 return r; 1132 } 1133 1134 if (vm_flush_needed) { 1135 mutex_lock(&id_mgr->lock); 1136 dma_fence_put(id->last_flush); 1137 id->last_flush = dma_fence_get(fence); 1138 id->current_gpu_reset_count = 1139 atomic_read(&adev->gpu_reset_counter); 1140 mutex_unlock(&id_mgr->lock); 1141 } 1142 1143 if (pasid_mapping_needed) { 1144 mutex_lock(&id_mgr->lock); 1145 id->pasid = job->pasid; 1146 dma_fence_put(id->pasid_mapping); 1147 id->pasid_mapping = dma_fence_get(fence); 1148 mutex_unlock(&id_mgr->lock); 1149 } 1150 dma_fence_put(fence); 1151 1152 if (ring->funcs->emit_gds_switch && gds_switch_needed) { 1153 id->gds_base = job->gds_base; 1154 id->gds_size = job->gds_size; 1155 id->gws_base = job->gws_base; 1156 id->gws_size = job->gws_size; 1157 id->oa_base = job->oa_base; 1158 id->oa_size = job->oa_size; 1159 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base, 1160 job->gds_size, job->gws_base, 1161 job->gws_size, job->oa_base, 1162 job->oa_size); 1163 } 1164 1165 if (ring->funcs->patch_cond_exec) 1166 amdgpu_ring_patch_cond_exec(ring, patch_offset); 1167 1168 /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */ 1169 if (ring->funcs->emit_switch_buffer) { 1170 amdgpu_ring_emit_switch_buffer(ring); 1171 amdgpu_ring_emit_switch_buffer(ring); 1172 } 1173 return 0; 1174 } 1175 1176 /** 1177 * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo 1178 * 1179 * @vm: requested vm 1180 * @bo: requested buffer object 1181 * 1182 * Find @bo inside the requested vm. 1183 * Search inside the @bos vm list for the requested vm 1184 * Returns the found bo_va or NULL if none is found 1185 * 1186 * Object has to be reserved! 1187 * 1188 * Returns: 1189 * Found bo_va or NULL. 1190 */ 1191 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm, 1192 struct amdgpu_bo *bo) 1193 { 1194 struct amdgpu_vm_bo_base *base; 1195 1196 for (base = bo->vm_bo; base; base = base->next) { 1197 if (base->vm != vm) 1198 continue; 1199 1200 return container_of(base, struct amdgpu_bo_va, base); 1201 } 1202 return NULL; 1203 } 1204 1205 /** 1206 * amdgpu_vm_map_gart - Resolve gart mapping of addr 1207 * 1208 * @pages_addr: optional DMA address to use for lookup 1209 * @addr: the unmapped addr 1210 * 1211 * Look up the physical address of the page that the pte resolves 1212 * to. 1213 * 1214 * Returns: 1215 * The pointer for the page table entry. 1216 */ 1217 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr) 1218 { 1219 uint64_t result; 1220 1221 /* page table offset */ 1222 result = pages_addr[addr >> PAGE_SHIFT]; 1223 1224 /* in case cpu page size != gpu page size*/ 1225 result |= addr & (~PAGE_MASK); 1226 1227 result &= 0xFFFFFFFFFFFFF000ULL; 1228 1229 return result; 1230 } 1231 1232 /** 1233 * amdgpu_vm_update_pde - update a single level in the hierarchy 1234 * 1235 * @params: parameters for the update 1236 * @vm: requested vm 1237 * @entry: entry to update 1238 * 1239 * Makes sure the requested entry in parent is up to date. 1240 */ 1241 static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params, 1242 struct amdgpu_vm *vm, 1243 struct amdgpu_vm_pt *entry) 1244 { 1245 struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry); 1246 struct amdgpu_bo *bo = parent->base.bo, *pbo; 1247 uint64_t pde, pt, flags; 1248 unsigned level; 1249 1250 for (level = 0, pbo = bo->parent; pbo; ++level) 1251 pbo = pbo->parent; 1252 1253 level += params->adev->vm_manager.root_level; 1254 amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags); 1255 pde = (entry - parent->entries) * 8; 1256 return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags); 1257 } 1258 1259 /** 1260 * amdgpu_vm_invalidate_pds - mark all PDs as invalid 1261 * 1262 * @adev: amdgpu_device pointer 1263 * @vm: related vm 1264 * 1265 * Mark all PD level as invalid after an error. 1266 */ 1267 static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev, 1268 struct amdgpu_vm *vm) 1269 { 1270 struct amdgpu_vm_pt_cursor cursor; 1271 struct amdgpu_vm_pt *entry; 1272 1273 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) 1274 if (entry->base.bo && !entry->base.moved) 1275 amdgpu_vm_bo_relocated(&entry->base); 1276 } 1277 1278 /** 1279 * amdgpu_vm_update_pdes - make sure that all directories are valid 1280 * 1281 * @adev: amdgpu_device pointer 1282 * @vm: requested vm 1283 * @immediate: submit immediately to the paging queue 1284 * 1285 * Makes sure all directories are up to date. 1286 * 1287 * Returns: 1288 * 0 for success, error for failure. 1289 */ 1290 int amdgpu_vm_update_pdes(struct amdgpu_device *adev, 1291 struct amdgpu_vm *vm, bool immediate) 1292 { 1293 struct amdgpu_vm_update_params params; 1294 int r; 1295 1296 if (list_empty(&vm->relocated)) 1297 return 0; 1298 1299 memset(¶ms, 0, sizeof(params)); 1300 params.adev = adev; 1301 params.vm = vm; 1302 params.immediate = immediate; 1303 1304 r = vm->update_funcs->prepare(¶ms, NULL, AMDGPU_SYNC_EXPLICIT); 1305 if (r) 1306 return r; 1307 1308 while (!list_empty(&vm->relocated)) { 1309 struct amdgpu_vm_pt *entry; 1310 1311 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt, 1312 base.vm_status); 1313 amdgpu_vm_bo_idle(&entry->base); 1314 1315 r = amdgpu_vm_update_pde(¶ms, vm, entry); 1316 if (r) 1317 goto error; 1318 } 1319 1320 r = vm->update_funcs->commit(¶ms, &vm->last_update); 1321 if (r) 1322 goto error; 1323 return 0; 1324 1325 error: 1326 amdgpu_vm_invalidate_pds(adev, vm); 1327 return r; 1328 } 1329 1330 /* 1331 * amdgpu_vm_update_flags - figure out flags for PTE updates 1332 * 1333 * Make sure to set the right flags for the PTEs at the desired level. 1334 */ 1335 static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params, 1336 struct amdgpu_bo *bo, unsigned level, 1337 uint64_t pe, uint64_t addr, 1338 unsigned count, uint32_t incr, 1339 uint64_t flags) 1340 1341 { 1342 if (level != AMDGPU_VM_PTB) { 1343 flags |= AMDGPU_PDE_PTE; 1344 amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags); 1345 1346 } else if (params->adev->asic_type >= CHIP_VEGA10 && 1347 !(flags & AMDGPU_PTE_VALID) && 1348 !(flags & AMDGPU_PTE_PRT)) { 1349 1350 /* Workaround for fault priority problem on GMC9 */ 1351 flags |= AMDGPU_PTE_EXECUTABLE; 1352 } 1353 1354 params->vm->update_funcs->update(params, bo, pe, addr, count, incr, 1355 flags); 1356 } 1357 1358 /** 1359 * amdgpu_vm_fragment - get fragment for PTEs 1360 * 1361 * @params: see amdgpu_vm_update_params definition 1362 * @start: first PTE to handle 1363 * @end: last PTE to handle 1364 * @flags: hw mapping flags 1365 * @frag: resulting fragment size 1366 * @frag_end: end of this fragment 1367 * 1368 * Returns the first possible fragment for the start and end address. 1369 */ 1370 static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params, 1371 uint64_t start, uint64_t end, uint64_t flags, 1372 unsigned int *frag, uint64_t *frag_end) 1373 { 1374 /** 1375 * The MC L1 TLB supports variable sized pages, based on a fragment 1376 * field in the PTE. When this field is set to a non-zero value, page 1377 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 1378 * flags are considered valid for all PTEs within the fragment range 1379 * and corresponding mappings are assumed to be physically contiguous. 1380 * 1381 * The L1 TLB can store a single PTE for the whole fragment, 1382 * significantly increasing the space available for translation 1383 * caching. This leads to large improvements in throughput when the 1384 * TLB is under pressure. 1385 * 1386 * The L2 TLB distributes small and large fragments into two 1387 * asymmetric partitions. The large fragment cache is significantly 1388 * larger. Thus, we try to use large fragments wherever possible. 1389 * Userspace can support this by aligning virtual base address and 1390 * allocation size to the fragment size. 1391 * 1392 * Starting with Vega10 the fragment size only controls the L1. The L2 1393 * is now directly feed with small/huge/giant pages from the walker. 1394 */ 1395 unsigned max_frag; 1396 1397 if (params->adev->asic_type < CHIP_VEGA10) 1398 max_frag = params->adev->vm_manager.fragment_size; 1399 else 1400 max_frag = 31; 1401 1402 /* system pages are non continuously */ 1403 if (params->pages_addr) { 1404 *frag = 0; 1405 *frag_end = end; 1406 return; 1407 } 1408 1409 /* This intentionally wraps around if no bit is set */ 1410 *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1); 1411 if (*frag >= max_frag) { 1412 *frag = max_frag; 1413 *frag_end = end & ~((1ULL << max_frag) - 1); 1414 } else { 1415 *frag_end = start + (1 << *frag); 1416 } 1417 } 1418 1419 /** 1420 * amdgpu_vm_update_ptes - make sure that page tables are valid 1421 * 1422 * @params: see amdgpu_vm_update_params definition 1423 * @start: start of GPU address range 1424 * @end: end of GPU address range 1425 * @dst: destination address to map to, the next dst inside the function 1426 * @flags: mapping flags 1427 * 1428 * Update the page tables in the range @start - @end. 1429 * 1430 * Returns: 1431 * 0 for success, -EINVAL for failure. 1432 */ 1433 static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, 1434 uint64_t start, uint64_t end, 1435 uint64_t dst, uint64_t flags) 1436 { 1437 struct amdgpu_device *adev = params->adev; 1438 struct amdgpu_vm_pt_cursor cursor; 1439 uint64_t frag_start = start, frag_end; 1440 unsigned int frag; 1441 int r; 1442 1443 /* figure out the initial fragment */ 1444 amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end); 1445 1446 /* walk over the address space and update the PTs */ 1447 amdgpu_vm_pt_start(adev, params->vm, start, &cursor); 1448 while (cursor.pfn < end) { 1449 unsigned shift, parent_shift, mask; 1450 uint64_t incr, entry_end, pe_start; 1451 struct amdgpu_bo *pt; 1452 1453 if (!params->unlocked) { 1454 /* make sure that the page tables covering the 1455 * address range are actually allocated 1456 */ 1457 r = amdgpu_vm_alloc_pts(params->adev, params->vm, 1458 &cursor, params->immediate); 1459 if (r) 1460 return r; 1461 } 1462 1463 shift = amdgpu_vm_level_shift(adev, cursor.level); 1464 parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1); 1465 if (params->unlocked) { 1466 /* Unlocked updates are only allowed on the leaves */ 1467 if (amdgpu_vm_pt_descendant(adev, &cursor)) 1468 continue; 1469 } else if (adev->asic_type < CHIP_VEGA10 && 1470 (flags & AMDGPU_PTE_VALID)) { 1471 /* No huge page support before GMC v9 */ 1472 if (cursor.level != AMDGPU_VM_PTB) { 1473 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 1474 return -ENOENT; 1475 continue; 1476 } 1477 } else if (frag < shift) { 1478 /* We can't use this level when the fragment size is 1479 * smaller than the address shift. Go to the next 1480 * child entry and try again. 1481 */ 1482 if (amdgpu_vm_pt_descendant(adev, &cursor)) 1483 continue; 1484 } else if (frag >= parent_shift) { 1485 /* If the fragment size is even larger than the parent 1486 * shift we should go up one level and check it again. 1487 */ 1488 if (!amdgpu_vm_pt_ancestor(&cursor)) 1489 return -EINVAL; 1490 continue; 1491 } 1492 1493 pt = cursor.entry->base.bo; 1494 if (!pt) { 1495 /* We need all PDs and PTs for mapping something, */ 1496 if (flags & AMDGPU_PTE_VALID) 1497 return -ENOENT; 1498 1499 /* but unmapping something can happen at a higher 1500 * level. 1501 */ 1502 if (!amdgpu_vm_pt_ancestor(&cursor)) 1503 return -EINVAL; 1504 1505 pt = cursor.entry->base.bo; 1506 shift = parent_shift; 1507 frag_end = max(frag_end, ALIGN(frag_start + 1, 1508 1ULL << shift)); 1509 } 1510 1511 /* Looks good so far, calculate parameters for the update */ 1512 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift; 1513 mask = amdgpu_vm_entries_mask(adev, cursor.level); 1514 pe_start = ((cursor.pfn >> shift) & mask) * 8; 1515 entry_end = ((uint64_t)mask + 1) << shift; 1516 entry_end += cursor.pfn & ~(entry_end - 1); 1517 entry_end = min(entry_end, end); 1518 1519 do { 1520 struct amdgpu_vm *vm = params->vm; 1521 uint64_t upd_end = min(entry_end, frag_end); 1522 unsigned nptes = (upd_end - frag_start) >> shift; 1523 uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag); 1524 1525 /* This can happen when we set higher level PDs to 1526 * silent to stop fault floods. 1527 */ 1528 nptes = max(nptes, 1u); 1529 1530 trace_amdgpu_vm_update_ptes(params, frag_start, upd_end, 1531 nptes, dst, incr, upd_flags, 1532 vm->task_info.pid, 1533 vm->immediate.fence_context); 1534 amdgpu_vm_update_flags(params, pt, cursor.level, 1535 pe_start, dst, nptes, incr, 1536 upd_flags); 1537 1538 pe_start += nptes * 8; 1539 dst += nptes * incr; 1540 1541 frag_start = upd_end; 1542 if (frag_start >= frag_end) { 1543 /* figure out the next fragment */ 1544 amdgpu_vm_fragment(params, frag_start, end, 1545 flags, &frag, &frag_end); 1546 if (frag < shift) 1547 break; 1548 } 1549 } while (frag_start < entry_end); 1550 1551 if (amdgpu_vm_pt_descendant(adev, &cursor)) { 1552 /* Free all child entries. 1553 * Update the tables with the flags and addresses and free up subsequent 1554 * tables in the case of huge pages or freed up areas. 1555 * This is the maximum you can free, because all other page tables are not 1556 * completely covered by the range and so potentially still in use. 1557 */ 1558 while (cursor.pfn < frag_start) { 1559 amdgpu_vm_free_pts(adev, params->vm, &cursor); 1560 amdgpu_vm_pt_next(adev, &cursor); 1561 } 1562 1563 } else if (frag >= shift) { 1564 /* or just move on to the next on the same level. */ 1565 amdgpu_vm_pt_next(adev, &cursor); 1566 } 1567 } 1568 1569 return 0; 1570 } 1571 1572 /** 1573 * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table 1574 * 1575 * @adev: amdgpu_device pointer of the VM 1576 * @bo_adev: amdgpu_device pointer of the mapped BO 1577 * @vm: requested vm 1578 * @immediate: immediate submission in a page fault 1579 * @unlocked: unlocked invalidation during MM callback 1580 * @resv: fences we need to sync to 1581 * @start: start of mapped range 1582 * @last: last mapped entry 1583 * @flags: flags for the entries 1584 * @offset: offset into nodes and pages_addr 1585 * @nodes: array of drm_mm_nodes with the MC addresses 1586 * @pages_addr: DMA addresses to use for mapping 1587 * @fence: optional resulting fence 1588 * 1589 * Fill in the page table entries between @start and @last. 1590 * 1591 * Returns: 1592 * 0 for success, -EINVAL for failure. 1593 */ 1594 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev, 1595 struct amdgpu_device *bo_adev, 1596 struct amdgpu_vm *vm, bool immediate, 1597 bool unlocked, struct dma_resv *resv, 1598 uint64_t start, uint64_t last, 1599 uint64_t flags, uint64_t offset, 1600 struct drm_mm_node *nodes, 1601 dma_addr_t *pages_addr, 1602 struct dma_fence **fence) 1603 { 1604 struct amdgpu_vm_update_params params; 1605 enum amdgpu_sync_mode sync_mode; 1606 uint64_t pfn; 1607 int r; 1608 1609 memset(¶ms, 0, sizeof(params)); 1610 params.adev = adev; 1611 params.vm = vm; 1612 params.immediate = immediate; 1613 params.pages_addr = pages_addr; 1614 params.unlocked = unlocked; 1615 1616 /* Implicitly sync to command submissions in the same VM before 1617 * unmapping. Sync to moving fences before mapping. 1618 */ 1619 if (!(flags & AMDGPU_PTE_VALID)) 1620 sync_mode = AMDGPU_SYNC_EQ_OWNER; 1621 else 1622 sync_mode = AMDGPU_SYNC_EXPLICIT; 1623 1624 pfn = offset >> PAGE_SHIFT; 1625 if (nodes) { 1626 while (pfn >= nodes->size) { 1627 pfn -= nodes->size; 1628 ++nodes; 1629 } 1630 } 1631 1632 amdgpu_vm_eviction_lock(vm); 1633 if (vm->evicting) { 1634 r = -EBUSY; 1635 goto error_unlock; 1636 } 1637 1638 if (!unlocked && !dma_fence_is_signaled(vm->last_unlocked)) { 1639 struct dma_fence *tmp = dma_fence_get_stub(); 1640 1641 amdgpu_bo_fence(vm->root.base.bo, vm->last_unlocked, true); 1642 swap(vm->last_unlocked, tmp); 1643 dma_fence_put(tmp); 1644 } 1645 1646 r = vm->update_funcs->prepare(¶ms, resv, sync_mode); 1647 if (r) 1648 goto error_unlock; 1649 1650 do { 1651 uint64_t tmp, num_entries, addr; 1652 1653 1654 num_entries = last - start + 1; 1655 if (nodes) { 1656 addr = nodes->start << PAGE_SHIFT; 1657 num_entries = min((nodes->size - pfn) * 1658 AMDGPU_GPU_PAGES_IN_CPU_PAGE, num_entries); 1659 } else { 1660 addr = 0; 1661 } 1662 1663 if (pages_addr) { 1664 bool contiguous = true; 1665 1666 if (num_entries > AMDGPU_GPU_PAGES_IN_CPU_PAGE) { 1667 uint64_t count; 1668 1669 contiguous = pages_addr[pfn + 1] == 1670 pages_addr[pfn] + PAGE_SIZE; 1671 1672 tmp = num_entries / 1673 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1674 for (count = 2; count < tmp; ++count) { 1675 uint64_t idx = pfn + count; 1676 1677 if (contiguous != (pages_addr[idx] == 1678 pages_addr[idx - 1] + PAGE_SIZE)) 1679 break; 1680 } 1681 num_entries = count * 1682 AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1683 } 1684 1685 if (!contiguous) { 1686 addr = pfn << PAGE_SHIFT; 1687 params.pages_addr = pages_addr; 1688 } else { 1689 addr = pages_addr[pfn]; 1690 params.pages_addr = NULL; 1691 } 1692 1693 } else if (flags & (AMDGPU_PTE_VALID | AMDGPU_PTE_PRT)) { 1694 addr += bo_adev->vm_manager.vram_base_offset; 1695 addr += pfn << PAGE_SHIFT; 1696 } 1697 1698 tmp = start + num_entries; 1699 r = amdgpu_vm_update_ptes(¶ms, start, tmp, addr, flags); 1700 if (r) 1701 goto error_unlock; 1702 1703 pfn += num_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE; 1704 if (nodes && nodes->size == pfn) { 1705 pfn = 0; 1706 ++nodes; 1707 } 1708 start = tmp; 1709 1710 } while (unlikely(start != last + 1)); 1711 1712 r = vm->update_funcs->commit(¶ms, fence); 1713 1714 error_unlock: 1715 amdgpu_vm_eviction_unlock(vm); 1716 return r; 1717 } 1718 1719 /** 1720 * amdgpu_vm_bo_update - update all BO mappings in the vm page table 1721 * 1722 * @adev: amdgpu_device pointer 1723 * @bo_va: requested BO and VM object 1724 * @clear: if true clear the entries 1725 * 1726 * Fill in the page table entries for @bo_va. 1727 * 1728 * Returns: 1729 * 0 for success, -EINVAL for failure. 1730 */ 1731 int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va, 1732 bool clear) 1733 { 1734 struct amdgpu_bo *bo = bo_va->base.bo; 1735 struct amdgpu_vm *vm = bo_va->base.vm; 1736 struct amdgpu_bo_va_mapping *mapping; 1737 dma_addr_t *pages_addr = NULL; 1738 struct ttm_resource *mem; 1739 struct drm_mm_node *nodes; 1740 struct dma_fence **last_update; 1741 struct dma_resv *resv; 1742 uint64_t flags; 1743 struct amdgpu_device *bo_adev = adev; 1744 int r; 1745 1746 if (clear || !bo) { 1747 mem = NULL; 1748 nodes = NULL; 1749 resv = vm->root.base.bo->tbo.base.resv; 1750 } else { 1751 struct drm_gem_object *obj = &bo->tbo.base; 1752 1753 resv = bo->tbo.base.resv; 1754 if (obj->import_attach && bo_va->is_xgmi) { 1755 struct dma_buf *dma_buf = obj->import_attach->dmabuf; 1756 struct drm_gem_object *gobj = dma_buf->priv; 1757 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); 1758 1759 if (abo->tbo.mem.mem_type == TTM_PL_VRAM) 1760 bo = gem_to_amdgpu_bo(gobj); 1761 } 1762 mem = &bo->tbo.mem; 1763 nodes = mem->mm_node; 1764 if (mem->mem_type == TTM_PL_TT) 1765 pages_addr = bo->tbo.ttm->dma_address; 1766 } 1767 1768 if (bo) { 1769 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem); 1770 1771 if (amdgpu_bo_encrypted(bo)) 1772 flags |= AMDGPU_PTE_TMZ; 1773 1774 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 1775 } else { 1776 flags = 0x0; 1777 } 1778 1779 if (clear || (bo && bo->tbo.base.resv == 1780 vm->root.base.bo->tbo.base.resv)) 1781 last_update = &vm->last_update; 1782 else 1783 last_update = &bo_va->last_pt_update; 1784 1785 if (!clear && bo_va->base.moved) { 1786 bo_va->base.moved = false; 1787 list_splice_init(&bo_va->valids, &bo_va->invalids); 1788 1789 } else if (bo_va->cleared != clear) { 1790 list_splice_init(&bo_va->valids, &bo_va->invalids); 1791 } 1792 1793 list_for_each_entry(mapping, &bo_va->invalids, list) { 1794 uint64_t update_flags = flags; 1795 1796 /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here 1797 * but in case of something, we filter the flags in first place 1798 */ 1799 if (!(mapping->flags & AMDGPU_PTE_READABLE)) 1800 update_flags &= ~AMDGPU_PTE_READABLE; 1801 if (!(mapping->flags & AMDGPU_PTE_WRITEABLE)) 1802 update_flags &= ~AMDGPU_PTE_WRITEABLE; 1803 1804 /* Apply ASIC specific mapping flags */ 1805 amdgpu_gmc_get_vm_pte(adev, mapping, &update_flags); 1806 1807 trace_amdgpu_vm_bo_update(mapping); 1808 1809 r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false, 1810 resv, mapping->start, 1811 mapping->last, update_flags, 1812 mapping->offset, nodes, 1813 pages_addr, last_update); 1814 if (r) 1815 return r; 1816 } 1817 1818 /* If the BO is not in its preferred location add it back to 1819 * the evicted list so that it gets validated again on the 1820 * next command submission. 1821 */ 1822 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { 1823 uint32_t mem_type = bo->tbo.mem.mem_type; 1824 1825 if (!(bo->preferred_domains & 1826 amdgpu_mem_type_to_domain(mem_type))) 1827 amdgpu_vm_bo_evicted(&bo_va->base); 1828 else 1829 amdgpu_vm_bo_idle(&bo_va->base); 1830 } else { 1831 amdgpu_vm_bo_done(&bo_va->base); 1832 } 1833 1834 list_splice_init(&bo_va->invalids, &bo_va->valids); 1835 bo_va->cleared = clear; 1836 1837 if (trace_amdgpu_vm_bo_mapping_enabled()) { 1838 list_for_each_entry(mapping, &bo_va->valids, list) 1839 trace_amdgpu_vm_bo_mapping(mapping); 1840 } 1841 1842 return 0; 1843 } 1844 1845 /** 1846 * amdgpu_vm_update_prt_state - update the global PRT state 1847 * 1848 * @adev: amdgpu_device pointer 1849 */ 1850 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev) 1851 { 1852 unsigned long flags; 1853 bool enable; 1854 1855 spin_lock_irqsave(&adev->vm_manager.prt_lock, flags); 1856 enable = !!atomic_read(&adev->vm_manager.num_prt_users); 1857 adev->gmc.gmc_funcs->set_prt(adev, enable); 1858 spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags); 1859 } 1860 1861 /** 1862 * amdgpu_vm_prt_get - add a PRT user 1863 * 1864 * @adev: amdgpu_device pointer 1865 */ 1866 static void amdgpu_vm_prt_get(struct amdgpu_device *adev) 1867 { 1868 if (!adev->gmc.gmc_funcs->set_prt) 1869 return; 1870 1871 if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1) 1872 amdgpu_vm_update_prt_state(adev); 1873 } 1874 1875 /** 1876 * amdgpu_vm_prt_put - drop a PRT user 1877 * 1878 * @adev: amdgpu_device pointer 1879 */ 1880 static void amdgpu_vm_prt_put(struct amdgpu_device *adev) 1881 { 1882 if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0) 1883 amdgpu_vm_update_prt_state(adev); 1884 } 1885 1886 /** 1887 * amdgpu_vm_prt_cb - callback for updating the PRT status 1888 * 1889 * @fence: fence for the callback 1890 * @_cb: the callback function 1891 */ 1892 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb) 1893 { 1894 struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb); 1895 1896 amdgpu_vm_prt_put(cb->adev); 1897 kfree(cb); 1898 } 1899 1900 /** 1901 * amdgpu_vm_add_prt_cb - add callback for updating the PRT status 1902 * 1903 * @adev: amdgpu_device pointer 1904 * @fence: fence for the callback 1905 */ 1906 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev, 1907 struct dma_fence *fence) 1908 { 1909 struct amdgpu_prt_cb *cb; 1910 1911 if (!adev->gmc.gmc_funcs->set_prt) 1912 return; 1913 1914 cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL); 1915 if (!cb) { 1916 /* Last resort when we are OOM */ 1917 if (fence) 1918 dma_fence_wait(fence, false); 1919 1920 amdgpu_vm_prt_put(adev); 1921 } else { 1922 cb->adev = adev; 1923 if (!fence || dma_fence_add_callback(fence, &cb->cb, 1924 amdgpu_vm_prt_cb)) 1925 amdgpu_vm_prt_cb(fence, &cb->cb); 1926 } 1927 } 1928 1929 /** 1930 * amdgpu_vm_free_mapping - free a mapping 1931 * 1932 * @adev: amdgpu_device pointer 1933 * @vm: requested vm 1934 * @mapping: mapping to be freed 1935 * @fence: fence of the unmap operation 1936 * 1937 * Free a mapping and make sure we decrease the PRT usage count if applicable. 1938 */ 1939 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev, 1940 struct amdgpu_vm *vm, 1941 struct amdgpu_bo_va_mapping *mapping, 1942 struct dma_fence *fence) 1943 { 1944 if (mapping->flags & AMDGPU_PTE_PRT) 1945 amdgpu_vm_add_prt_cb(adev, fence); 1946 kfree(mapping); 1947 } 1948 1949 /** 1950 * amdgpu_vm_prt_fini - finish all prt mappings 1951 * 1952 * @adev: amdgpu_device pointer 1953 * @vm: requested vm 1954 * 1955 * Register a cleanup callback to disable PRT support after VM dies. 1956 */ 1957 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 1958 { 1959 struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; 1960 struct dma_fence *excl, **shared; 1961 unsigned i, shared_count; 1962 int r; 1963 1964 r = dma_resv_get_fences_rcu(resv, &excl, 1965 &shared_count, &shared); 1966 if (r) { 1967 /* Not enough memory to grab the fence list, as last resort 1968 * block for all the fences to complete. 1969 */ 1970 dma_resv_wait_timeout_rcu(resv, true, false, 1971 MAX_SCHEDULE_TIMEOUT); 1972 return; 1973 } 1974 1975 /* Add a callback for each fence in the reservation object */ 1976 amdgpu_vm_prt_get(adev); 1977 amdgpu_vm_add_prt_cb(adev, excl); 1978 1979 for (i = 0; i < shared_count; ++i) { 1980 amdgpu_vm_prt_get(adev); 1981 amdgpu_vm_add_prt_cb(adev, shared[i]); 1982 } 1983 1984 kfree(shared); 1985 } 1986 1987 /** 1988 * amdgpu_vm_clear_freed - clear freed BOs in the PT 1989 * 1990 * @adev: amdgpu_device pointer 1991 * @vm: requested vm 1992 * @fence: optional resulting fence (unchanged if no work needed to be done 1993 * or if an error occurred) 1994 * 1995 * Make sure all freed BOs are cleared in the PT. 1996 * PTs have to be reserved and mutex must be locked! 1997 * 1998 * Returns: 1999 * 0 for success. 2000 * 2001 */ 2002 int amdgpu_vm_clear_freed(struct amdgpu_device *adev, 2003 struct amdgpu_vm *vm, 2004 struct dma_fence **fence) 2005 { 2006 struct dma_resv *resv = vm->root.base.bo->tbo.base.resv; 2007 struct amdgpu_bo_va_mapping *mapping; 2008 uint64_t init_pte_value = 0; 2009 struct dma_fence *f = NULL; 2010 int r; 2011 2012 while (!list_empty(&vm->freed)) { 2013 mapping = list_first_entry(&vm->freed, 2014 struct amdgpu_bo_va_mapping, list); 2015 list_del(&mapping->list); 2016 2017 if (vm->pte_support_ats && 2018 mapping->start < AMDGPU_GMC_HOLE_START) 2019 init_pte_value = AMDGPU_PTE_DEFAULT_ATC; 2020 2021 r = amdgpu_vm_bo_update_mapping(adev, adev, vm, false, false, 2022 resv, mapping->start, 2023 mapping->last, init_pte_value, 2024 0, NULL, NULL, &f); 2025 amdgpu_vm_free_mapping(adev, vm, mapping, f); 2026 if (r) { 2027 dma_fence_put(f); 2028 return r; 2029 } 2030 } 2031 2032 if (fence && f) { 2033 dma_fence_put(*fence); 2034 *fence = f; 2035 } else { 2036 dma_fence_put(f); 2037 } 2038 2039 return 0; 2040 2041 } 2042 2043 /** 2044 * amdgpu_vm_handle_moved - handle moved BOs in the PT 2045 * 2046 * @adev: amdgpu_device pointer 2047 * @vm: requested vm 2048 * 2049 * Make sure all BOs which are moved are updated in the PTs. 2050 * 2051 * Returns: 2052 * 0 for success. 2053 * 2054 * PTs have to be reserved! 2055 */ 2056 int amdgpu_vm_handle_moved(struct amdgpu_device *adev, 2057 struct amdgpu_vm *vm) 2058 { 2059 struct amdgpu_bo_va *bo_va, *tmp; 2060 struct dma_resv *resv; 2061 bool clear; 2062 int r; 2063 2064 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { 2065 /* Per VM BOs never need to bo cleared in the page tables */ 2066 r = amdgpu_vm_bo_update(adev, bo_va, false); 2067 if (r) 2068 return r; 2069 } 2070 2071 spin_lock(&vm->invalidated_lock); 2072 while (!list_empty(&vm->invalidated)) { 2073 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va, 2074 base.vm_status); 2075 resv = bo_va->base.bo->tbo.base.resv; 2076 spin_unlock(&vm->invalidated_lock); 2077 2078 /* Try to reserve the BO to avoid clearing its ptes */ 2079 if (!amdgpu_vm_debug && dma_resv_trylock(resv)) 2080 clear = false; 2081 /* Somebody else is using the BO right now */ 2082 else 2083 clear = true; 2084 2085 r = amdgpu_vm_bo_update(adev, bo_va, clear); 2086 if (r) 2087 return r; 2088 2089 if (!clear) 2090 dma_resv_unlock(resv); 2091 spin_lock(&vm->invalidated_lock); 2092 } 2093 spin_unlock(&vm->invalidated_lock); 2094 2095 return 0; 2096 } 2097 2098 /** 2099 * amdgpu_vm_bo_add - add a bo to a specific vm 2100 * 2101 * @adev: amdgpu_device pointer 2102 * @vm: requested vm 2103 * @bo: amdgpu buffer object 2104 * 2105 * Add @bo into the requested vm. 2106 * Add @bo to the list of bos associated with the vm 2107 * 2108 * Returns: 2109 * Newly added bo_va or NULL for failure 2110 * 2111 * Object has to be reserved! 2112 */ 2113 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev, 2114 struct amdgpu_vm *vm, 2115 struct amdgpu_bo *bo) 2116 { 2117 struct amdgpu_bo_va *bo_va; 2118 2119 bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL); 2120 if (bo_va == NULL) { 2121 return NULL; 2122 } 2123 amdgpu_vm_bo_base_init(&bo_va->base, vm, bo); 2124 2125 bo_va->ref_count = 1; 2126 INIT_LIST_HEAD(&bo_va->valids); 2127 INIT_LIST_HEAD(&bo_va->invalids); 2128 2129 if (!bo) 2130 return bo_va; 2131 2132 if (amdgpu_dmabuf_is_xgmi_accessible(adev, bo)) { 2133 bo_va->is_xgmi = true; 2134 /* Power up XGMI if it can be potentially used */ 2135 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MAX_VEGA20); 2136 } 2137 2138 return bo_va; 2139 } 2140 2141 2142 /** 2143 * amdgpu_vm_bo_insert_map - insert a new mapping 2144 * 2145 * @adev: amdgpu_device pointer 2146 * @bo_va: bo_va to store the address 2147 * @mapping: the mapping to insert 2148 * 2149 * Insert a new mapping into all structures. 2150 */ 2151 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev, 2152 struct amdgpu_bo_va *bo_va, 2153 struct amdgpu_bo_va_mapping *mapping) 2154 { 2155 struct amdgpu_vm *vm = bo_va->base.vm; 2156 struct amdgpu_bo *bo = bo_va->base.bo; 2157 2158 mapping->bo_va = bo_va; 2159 list_add(&mapping->list, &bo_va->invalids); 2160 amdgpu_vm_it_insert(mapping, &vm->va); 2161 2162 if (mapping->flags & AMDGPU_PTE_PRT) 2163 amdgpu_vm_prt_get(adev); 2164 2165 if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv && 2166 !bo_va->base.moved) { 2167 list_move(&bo_va->base.vm_status, &vm->moved); 2168 } 2169 trace_amdgpu_vm_bo_map(bo_va, mapping); 2170 } 2171 2172 /** 2173 * amdgpu_vm_bo_map - map bo inside a vm 2174 * 2175 * @adev: amdgpu_device pointer 2176 * @bo_va: bo_va to store the address 2177 * @saddr: where to map the BO 2178 * @offset: requested offset in the BO 2179 * @size: BO size in bytes 2180 * @flags: attributes of pages (read/write/valid/etc.) 2181 * 2182 * Add a mapping of the BO at the specefied addr into the VM. 2183 * 2184 * Returns: 2185 * 0 for success, error for failure. 2186 * 2187 * Object has to be reserved and unreserved outside! 2188 */ 2189 int amdgpu_vm_bo_map(struct amdgpu_device *adev, 2190 struct amdgpu_bo_va *bo_va, 2191 uint64_t saddr, uint64_t offset, 2192 uint64_t size, uint64_t flags) 2193 { 2194 struct amdgpu_bo_va_mapping *mapping, *tmp; 2195 struct amdgpu_bo *bo = bo_va->base.bo; 2196 struct amdgpu_vm *vm = bo_va->base.vm; 2197 uint64_t eaddr; 2198 2199 /* validate the parameters */ 2200 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 2201 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 2202 return -EINVAL; 2203 2204 /* make sure object fit at this offset */ 2205 eaddr = saddr + size - 1; 2206 if (saddr >= eaddr || 2207 (bo && offset + size > amdgpu_bo_size(bo)) || 2208 (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT)) 2209 return -EINVAL; 2210 2211 saddr /= AMDGPU_GPU_PAGE_SIZE; 2212 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2213 2214 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2215 if (tmp) { 2216 /* bo and tmp overlap, invalid addr */ 2217 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with " 2218 "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr, 2219 tmp->start, tmp->last + 1); 2220 return -EINVAL; 2221 } 2222 2223 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2224 if (!mapping) 2225 return -ENOMEM; 2226 2227 mapping->start = saddr; 2228 mapping->last = eaddr; 2229 mapping->offset = offset; 2230 mapping->flags = flags; 2231 2232 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 2233 2234 return 0; 2235 } 2236 2237 /** 2238 * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings 2239 * 2240 * @adev: amdgpu_device pointer 2241 * @bo_va: bo_va to store the address 2242 * @saddr: where to map the BO 2243 * @offset: requested offset in the BO 2244 * @size: BO size in bytes 2245 * @flags: attributes of pages (read/write/valid/etc.) 2246 * 2247 * Add a mapping of the BO at the specefied addr into the VM. Replace existing 2248 * mappings as we do so. 2249 * 2250 * Returns: 2251 * 0 for success, error for failure. 2252 * 2253 * Object has to be reserved and unreserved outside! 2254 */ 2255 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev, 2256 struct amdgpu_bo_va *bo_va, 2257 uint64_t saddr, uint64_t offset, 2258 uint64_t size, uint64_t flags) 2259 { 2260 struct amdgpu_bo_va_mapping *mapping; 2261 struct amdgpu_bo *bo = bo_va->base.bo; 2262 uint64_t eaddr; 2263 int r; 2264 2265 /* validate the parameters */ 2266 if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK || 2267 size == 0 || size & AMDGPU_GPU_PAGE_MASK) 2268 return -EINVAL; 2269 2270 /* make sure object fit at this offset */ 2271 eaddr = saddr + size - 1; 2272 if (saddr >= eaddr || 2273 (bo && offset + size > amdgpu_bo_size(bo)) || 2274 (eaddr >= adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT)) 2275 return -EINVAL; 2276 2277 /* Allocate all the needed memory */ 2278 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2279 if (!mapping) 2280 return -ENOMEM; 2281 2282 r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size); 2283 if (r) { 2284 kfree(mapping); 2285 return r; 2286 } 2287 2288 saddr /= AMDGPU_GPU_PAGE_SIZE; 2289 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2290 2291 mapping->start = saddr; 2292 mapping->last = eaddr; 2293 mapping->offset = offset; 2294 mapping->flags = flags; 2295 2296 amdgpu_vm_bo_insert_map(adev, bo_va, mapping); 2297 2298 return 0; 2299 } 2300 2301 /** 2302 * amdgpu_vm_bo_unmap - remove bo mapping from vm 2303 * 2304 * @adev: amdgpu_device pointer 2305 * @bo_va: bo_va to remove the address from 2306 * @saddr: where to the BO is mapped 2307 * 2308 * Remove a mapping of the BO at the specefied addr from the VM. 2309 * 2310 * Returns: 2311 * 0 for success, error for failure. 2312 * 2313 * Object has to be reserved and unreserved outside! 2314 */ 2315 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev, 2316 struct amdgpu_bo_va *bo_va, 2317 uint64_t saddr) 2318 { 2319 struct amdgpu_bo_va_mapping *mapping; 2320 struct amdgpu_vm *vm = bo_va->base.vm; 2321 bool valid = true; 2322 2323 saddr /= AMDGPU_GPU_PAGE_SIZE; 2324 2325 list_for_each_entry(mapping, &bo_va->valids, list) { 2326 if (mapping->start == saddr) 2327 break; 2328 } 2329 2330 if (&mapping->list == &bo_va->valids) { 2331 valid = false; 2332 2333 list_for_each_entry(mapping, &bo_va->invalids, list) { 2334 if (mapping->start == saddr) 2335 break; 2336 } 2337 2338 if (&mapping->list == &bo_va->invalids) 2339 return -ENOENT; 2340 } 2341 2342 list_del(&mapping->list); 2343 amdgpu_vm_it_remove(mapping, &vm->va); 2344 mapping->bo_va = NULL; 2345 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2346 2347 if (valid) 2348 list_add(&mapping->list, &vm->freed); 2349 else 2350 amdgpu_vm_free_mapping(adev, vm, mapping, 2351 bo_va->last_pt_update); 2352 2353 return 0; 2354 } 2355 2356 /** 2357 * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range 2358 * 2359 * @adev: amdgpu_device pointer 2360 * @vm: VM structure to use 2361 * @saddr: start of the range 2362 * @size: size of the range 2363 * 2364 * Remove all mappings in a range, split them as appropriate. 2365 * 2366 * Returns: 2367 * 0 for success, error for failure. 2368 */ 2369 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev, 2370 struct amdgpu_vm *vm, 2371 uint64_t saddr, uint64_t size) 2372 { 2373 struct amdgpu_bo_va_mapping *before, *after, *tmp, *next; 2374 LIST_HEAD(removed); 2375 uint64_t eaddr; 2376 2377 eaddr = saddr + size - 1; 2378 saddr /= AMDGPU_GPU_PAGE_SIZE; 2379 eaddr /= AMDGPU_GPU_PAGE_SIZE; 2380 2381 /* Allocate all the needed memory */ 2382 before = kzalloc(sizeof(*before), GFP_KERNEL); 2383 if (!before) 2384 return -ENOMEM; 2385 INIT_LIST_HEAD(&before->list); 2386 2387 after = kzalloc(sizeof(*after), GFP_KERNEL); 2388 if (!after) { 2389 kfree(before); 2390 return -ENOMEM; 2391 } 2392 INIT_LIST_HEAD(&after->list); 2393 2394 /* Now gather all removed mappings */ 2395 tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr); 2396 while (tmp) { 2397 /* Remember mapping split at the start */ 2398 if (tmp->start < saddr) { 2399 before->start = tmp->start; 2400 before->last = saddr - 1; 2401 before->offset = tmp->offset; 2402 before->flags = tmp->flags; 2403 before->bo_va = tmp->bo_va; 2404 list_add(&before->list, &tmp->bo_va->invalids); 2405 } 2406 2407 /* Remember mapping split at the end */ 2408 if (tmp->last > eaddr) { 2409 after->start = eaddr + 1; 2410 after->last = tmp->last; 2411 after->offset = tmp->offset; 2412 after->offset += after->start - tmp->start; 2413 after->flags = tmp->flags; 2414 after->bo_va = tmp->bo_va; 2415 list_add(&after->list, &tmp->bo_va->invalids); 2416 } 2417 2418 list_del(&tmp->list); 2419 list_add(&tmp->list, &removed); 2420 2421 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr); 2422 } 2423 2424 /* And free them up */ 2425 list_for_each_entry_safe(tmp, next, &removed, list) { 2426 amdgpu_vm_it_remove(tmp, &vm->va); 2427 list_del(&tmp->list); 2428 2429 if (tmp->start < saddr) 2430 tmp->start = saddr; 2431 if (tmp->last > eaddr) 2432 tmp->last = eaddr; 2433 2434 tmp->bo_va = NULL; 2435 list_add(&tmp->list, &vm->freed); 2436 trace_amdgpu_vm_bo_unmap(NULL, tmp); 2437 } 2438 2439 /* Insert partial mapping before the range */ 2440 if (!list_empty(&before->list)) { 2441 amdgpu_vm_it_insert(before, &vm->va); 2442 if (before->flags & AMDGPU_PTE_PRT) 2443 amdgpu_vm_prt_get(adev); 2444 } else { 2445 kfree(before); 2446 } 2447 2448 /* Insert partial mapping after the range */ 2449 if (!list_empty(&after->list)) { 2450 amdgpu_vm_it_insert(after, &vm->va); 2451 if (after->flags & AMDGPU_PTE_PRT) 2452 amdgpu_vm_prt_get(adev); 2453 } else { 2454 kfree(after); 2455 } 2456 2457 return 0; 2458 } 2459 2460 /** 2461 * amdgpu_vm_bo_lookup_mapping - find mapping by address 2462 * 2463 * @vm: the requested VM 2464 * @addr: the address 2465 * 2466 * Find a mapping by it's address. 2467 * 2468 * Returns: 2469 * The amdgpu_bo_va_mapping matching for addr or NULL 2470 * 2471 */ 2472 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm, 2473 uint64_t addr) 2474 { 2475 return amdgpu_vm_it_iter_first(&vm->va, addr, addr); 2476 } 2477 2478 /** 2479 * amdgpu_vm_bo_trace_cs - trace all reserved mappings 2480 * 2481 * @vm: the requested vm 2482 * @ticket: CS ticket 2483 * 2484 * Trace all mappings of BOs reserved during a command submission. 2485 */ 2486 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) 2487 { 2488 struct amdgpu_bo_va_mapping *mapping; 2489 2490 if (!trace_amdgpu_vm_bo_cs_enabled()) 2491 return; 2492 2493 for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping; 2494 mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) { 2495 if (mapping->bo_va && mapping->bo_va->base.bo) { 2496 struct amdgpu_bo *bo; 2497 2498 bo = mapping->bo_va->base.bo; 2499 if (dma_resv_locking_ctx(bo->tbo.base.resv) != 2500 ticket) 2501 continue; 2502 } 2503 2504 trace_amdgpu_vm_bo_cs(mapping); 2505 } 2506 } 2507 2508 /** 2509 * amdgpu_vm_bo_rmv - remove a bo to a specific vm 2510 * 2511 * @adev: amdgpu_device pointer 2512 * @bo_va: requested bo_va 2513 * 2514 * Remove @bo_va->bo from the requested vm. 2515 * 2516 * Object have to be reserved! 2517 */ 2518 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev, 2519 struct amdgpu_bo_va *bo_va) 2520 { 2521 struct amdgpu_bo_va_mapping *mapping, *next; 2522 struct amdgpu_bo *bo = bo_va->base.bo; 2523 struct amdgpu_vm *vm = bo_va->base.vm; 2524 struct amdgpu_vm_bo_base **base; 2525 2526 if (bo) { 2527 if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 2528 vm->bulk_moveable = false; 2529 2530 for (base = &bo_va->base.bo->vm_bo; *base; 2531 base = &(*base)->next) { 2532 if (*base != &bo_va->base) 2533 continue; 2534 2535 *base = bo_va->base.next; 2536 break; 2537 } 2538 } 2539 2540 spin_lock(&vm->invalidated_lock); 2541 list_del(&bo_va->base.vm_status); 2542 spin_unlock(&vm->invalidated_lock); 2543 2544 list_for_each_entry_safe(mapping, next, &bo_va->valids, list) { 2545 list_del(&mapping->list); 2546 amdgpu_vm_it_remove(mapping, &vm->va); 2547 mapping->bo_va = NULL; 2548 trace_amdgpu_vm_bo_unmap(bo_va, mapping); 2549 list_add(&mapping->list, &vm->freed); 2550 } 2551 list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) { 2552 list_del(&mapping->list); 2553 amdgpu_vm_it_remove(mapping, &vm->va); 2554 amdgpu_vm_free_mapping(adev, vm, mapping, 2555 bo_va->last_pt_update); 2556 } 2557 2558 dma_fence_put(bo_va->last_pt_update); 2559 2560 if (bo && bo_va->is_xgmi) 2561 amdgpu_xgmi_set_pstate(adev, AMDGPU_XGMI_PSTATE_MIN); 2562 2563 kfree(bo_va); 2564 } 2565 2566 /** 2567 * amdgpu_vm_evictable - check if we can evict a VM 2568 * 2569 * @bo: A page table of the VM. 2570 * 2571 * Check if it is possible to evict a VM. 2572 */ 2573 bool amdgpu_vm_evictable(struct amdgpu_bo *bo) 2574 { 2575 struct amdgpu_vm_bo_base *bo_base = bo->vm_bo; 2576 2577 /* Page tables of a destroyed VM can go away immediately */ 2578 if (!bo_base || !bo_base->vm) 2579 return true; 2580 2581 /* Don't evict VM page tables while they are busy */ 2582 if (!dma_resv_test_signaled_rcu(bo->tbo.base.resv, true)) 2583 return false; 2584 2585 /* Try to block ongoing updates */ 2586 if (!amdgpu_vm_eviction_trylock(bo_base->vm)) 2587 return false; 2588 2589 /* Don't evict VM page tables while they are updated */ 2590 if (!dma_fence_is_signaled(bo_base->vm->last_unlocked)) { 2591 amdgpu_vm_eviction_unlock(bo_base->vm); 2592 return false; 2593 } 2594 2595 bo_base->vm->evicting = true; 2596 amdgpu_vm_eviction_unlock(bo_base->vm); 2597 return true; 2598 } 2599 2600 /** 2601 * amdgpu_vm_bo_invalidate - mark the bo as invalid 2602 * 2603 * @adev: amdgpu_device pointer 2604 * @bo: amdgpu buffer object 2605 * @evicted: is the BO evicted 2606 * 2607 * Mark @bo as invalid. 2608 */ 2609 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev, 2610 struct amdgpu_bo *bo, bool evicted) 2611 { 2612 struct amdgpu_vm_bo_base *bo_base; 2613 2614 /* shadow bo doesn't have bo base, its validation needs its parent */ 2615 if (bo->parent && bo->parent->shadow == bo) 2616 bo = bo->parent; 2617 2618 for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) { 2619 struct amdgpu_vm *vm = bo_base->vm; 2620 2621 if (evicted && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) { 2622 amdgpu_vm_bo_evicted(bo_base); 2623 continue; 2624 } 2625 2626 if (bo_base->moved) 2627 continue; 2628 bo_base->moved = true; 2629 2630 if (bo->tbo.type == ttm_bo_type_kernel) 2631 amdgpu_vm_bo_relocated(bo_base); 2632 else if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) 2633 amdgpu_vm_bo_moved(bo_base); 2634 else 2635 amdgpu_vm_bo_invalidated(bo_base); 2636 } 2637 } 2638 2639 /** 2640 * amdgpu_vm_get_block_size - calculate VM page table size as power of two 2641 * 2642 * @vm_size: VM size 2643 * 2644 * Returns: 2645 * VM page table as power of two 2646 */ 2647 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size) 2648 { 2649 /* Total bits covered by PD + PTs */ 2650 unsigned bits = ilog2(vm_size) + 18; 2651 2652 /* Make sure the PD is 4K in size up to 8GB address space. 2653 Above that split equal between PD and PTs */ 2654 if (vm_size <= 8) 2655 return (bits - 9); 2656 else 2657 return ((bits + 3) / 2); 2658 } 2659 2660 /** 2661 * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size 2662 * 2663 * @adev: amdgpu_device pointer 2664 * @min_vm_size: the minimum vm size in GB if it's set auto 2665 * @fragment_size_default: Default PTE fragment size 2666 * @max_level: max VMPT level 2667 * @max_bits: max address space size in bits 2668 * 2669 */ 2670 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size, 2671 uint32_t fragment_size_default, unsigned max_level, 2672 unsigned max_bits) 2673 { 2674 unsigned int max_size = 1 << (max_bits - 30); 2675 unsigned int vm_size; 2676 uint64_t tmp; 2677 2678 /* adjust vm size first */ 2679 if (amdgpu_vm_size != -1) { 2680 vm_size = amdgpu_vm_size; 2681 if (vm_size > max_size) { 2682 dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n", 2683 amdgpu_vm_size, max_size); 2684 vm_size = max_size; 2685 } 2686 } else { 2687 struct sysinfo si; 2688 unsigned int phys_ram_gb; 2689 2690 /* Optimal VM size depends on the amount of physical 2691 * RAM available. Underlying requirements and 2692 * assumptions: 2693 * 2694 * - Need to map system memory and VRAM from all GPUs 2695 * - VRAM from other GPUs not known here 2696 * - Assume VRAM <= system memory 2697 * - On GFX8 and older, VM space can be segmented for 2698 * different MTYPEs 2699 * - Need to allow room for fragmentation, guard pages etc. 2700 * 2701 * This adds up to a rough guess of system memory x3. 2702 * Round up to power of two to maximize the available 2703 * VM size with the given page table size. 2704 */ 2705 si_meminfo(&si); 2706 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit + 2707 (1 << 30) - 1) >> 30; 2708 vm_size = roundup_pow_of_two( 2709 min(max(phys_ram_gb * 3, min_vm_size), max_size)); 2710 } 2711 2712 adev->vm_manager.max_pfn = (uint64_t)vm_size << 18; 2713 2714 tmp = roundup_pow_of_two(adev->vm_manager.max_pfn); 2715 if (amdgpu_vm_block_size != -1) 2716 tmp >>= amdgpu_vm_block_size - 9; 2717 tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1; 2718 adev->vm_manager.num_level = min(max_level, (unsigned)tmp); 2719 switch (adev->vm_manager.num_level) { 2720 case 3: 2721 adev->vm_manager.root_level = AMDGPU_VM_PDB2; 2722 break; 2723 case 2: 2724 adev->vm_manager.root_level = AMDGPU_VM_PDB1; 2725 break; 2726 case 1: 2727 adev->vm_manager.root_level = AMDGPU_VM_PDB0; 2728 break; 2729 default: 2730 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n"); 2731 } 2732 /* block size depends on vm size and hw setup*/ 2733 if (amdgpu_vm_block_size != -1) 2734 adev->vm_manager.block_size = 2735 min((unsigned)amdgpu_vm_block_size, max_bits 2736 - AMDGPU_GPU_PAGE_SHIFT 2737 - 9 * adev->vm_manager.num_level); 2738 else if (adev->vm_manager.num_level > 1) 2739 adev->vm_manager.block_size = 9; 2740 else 2741 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp); 2742 2743 if (amdgpu_vm_fragment_size == -1) 2744 adev->vm_manager.fragment_size = fragment_size_default; 2745 else 2746 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size; 2747 2748 DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n", 2749 vm_size, adev->vm_manager.num_level + 1, 2750 adev->vm_manager.block_size, 2751 adev->vm_manager.fragment_size); 2752 } 2753 2754 /** 2755 * amdgpu_vm_wait_idle - wait for the VM to become idle 2756 * 2757 * @vm: VM object to wait for 2758 * @timeout: timeout to wait for VM to become idle 2759 */ 2760 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) 2761 { 2762 timeout = dma_resv_wait_timeout_rcu(vm->root.base.bo->tbo.base.resv, 2763 true, true, timeout); 2764 if (timeout <= 0) 2765 return timeout; 2766 2767 return dma_fence_wait_timeout(vm->last_unlocked, true, timeout); 2768 } 2769 2770 /** 2771 * amdgpu_vm_init - initialize a vm instance 2772 * 2773 * @adev: amdgpu_device pointer 2774 * @vm: requested vm 2775 * @vm_context: Indicates if it GFX or Compute context 2776 * @pasid: Process address space identifier 2777 * 2778 * Init @vm fields. 2779 * 2780 * Returns: 2781 * 0 for success, error for failure. 2782 */ 2783 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, 2784 int vm_context, u32 pasid) 2785 { 2786 struct amdgpu_bo_param bp; 2787 struct amdgpu_bo *root; 2788 int r, i; 2789 2790 vm->va = RB_ROOT_CACHED; 2791 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 2792 vm->reserved_vmid[i] = NULL; 2793 INIT_LIST_HEAD(&vm->evicted); 2794 INIT_LIST_HEAD(&vm->relocated); 2795 INIT_LIST_HEAD(&vm->moved); 2796 INIT_LIST_HEAD(&vm->idle); 2797 INIT_LIST_HEAD(&vm->invalidated); 2798 spin_lock_init(&vm->invalidated_lock); 2799 INIT_LIST_HEAD(&vm->freed); 2800 INIT_LIST_HEAD(&vm->done); 2801 2802 /* create scheduler entities for page table updates */ 2803 r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL, 2804 adev->vm_manager.vm_pte_scheds, 2805 adev->vm_manager.vm_pte_num_scheds, NULL); 2806 if (r) 2807 return r; 2808 2809 r = drm_sched_entity_init(&vm->delayed, DRM_SCHED_PRIORITY_NORMAL, 2810 adev->vm_manager.vm_pte_scheds, 2811 adev->vm_manager.vm_pte_num_scheds, NULL); 2812 if (r) 2813 goto error_free_immediate; 2814 2815 vm->pte_support_ats = false; 2816 vm->is_compute_context = false; 2817 2818 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) { 2819 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2820 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2821 2822 if (adev->asic_type == CHIP_RAVEN) 2823 vm->pte_support_ats = true; 2824 } else { 2825 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2826 AMDGPU_VM_USE_CPU_FOR_GFX); 2827 } 2828 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2829 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2830 WARN_ONCE((vm->use_cpu_for_update && 2831 !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2832 "CPU update of VM recommended only for large BAR system\n"); 2833 2834 if (vm->use_cpu_for_update) 2835 vm->update_funcs = &amdgpu_vm_cpu_funcs; 2836 else 2837 vm->update_funcs = &amdgpu_vm_sdma_funcs; 2838 vm->last_update = NULL; 2839 vm->last_unlocked = dma_fence_get_stub(); 2840 2841 mutex_init(&vm->eviction_lock); 2842 vm->evicting = false; 2843 2844 amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, false, &bp); 2845 if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) 2846 bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW; 2847 r = amdgpu_bo_create(adev, &bp, &root); 2848 if (r) 2849 goto error_free_delayed; 2850 2851 r = amdgpu_bo_reserve(root, true); 2852 if (r) 2853 goto error_free_root; 2854 2855 r = dma_resv_reserve_shared(root->tbo.base.resv, 1); 2856 if (r) 2857 goto error_unreserve; 2858 2859 amdgpu_vm_bo_base_init(&vm->root.base, vm, root); 2860 2861 r = amdgpu_vm_clear_bo(adev, vm, root, false); 2862 if (r) 2863 goto error_unreserve; 2864 2865 amdgpu_bo_unreserve(vm->root.base.bo); 2866 2867 if (pasid) { 2868 unsigned long flags; 2869 2870 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2871 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, 2872 GFP_ATOMIC); 2873 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2874 if (r < 0) 2875 goto error_free_root; 2876 2877 vm->pasid = pasid; 2878 } 2879 2880 INIT_KFIFO(vm->faults); 2881 2882 return 0; 2883 2884 error_unreserve: 2885 amdgpu_bo_unreserve(vm->root.base.bo); 2886 2887 error_free_root: 2888 amdgpu_bo_unref(&vm->root.base.bo->shadow); 2889 amdgpu_bo_unref(&vm->root.base.bo); 2890 vm->root.base.bo = NULL; 2891 2892 error_free_delayed: 2893 dma_fence_put(vm->last_unlocked); 2894 drm_sched_entity_destroy(&vm->delayed); 2895 2896 error_free_immediate: 2897 drm_sched_entity_destroy(&vm->immediate); 2898 2899 return r; 2900 } 2901 2902 /** 2903 * amdgpu_vm_check_clean_reserved - check if a VM is clean 2904 * 2905 * @adev: amdgpu_device pointer 2906 * @vm: the VM to check 2907 * 2908 * check all entries of the root PD, if any subsequent PDs are allocated, 2909 * it means there are page table creating and filling, and is no a clean 2910 * VM 2911 * 2912 * Returns: 2913 * 0 if this VM is clean 2914 */ 2915 static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev, 2916 struct amdgpu_vm *vm) 2917 { 2918 enum amdgpu_vm_level root = adev->vm_manager.root_level; 2919 unsigned int entries = amdgpu_vm_num_entries(adev, root); 2920 unsigned int i = 0; 2921 2922 if (!(vm->root.entries)) 2923 return 0; 2924 2925 for (i = 0; i < entries; i++) { 2926 if (vm->root.entries[i].base.bo) 2927 return -EINVAL; 2928 } 2929 2930 return 0; 2931 } 2932 2933 /** 2934 * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM 2935 * 2936 * @adev: amdgpu_device pointer 2937 * @vm: requested vm 2938 * @pasid: pasid to use 2939 * 2940 * This only works on GFX VMs that don't have any BOs added and no 2941 * page tables allocated yet. 2942 * 2943 * Changes the following VM parameters: 2944 * - use_cpu_for_update 2945 * - pte_supports_ats 2946 * - pasid (old PASID is released, because compute manages its own PASIDs) 2947 * 2948 * Reinitializes the page directory to reflect the changed ATS 2949 * setting. 2950 * 2951 * Returns: 2952 * 0 for success, -errno for errors. 2953 */ 2954 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, 2955 u32 pasid) 2956 { 2957 bool pte_support_ats = (adev->asic_type == CHIP_RAVEN); 2958 int r; 2959 2960 r = amdgpu_bo_reserve(vm->root.base.bo, true); 2961 if (r) 2962 return r; 2963 2964 /* Sanity checks */ 2965 r = amdgpu_vm_check_clean_reserved(adev, vm); 2966 if (r) 2967 goto unreserve_bo; 2968 2969 if (pasid) { 2970 unsigned long flags; 2971 2972 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 2973 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1, 2974 GFP_ATOMIC); 2975 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 2976 2977 if (r == -ENOSPC) 2978 goto unreserve_bo; 2979 r = 0; 2980 } 2981 2982 /* Check if PD needs to be reinitialized and do it before 2983 * changing any other state, in case it fails. 2984 */ 2985 if (pte_support_ats != vm->pte_support_ats) { 2986 vm->pte_support_ats = pte_support_ats; 2987 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo, false); 2988 if (r) 2989 goto free_idr; 2990 } 2991 2992 /* Update VM state */ 2993 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode & 2994 AMDGPU_VM_USE_CPU_FOR_COMPUTE); 2995 DRM_DEBUG_DRIVER("VM update mode is %s\n", 2996 vm->use_cpu_for_update ? "CPU" : "SDMA"); 2997 WARN_ONCE((vm->use_cpu_for_update && 2998 !amdgpu_gmc_vram_full_visible(&adev->gmc)), 2999 "CPU update of VM recommended only for large BAR system\n"); 3000 3001 if (vm->use_cpu_for_update) { 3002 /* Sync with last SDMA update/clear before switching to CPU */ 3003 r = amdgpu_bo_sync_wait(vm->root.base.bo, 3004 AMDGPU_FENCE_OWNER_UNDEFINED, true); 3005 if (r) 3006 goto free_idr; 3007 3008 vm->update_funcs = &amdgpu_vm_cpu_funcs; 3009 } else { 3010 vm->update_funcs = &amdgpu_vm_sdma_funcs; 3011 } 3012 dma_fence_put(vm->last_update); 3013 vm->last_update = NULL; 3014 vm->is_compute_context = true; 3015 3016 if (vm->pasid) { 3017 unsigned long flags; 3018 3019 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 3020 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 3021 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 3022 3023 /* Free the original amdgpu allocated pasid 3024 * Will be replaced with kfd allocated pasid 3025 */ 3026 amdgpu_pasid_free(vm->pasid); 3027 vm->pasid = 0; 3028 } 3029 3030 /* Free the shadow bo for compute VM */ 3031 amdgpu_bo_unref(&vm->root.base.bo->shadow); 3032 3033 if (pasid) 3034 vm->pasid = pasid; 3035 3036 goto unreserve_bo; 3037 3038 free_idr: 3039 if (pasid) { 3040 unsigned long flags; 3041 3042 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 3043 idr_remove(&adev->vm_manager.pasid_idr, pasid); 3044 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 3045 } 3046 unreserve_bo: 3047 amdgpu_bo_unreserve(vm->root.base.bo); 3048 return r; 3049 } 3050 3051 /** 3052 * amdgpu_vm_release_compute - release a compute vm 3053 * @adev: amdgpu_device pointer 3054 * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute 3055 * 3056 * This is a correspondant of amdgpu_vm_make_compute. It decouples compute 3057 * pasid from vm. Compute should stop use of vm after this call. 3058 */ 3059 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) 3060 { 3061 if (vm->pasid) { 3062 unsigned long flags; 3063 3064 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 3065 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 3066 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 3067 } 3068 vm->pasid = 0; 3069 vm->is_compute_context = false; 3070 } 3071 3072 /** 3073 * amdgpu_vm_fini - tear down a vm instance 3074 * 3075 * @adev: amdgpu_device pointer 3076 * @vm: requested vm 3077 * 3078 * Tear down @vm. 3079 * Unbind the VM and remove all bos from the vm bo list 3080 */ 3081 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) 3082 { 3083 struct amdgpu_bo_va_mapping *mapping, *tmp; 3084 bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt; 3085 struct amdgpu_bo *root; 3086 int i; 3087 3088 amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm); 3089 3090 root = amdgpu_bo_ref(vm->root.base.bo); 3091 amdgpu_bo_reserve(root, true); 3092 if (vm->pasid) { 3093 unsigned long flags; 3094 3095 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 3096 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid); 3097 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 3098 vm->pasid = 0; 3099 } 3100 3101 dma_fence_wait(vm->last_unlocked, false); 3102 dma_fence_put(vm->last_unlocked); 3103 3104 list_for_each_entry_safe(mapping, tmp, &vm->freed, list) { 3105 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) { 3106 amdgpu_vm_prt_fini(adev, vm); 3107 prt_fini_needed = false; 3108 } 3109 3110 list_del(&mapping->list); 3111 amdgpu_vm_free_mapping(adev, vm, mapping, NULL); 3112 } 3113 3114 amdgpu_vm_free_pts(adev, vm, NULL); 3115 amdgpu_bo_unreserve(root); 3116 amdgpu_bo_unref(&root); 3117 WARN_ON(vm->root.base.bo); 3118 3119 drm_sched_entity_destroy(&vm->immediate); 3120 drm_sched_entity_destroy(&vm->delayed); 3121 3122 if (!RB_EMPTY_ROOT(&vm->va.rb_root)) { 3123 dev_err(adev->dev, "still active bo inside vm\n"); 3124 } 3125 rbtree_postorder_for_each_entry_safe(mapping, tmp, 3126 &vm->va.rb_root, rb) { 3127 /* Don't remove the mapping here, we don't want to trigger a 3128 * rebalance and the tree is about to be destroyed anyway. 3129 */ 3130 list_del(&mapping->list); 3131 kfree(mapping); 3132 } 3133 3134 dma_fence_put(vm->last_update); 3135 for (i = 0; i < AMDGPU_MAX_VMHUBS; i++) 3136 amdgpu_vmid_free_reserved(adev, vm, i); 3137 } 3138 3139 /** 3140 * amdgpu_vm_manager_init - init the VM manager 3141 * 3142 * @adev: amdgpu_device pointer 3143 * 3144 * Initialize the VM manager structures 3145 */ 3146 void amdgpu_vm_manager_init(struct amdgpu_device *adev) 3147 { 3148 unsigned i; 3149 3150 amdgpu_vmid_mgr_init(adev); 3151 3152 adev->vm_manager.fence_context = 3153 dma_fence_context_alloc(AMDGPU_MAX_RINGS); 3154 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) 3155 adev->vm_manager.seqno[i] = 0; 3156 3157 spin_lock_init(&adev->vm_manager.prt_lock); 3158 atomic_set(&adev->vm_manager.num_prt_users, 0); 3159 3160 /* If not overridden by the user, by default, only in large BAR systems 3161 * Compute VM tables will be updated by CPU 3162 */ 3163 #ifdef CONFIG_X86_64 3164 if (amdgpu_vm_update_mode == -1) { 3165 if (amdgpu_gmc_vram_full_visible(&adev->gmc)) 3166 adev->vm_manager.vm_update_mode = 3167 AMDGPU_VM_USE_CPU_FOR_COMPUTE; 3168 else 3169 adev->vm_manager.vm_update_mode = 0; 3170 } else 3171 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode; 3172 #else 3173 adev->vm_manager.vm_update_mode = 0; 3174 #endif 3175 3176 idr_init(&adev->vm_manager.pasid_idr); 3177 spin_lock_init(&adev->vm_manager.pasid_lock); 3178 } 3179 3180 /** 3181 * amdgpu_vm_manager_fini - cleanup VM manager 3182 * 3183 * @adev: amdgpu_device pointer 3184 * 3185 * Cleanup the VM manager and free resources. 3186 */ 3187 void amdgpu_vm_manager_fini(struct amdgpu_device *adev) 3188 { 3189 WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr)); 3190 idr_destroy(&adev->vm_manager.pasid_idr); 3191 3192 amdgpu_vmid_mgr_fini(adev); 3193 } 3194 3195 /** 3196 * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs. 3197 * 3198 * @dev: drm device pointer 3199 * @data: drm_amdgpu_vm 3200 * @filp: drm file pointer 3201 * 3202 * Returns: 3203 * 0 for success, -errno for errors. 3204 */ 3205 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 3206 { 3207 union drm_amdgpu_vm *args = data; 3208 struct amdgpu_device *adev = drm_to_adev(dev); 3209 struct amdgpu_fpriv *fpriv = filp->driver_priv; 3210 long timeout = msecs_to_jiffies(2000); 3211 int r; 3212 3213 switch (args->in.op) { 3214 case AMDGPU_VM_OP_RESERVE_VMID: 3215 /* We only have requirement to reserve vmid from gfxhub */ 3216 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, 3217 AMDGPU_GFXHUB_0); 3218 if (r) 3219 return r; 3220 break; 3221 case AMDGPU_VM_OP_UNRESERVE_VMID: 3222 if (amdgpu_sriov_runtime(adev)) 3223 timeout = 8 * timeout; 3224 3225 /* Wait vm idle to make sure the vmid set in SPM_VMID is 3226 * not referenced anymore. 3227 */ 3228 r = amdgpu_bo_reserve(fpriv->vm.root.base.bo, true); 3229 if (r) 3230 return r; 3231 3232 r = amdgpu_vm_wait_idle(&fpriv->vm, timeout); 3233 if (r < 0) 3234 return r; 3235 3236 amdgpu_bo_unreserve(fpriv->vm.root.base.bo); 3237 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0); 3238 break; 3239 default: 3240 return -EINVAL; 3241 } 3242 3243 return 0; 3244 } 3245 3246 /** 3247 * amdgpu_vm_get_task_info - Extracts task info for a PASID. 3248 * 3249 * @adev: drm device pointer 3250 * @pasid: PASID identifier for VM 3251 * @task_info: task_info to fill. 3252 */ 3253 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, u32 pasid, 3254 struct amdgpu_task_info *task_info) 3255 { 3256 struct amdgpu_vm *vm; 3257 unsigned long flags; 3258 3259 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags); 3260 3261 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); 3262 if (vm) 3263 *task_info = vm->task_info; 3264 3265 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags); 3266 } 3267 3268 /** 3269 * amdgpu_vm_set_task_info - Sets VMs task info. 3270 * 3271 * @vm: vm for which to set the info 3272 */ 3273 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) 3274 { 3275 if (vm->task_info.pid) 3276 return; 3277 3278 vm->task_info.pid = current->pid; 3279 get_task_comm(vm->task_info.task_name, current); 3280 3281 if (current->group_leader->mm != current->mm) 3282 return; 3283 3284 vm->task_info.tgid = current->group_leader->pid; 3285 get_task_comm(vm->task_info.process_name, current->group_leader); 3286 } 3287 3288 /** 3289 * amdgpu_vm_handle_fault - graceful handling of VM faults. 3290 * @adev: amdgpu device pointer 3291 * @pasid: PASID of the VM 3292 * @addr: Address of the fault 3293 * 3294 * Try to gracefully handle a VM fault. Return true if the fault was handled and 3295 * shouldn't be reported any more. 3296 */ 3297 bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, 3298 uint64_t addr) 3299 { 3300 struct amdgpu_bo *root; 3301 uint64_t value, flags; 3302 struct amdgpu_vm *vm; 3303 long r; 3304 3305 spin_lock(&adev->vm_manager.pasid_lock); 3306 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); 3307 if (vm) 3308 root = amdgpu_bo_ref(vm->root.base.bo); 3309 else 3310 root = NULL; 3311 spin_unlock(&adev->vm_manager.pasid_lock); 3312 3313 if (!root) 3314 return false; 3315 3316 r = amdgpu_bo_reserve(root, true); 3317 if (r) 3318 goto error_unref; 3319 3320 /* Double check that the VM still exists */ 3321 spin_lock(&adev->vm_manager.pasid_lock); 3322 vm = idr_find(&adev->vm_manager.pasid_idr, pasid); 3323 if (vm && vm->root.base.bo != root) 3324 vm = NULL; 3325 spin_unlock(&adev->vm_manager.pasid_lock); 3326 if (!vm) 3327 goto error_unlock; 3328 3329 addr /= AMDGPU_GPU_PAGE_SIZE; 3330 flags = AMDGPU_PTE_VALID | AMDGPU_PTE_SNOOPED | 3331 AMDGPU_PTE_SYSTEM; 3332 3333 if (vm->is_compute_context) { 3334 /* Intentionally setting invalid PTE flag 3335 * combination to force a no-retry-fault 3336 */ 3337 flags = AMDGPU_PTE_EXECUTABLE | AMDGPU_PDE_PTE | 3338 AMDGPU_PTE_TF; 3339 value = 0; 3340 3341 } else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) { 3342 /* Redirect the access to the dummy page */ 3343 value = adev->dummy_page_addr; 3344 flags |= AMDGPU_PTE_EXECUTABLE | AMDGPU_PTE_READABLE | 3345 AMDGPU_PTE_WRITEABLE; 3346 3347 } else { 3348 /* Let the hw retry silently on the PTE */ 3349 value = 0; 3350 } 3351 3352 r = amdgpu_vm_bo_update_mapping(adev, adev, vm, true, false, NULL, addr, 3353 addr, flags, value, NULL, NULL, 3354 NULL); 3355 if (r) 3356 goto error_unlock; 3357 3358 r = amdgpu_vm_update_pdes(adev, vm, true); 3359 3360 error_unlock: 3361 amdgpu_bo_unreserve(root); 3362 if (r < 0) 3363 DRM_ERROR("Can't handle page fault (%ld)\n", r); 3364 3365 error_unref: 3366 amdgpu_bo_unref(&root); 3367 3368 return false; 3369 } 3370 3371 #if defined(CONFIG_DEBUG_FS) 3372 /** 3373 * amdgpu_debugfs_vm_bo_info - print BO info for the VM 3374 * 3375 * @vm: Requested VM for printing BO info 3376 * @m: debugfs file 3377 * 3378 * Print BO information in debugfs file for the VM 3379 */ 3380 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m) 3381 { 3382 struct amdgpu_bo_va *bo_va, *tmp; 3383 u64 total_idle = 0; 3384 u64 total_evicted = 0; 3385 u64 total_relocated = 0; 3386 u64 total_moved = 0; 3387 u64 total_invalidated = 0; 3388 u64 total_done = 0; 3389 unsigned int total_idle_objs = 0; 3390 unsigned int total_evicted_objs = 0; 3391 unsigned int total_relocated_objs = 0; 3392 unsigned int total_moved_objs = 0; 3393 unsigned int total_invalidated_objs = 0; 3394 unsigned int total_done_objs = 0; 3395 unsigned int id = 0; 3396 3397 seq_puts(m, "\tIdle BOs:\n"); 3398 list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) { 3399 if (!bo_va->base.bo) 3400 continue; 3401 total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3402 } 3403 total_idle_objs = id; 3404 id = 0; 3405 3406 seq_puts(m, "\tEvicted BOs:\n"); 3407 list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) { 3408 if (!bo_va->base.bo) 3409 continue; 3410 total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3411 } 3412 total_evicted_objs = id; 3413 id = 0; 3414 3415 seq_puts(m, "\tRelocated BOs:\n"); 3416 list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) { 3417 if (!bo_va->base.bo) 3418 continue; 3419 total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3420 } 3421 total_relocated_objs = id; 3422 id = 0; 3423 3424 seq_puts(m, "\tMoved BOs:\n"); 3425 list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) { 3426 if (!bo_va->base.bo) 3427 continue; 3428 total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3429 } 3430 total_moved_objs = id; 3431 id = 0; 3432 3433 seq_puts(m, "\tInvalidated BOs:\n"); 3434 spin_lock(&vm->invalidated_lock); 3435 list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) { 3436 if (!bo_va->base.bo) 3437 continue; 3438 total_invalidated += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3439 } 3440 total_invalidated_objs = id; 3441 id = 0; 3442 3443 seq_puts(m, "\tDone BOs:\n"); 3444 list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) { 3445 if (!bo_va->base.bo) 3446 continue; 3447 total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m); 3448 } 3449 spin_unlock(&vm->invalidated_lock); 3450 total_done_objs = id; 3451 3452 seq_printf(m, "\tTotal idle size: %12lld\tobjs:\t%d\n", total_idle, 3453 total_idle_objs); 3454 seq_printf(m, "\tTotal evicted size: %12lld\tobjs:\t%d\n", total_evicted, 3455 total_evicted_objs); 3456 seq_printf(m, "\tTotal relocated size: %12lld\tobjs:\t%d\n", total_relocated, 3457 total_relocated_objs); 3458 seq_printf(m, "\tTotal moved size: %12lld\tobjs:\t%d\n", total_moved, 3459 total_moved_objs); 3460 seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated, 3461 total_invalidated_objs); 3462 seq_printf(m, "\tTotal done size: %12lld\tobjs:\t%d\n", total_done, 3463 total_done_objs); 3464 } 3465 #endif 3466