1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <drm/drm_drv.h> 25 26 #include "amdgpu.h" 27 #include "amdgpu_trace.h" 28 #include "amdgpu_vm.h" 29 #include "amdgpu_vm_internal.h" 30 #include "amdgpu_job.h" 31 32 /* 33 * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt 34 */ 35 struct amdgpu_vm_pt_cursor { 36 uint64_t pfn; 37 struct amdgpu_vm_bo_base *parent; 38 struct amdgpu_vm_bo_base *entry; 39 unsigned int level; 40 }; 41 42 /** 43 * amdgpu_vm_pt_level_shift - return the addr shift for each level 44 * 45 * @adev: amdgpu_device pointer 46 * @level: VMPT level 47 * 48 * Returns: 49 * The number of bits the pfn needs to be right shifted for a level. 50 */ 51 static unsigned int amdgpu_vm_pt_level_shift(struct amdgpu_device *adev, 52 unsigned int level) 53 { 54 switch (level) { 55 case AMDGPU_VM_PDB3: 56 case AMDGPU_VM_PDB2: 57 case AMDGPU_VM_PDB1: 58 case AMDGPU_VM_PDB0: 59 return 9 * (AMDGPU_VM_PDB0 - level) + 60 adev->vm_manager.block_size; 61 case AMDGPU_VM_PTB: 62 return 0; 63 default: 64 return ~0; 65 } 66 } 67 68 /** 69 * amdgpu_vm_pt_num_entries - return the number of entries in a PD/PT 70 * 71 * @adev: amdgpu_device pointer 72 * @level: VMPT level 73 * 74 * Returns: 75 * The number of entries in a page directory or page table. 76 */ 77 static unsigned int amdgpu_vm_pt_num_entries(struct amdgpu_device *adev, 78 unsigned int level) 79 { 80 unsigned int shift; 81 82 shift = amdgpu_vm_pt_level_shift(adev, adev->vm_manager.root_level); 83 if (level == adev->vm_manager.root_level) 84 /* For the root directory */ 85 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) 86 >> shift; 87 else if (level != AMDGPU_VM_PTB) 88 /* Everything in between */ 89 return 512; 90 91 /* For the page tables on the leaves */ 92 return AMDGPU_VM_PTE_COUNT(adev); 93 } 94 95 /** 96 * amdgpu_vm_pt_entries_mask - the mask to get the entry number of a PD/PT 97 * 98 * @adev: amdgpu_device pointer 99 * @level: VMPT level 100 * 101 * Returns: 102 * The mask to extract the entry number of a PD/PT from an address. 103 */ 104 static uint32_t amdgpu_vm_pt_entries_mask(struct amdgpu_device *adev, 105 unsigned int level) 106 { 107 if (level <= adev->vm_manager.root_level) 108 return 0xffffffff; 109 else if (level != AMDGPU_VM_PTB) 110 return 0x1ff; 111 else 112 return AMDGPU_VM_PTE_COUNT(adev) - 1; 113 } 114 115 /** 116 * amdgpu_vm_pt_size - returns the size of the page table in bytes 117 * 118 * @adev: amdgpu_device pointer 119 * @level: VMPT level 120 * 121 * Returns: 122 * The size of the BO for a page directory or page table in bytes. 123 */ 124 static unsigned int amdgpu_vm_pt_size(struct amdgpu_device *adev, 125 unsigned int level) 126 { 127 return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_pt_num_entries(adev, level) * 8); 128 } 129 130 /** 131 * amdgpu_vm_pt_parent - get the parent page directory 132 * 133 * @pt: child page table 134 * 135 * Helper to get the parent entry for the child page table. NULL if we are at 136 * the root page directory. 137 */ 138 static struct amdgpu_vm_bo_base * 139 amdgpu_vm_pt_parent(struct amdgpu_vm_bo_base *pt) 140 { 141 struct amdgpu_bo *parent = pt->bo->parent; 142 143 if (!parent) 144 return NULL; 145 146 return parent->vm_bo; 147 } 148 149 /** 150 * amdgpu_vm_pt_start - start PD/PT walk 151 * 152 * @adev: amdgpu_device pointer 153 * @vm: amdgpu_vm structure 154 * @start: start address of the walk 155 * @cursor: state to initialize 156 * 157 * Initialize a amdgpu_vm_pt_cursor to start a walk. 158 */ 159 static void amdgpu_vm_pt_start(struct amdgpu_device *adev, 160 struct amdgpu_vm *vm, uint64_t start, 161 struct amdgpu_vm_pt_cursor *cursor) 162 { 163 cursor->pfn = start; 164 cursor->parent = NULL; 165 cursor->entry = &vm->root; 166 cursor->level = adev->vm_manager.root_level; 167 } 168 169 /** 170 * amdgpu_vm_pt_descendant - go to child node 171 * 172 * @adev: amdgpu_device pointer 173 * @cursor: current state 174 * 175 * Walk to the child node of the current node. 176 * Returns: 177 * True if the walk was possible, false otherwise. 178 */ 179 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev, 180 struct amdgpu_vm_pt_cursor *cursor) 181 { 182 unsigned int mask, shift, idx; 183 184 if ((cursor->level == AMDGPU_VM_PTB) || !cursor->entry || 185 !cursor->entry->bo) 186 return false; 187 188 mask = amdgpu_vm_pt_entries_mask(adev, cursor->level); 189 shift = amdgpu_vm_pt_level_shift(adev, cursor->level); 190 191 ++cursor->level; 192 idx = (cursor->pfn >> shift) & mask; 193 cursor->parent = cursor->entry; 194 cursor->entry = &to_amdgpu_bo_vm(cursor->entry->bo)->entries[idx]; 195 return true; 196 } 197 198 /** 199 * amdgpu_vm_pt_sibling - go to sibling node 200 * 201 * @adev: amdgpu_device pointer 202 * @cursor: current state 203 * 204 * Walk to the sibling node of the current node. 205 * Returns: 206 * True if the walk was possible, false otherwise. 207 */ 208 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev, 209 struct amdgpu_vm_pt_cursor *cursor) 210 { 211 212 unsigned int shift, num_entries; 213 struct amdgpu_bo_vm *parent; 214 215 /* Root doesn't have a sibling */ 216 if (!cursor->parent) 217 return false; 218 219 /* Go to our parents and see if we got a sibling */ 220 shift = amdgpu_vm_pt_level_shift(adev, cursor->level - 1); 221 num_entries = amdgpu_vm_pt_num_entries(adev, cursor->level - 1); 222 parent = to_amdgpu_bo_vm(cursor->parent->bo); 223 224 if (cursor->entry == &parent->entries[num_entries - 1]) 225 return false; 226 227 cursor->pfn += 1ULL << shift; 228 cursor->pfn &= ~((1ULL << shift) - 1); 229 ++cursor->entry; 230 return true; 231 } 232 233 /** 234 * amdgpu_vm_pt_ancestor - go to parent node 235 * 236 * @cursor: current state 237 * 238 * Walk to the parent node of the current node. 239 * Returns: 240 * True if the walk was possible, false otherwise. 241 */ 242 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor) 243 { 244 if (!cursor->parent) 245 return false; 246 247 --cursor->level; 248 cursor->entry = cursor->parent; 249 cursor->parent = amdgpu_vm_pt_parent(cursor->parent); 250 return true; 251 } 252 253 /** 254 * amdgpu_vm_pt_next - get next PD/PT in hieratchy 255 * 256 * @adev: amdgpu_device pointer 257 * @cursor: current state 258 * 259 * Walk the PD/PT tree to the next node. 260 */ 261 static void amdgpu_vm_pt_next(struct amdgpu_device *adev, 262 struct amdgpu_vm_pt_cursor *cursor) 263 { 264 /* First try a newborn child */ 265 if (amdgpu_vm_pt_descendant(adev, cursor)) 266 return; 267 268 /* If that didn't worked try to find a sibling */ 269 while (!amdgpu_vm_pt_sibling(adev, cursor)) { 270 /* No sibling, go to our parents and grandparents */ 271 if (!amdgpu_vm_pt_ancestor(cursor)) { 272 cursor->pfn = ~0ll; 273 return; 274 } 275 } 276 } 277 278 /** 279 * amdgpu_vm_pt_first_dfs - start a deep first search 280 * 281 * @adev: amdgpu_device structure 282 * @vm: amdgpu_vm structure 283 * @start: optional cursor to start with 284 * @cursor: state to initialize 285 * 286 * Starts a deep first traversal of the PD/PT tree. 287 */ 288 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev, 289 struct amdgpu_vm *vm, 290 struct amdgpu_vm_pt_cursor *start, 291 struct amdgpu_vm_pt_cursor *cursor) 292 { 293 if (start) 294 *cursor = *start; 295 else 296 amdgpu_vm_pt_start(adev, vm, 0, cursor); 297 298 while (amdgpu_vm_pt_descendant(adev, cursor)) 299 ; 300 } 301 302 /** 303 * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue 304 * 305 * @start: starting point for the search 306 * @entry: current entry 307 * 308 * Returns: 309 * True when the search should continue, false otherwise. 310 */ 311 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start, 312 struct amdgpu_vm_bo_base *entry) 313 { 314 return entry && (!start || entry != start->entry); 315 } 316 317 /** 318 * amdgpu_vm_pt_next_dfs - get the next node for a deep first search 319 * 320 * @adev: amdgpu_device structure 321 * @cursor: current state 322 * 323 * Move the cursor to the next node in a deep first search. 324 */ 325 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev, 326 struct amdgpu_vm_pt_cursor *cursor) 327 { 328 if (!cursor->entry) 329 return; 330 331 if (!cursor->parent) 332 cursor->entry = NULL; 333 else if (amdgpu_vm_pt_sibling(adev, cursor)) 334 while (amdgpu_vm_pt_descendant(adev, cursor)) 335 ; 336 else 337 amdgpu_vm_pt_ancestor(cursor); 338 } 339 340 /* 341 * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs 342 */ 343 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry) \ 344 for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)), \ 345 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\ 346 amdgpu_vm_pt_continue_dfs((start), (entry)); \ 347 (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor))) 348 349 /** 350 * amdgpu_vm_pt_clear - initially clear the PDs/PTs 351 * 352 * @adev: amdgpu_device pointer 353 * @vm: VM to clear BO from 354 * @vmbo: BO to clear 355 * @immediate: use an immediate update 356 * 357 * Root PD needs to be reserved when calling this. 358 * 359 * Returns: 360 * 0 on success, errno otherwise. 361 */ 362 int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm, 363 struct amdgpu_bo_vm *vmbo, bool immediate) 364 { 365 unsigned int level = adev->vm_manager.root_level; 366 struct ttm_operation_ctx ctx = { true, false }; 367 struct amdgpu_vm_update_params params; 368 struct amdgpu_bo *ancestor = &vmbo->bo; 369 unsigned int entries; 370 struct amdgpu_bo *bo = &vmbo->bo; 371 uint64_t value = 0, flags = 0; 372 uint64_t addr; 373 int r, idx; 374 375 /* Figure out our place in the hierarchy */ 376 if (ancestor->parent) { 377 ++level; 378 while (ancestor->parent->parent) { 379 ++level; 380 ancestor = ancestor->parent; 381 } 382 } 383 384 entries = amdgpu_bo_size(bo) / 8; 385 386 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 387 if (r) 388 return r; 389 390 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 391 return -ENODEV; 392 393 r = vm->update_funcs->map_table(vmbo); 394 if (r) 395 goto exit; 396 397 memset(¶ms, 0, sizeof(params)); 398 params.adev = adev; 399 params.vm = vm; 400 params.immediate = immediate; 401 402 r = vm->update_funcs->prepare(¶ms, NULL, 403 AMDGPU_KERNEL_JOB_ID_VM_PT_CLEAR); 404 if (r) 405 goto exit; 406 407 addr = 0; 408 409 if (adev->asic_type >= CHIP_VEGA10) { 410 if (level != AMDGPU_VM_PTB) { 411 if (vm->is_npa) 412 flags = adev->gmc.noretry_flags; 413 /* Handle leaf PDEs as PTEs */ 414 flags |= AMDGPU_PDE_PTE_FLAG(adev); 415 amdgpu_gmc_get_vm_pde(adev, level, 416 &value, &flags); 417 } else if (vm->is_npa) { 418 flags = adev->gmc.noretry_flags; 419 } else { 420 /* Workaround for fault priority problem on GMC9 */ 421 flags = AMDGPU_PTE_EXECUTABLE | adev->gmc.init_pte_flags; 422 } 423 } 424 425 r = vm->update_funcs->update(¶ms, vmbo, addr, 0, entries, 426 value, flags); 427 if (r) 428 goto exit; 429 430 r = vm->update_funcs->commit(¶ms, &vm->last_update); 431 exit: 432 drm_dev_exit(idx); 433 return r; 434 } 435 436 /** 437 * amdgpu_vm_pt_create - create bo for PD/PT 438 * 439 * @adev: amdgpu_device pointer 440 * @vm: requesting vm 441 * @level: the page table level 442 * @immediate: use a immediate update 443 * @vmbo: pointer to the buffer object pointer 444 * @xcp_id: GPU partition id 445 */ 446 int amdgpu_vm_pt_create(struct amdgpu_device *adev, struct amdgpu_vm *vm, 447 int level, bool immediate, struct amdgpu_bo_vm **vmbo, 448 int32_t xcp_id) 449 { 450 struct amdgpu_bo_param bp; 451 unsigned int num_entries; 452 int r; 453 454 memset(&bp, 0, sizeof(bp)); 455 456 bp.size = amdgpu_vm_pt_size(adev, level); 457 bp.byte_align = AMDGPU_GPU_PAGE_SIZE; 458 459 if (!adev->gmc.is_app_apu) 460 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 461 else 462 bp.domain = AMDGPU_GEM_DOMAIN_GTT; 463 464 bp.domain = amdgpu_bo_get_preferred_domain(adev, bp.domain); 465 bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | 466 AMDGPU_GEM_CREATE_CPU_GTT_USWC; 467 468 if (level < AMDGPU_VM_PTB) 469 num_entries = amdgpu_vm_pt_num_entries(adev, level); 470 else 471 num_entries = 0; 472 473 bp.bo_ptr_size = struct_size((*vmbo), entries, num_entries); 474 475 if (vm->use_cpu_for_update) 476 bp.flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 477 478 bp.type = ttm_bo_type_kernel; 479 bp.no_wait_gpu = immediate; 480 bp.xcp_id_plus1 = xcp_id + 1; 481 482 if (vm->root.bo) 483 bp.resv = vm->root.bo->tbo.base.resv; 484 485 r = amdgpu_bo_create_vm(adev, &bp, vmbo); 486 if (r) 487 return r; 488 489 /* Assumes that reservation is shared with the VM root and that the 490 * reservation is locked 491 */ 492 if (vm->root.bo && vm->is_npa) { 493 struct amdgpu_bo *pt_bo = &(*vmbo)->bo; 494 495 r = amdgpu_bo_pin(pt_bo, AMDGPU_GEM_DOMAIN_VRAM); 496 if (r) { 497 amdgpu_bo_unref(&pt_bo); 498 return r; 499 } 500 } 501 502 return 0; 503 } 504 505 /** 506 * amdgpu_vm_pt_alloc - Allocate a specific page table 507 * 508 * @p: see amdgpu_vm_update_params definition 509 * @cursor: Which page table to allocate 510 * 511 * Make sure a specific page table or directory is allocated. 512 * 513 * Returns: 514 * 515 * 0 on success or a negative error code on failure. 516 */ 517 static int amdgpu_vm_pt_alloc(struct amdgpu_vm_update_params *p, 518 struct amdgpu_vm_pt_cursor *cursor) 519 { 520 struct amdgpu_vm_bo_base *entry = cursor->entry; 521 struct amdgpu_bo *pt_bo; 522 struct amdgpu_bo_vm *pt; 523 int r, r2; 524 525 if (entry->bo) 526 return 0; 527 528 amdgpu_vm_end_critical(p); 529 r = amdgpu_vm_pt_create(p->adev, p->vm, cursor->level, p->immediate, 530 &pt, p->vm->root.bo->xcp_id); 531 r2 = amdgpu_vm_begin_critical(p); 532 if (r) 533 return r; 534 535 pt_bo = &pt->bo; 536 if (r2) { 537 r = r2; 538 goto error_unpin; 539 } 540 541 /* Keep a reference to the root directory to avoid 542 * freeing them up in the wrong order. 543 */ 544 pt_bo->parent = amdgpu_bo_ref(cursor->parent->bo); 545 amdgpu_vm_bo_base_init(entry, p->vm, pt_bo); 546 r = amdgpu_vm_pt_clear(p->adev, p->vm, pt, p->immediate); 547 if (r) 548 goto error_unpin; 549 550 return 0; 551 552 error_unpin: 553 if (p->vm->is_npa) 554 amdgpu_bo_unpin(pt_bo); 555 amdgpu_bo_unref(&pt_bo); 556 return r; 557 } 558 559 /** 560 * amdgpu_vm_pt_free - free one PD/PT 561 * 562 * @entry: PDE to free 563 */ 564 static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry) 565 { 566 if (!entry->bo) 567 return; 568 569 if (entry->vm->is_npa) 570 amdgpu_bo_unpin(entry->bo); 571 572 amdgpu_vm_update_stats(entry, entry->bo->tbo.resource, -1); 573 entry->bo->vm_bo = NULL; 574 ttm_bo_set_bulk_move(&entry->bo->tbo, NULL); 575 576 list_del(&entry->vm_status); 577 amdgpu_bo_unref(&entry->bo); 578 } 579 580 /** 581 * amdgpu_vm_pt_free_list - free PD/PT levels 582 * 583 * @adev: amdgpu device structure 584 * @params: see amdgpu_vm_update_params definition 585 * 586 * Free the page directory objects saved in the flush list 587 */ 588 void amdgpu_vm_pt_free_list(struct amdgpu_device *adev, 589 struct amdgpu_vm_update_params *params) 590 { 591 struct amdgpu_vm_bo_base *entry, *next; 592 bool unlocked = params->unlocked; 593 594 if (list_empty(¶ms->tlb_flush_waitlist)) 595 return; 596 597 /* 598 * unlocked unmap clear page table leaves, warning to free the page entry. 599 */ 600 WARN_ON(unlocked); 601 602 list_for_each_entry_safe(entry, next, ¶ms->tlb_flush_waitlist, vm_status) 603 amdgpu_vm_pt_free(entry); 604 } 605 606 /** 607 * amdgpu_vm_pt_add_list - add PD/PT level to the flush list 608 * 609 * @params: parameters for the update 610 * @cursor: first PT entry to start DF search from, non NULL 611 * 612 * This list will be freed after TLB flush. 613 */ 614 static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params, 615 struct amdgpu_vm_pt_cursor *cursor) 616 { 617 struct amdgpu_vm_pt_cursor seek; 618 struct amdgpu_vm_bo_base *entry; 619 620 for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) { 621 if (entry && entry->bo) 622 list_move(&entry->vm_status, ¶ms->tlb_flush_waitlist); 623 } 624 625 /* enter start node now */ 626 list_move(&cursor->entry->vm_status, ¶ms->tlb_flush_waitlist); 627 } 628 629 /** 630 * amdgpu_vm_pt_free_root - free root PD 631 * @adev: amdgpu device structure 632 * @vm: amdgpu vm structure 633 * 634 * Free the root page directory and everything below it. 635 */ 636 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm) 637 { 638 struct amdgpu_vm_pt_cursor cursor; 639 struct amdgpu_vm_bo_base *entry; 640 641 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) { 642 if (entry) 643 amdgpu_vm_pt_free(entry); 644 } 645 } 646 647 /** 648 * amdgpu_vm_pde_update - update a single level in the hierarchy 649 * 650 * @params: parameters for the update 651 * @entry: entry to update 652 * 653 * Makes sure the requested entry in parent is up to date. 654 */ 655 int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params, 656 struct amdgpu_vm_bo_base *entry) 657 { 658 struct amdgpu_vm_bo_base *parent = amdgpu_vm_pt_parent(entry); 659 struct amdgpu_bo *bo, *pbo; 660 struct amdgpu_vm *vm = params->vm; 661 uint64_t pde, pt, flags; 662 unsigned int level; 663 664 if (WARN_ON(!parent)) 665 return -EINVAL; 666 667 bo = parent->bo; 668 for (level = 0, pbo = bo->parent; pbo; ++level) 669 pbo = pbo->parent; 670 671 level += params->adev->vm_manager.root_level; 672 amdgpu_gmc_get_pde_for_bo(entry->bo, level, &pt, &flags); 673 pde = (entry - to_amdgpu_bo_vm(parent->bo)->entries) * 8; 674 return vm->update_funcs->update(params, to_amdgpu_bo_vm(bo), pde, pt, 675 1, 0, flags); 676 } 677 678 /** 679 * amdgpu_vm_pte_update_noretry_flags - Update PTE no-retry flags 680 * 681 * @adev: amdgpu_device pointer 682 * @flags: pointer to PTE flags 683 * 684 * Update PTE no-retry flags when TF is enabled. 685 */ 686 static void amdgpu_vm_pte_update_noretry_flags(struct amdgpu_device *adev, 687 uint64_t *flags) 688 { 689 /* 690 * Update no-retry flags with the corresponding TF 691 * no-retry combination. 692 */ 693 if ((*flags & AMDGPU_VM_NORETRY_FLAGS) == AMDGPU_VM_NORETRY_FLAGS) { 694 *flags &= ~AMDGPU_VM_NORETRY_FLAGS; 695 *flags |= adev->gmc.noretry_flags; 696 } 697 } 698 699 /* 700 * amdgpu_vm_pte_update_flags - figure out flags for PTE updates 701 * 702 * Make sure to set the right flags for the PTEs at the desired level. 703 */ 704 static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params, 705 struct amdgpu_bo_vm *pt, 706 unsigned int level, 707 uint64_t pe, uint64_t addr, 708 unsigned int count, uint32_t incr, 709 uint64_t flags) 710 { 711 struct amdgpu_device *adev = params->adev; 712 713 if (level != AMDGPU_VM_PTB) { 714 flags |= AMDGPU_PDE_PTE_FLAG(params->adev); 715 amdgpu_gmc_get_vm_pde(adev, level, &addr, &flags); 716 717 } else if (adev->asic_type >= CHIP_VEGA10 && 718 !(flags & AMDGPU_PTE_VALID) && 719 !(flags & AMDGPU_PTE_PRT_FLAG(params->adev))) { 720 721 /* Workaround for fault priority problem on GMC9 and GFX12, 722 * EXECUTABLE for GMC9 fault priority and init_pte_flags 723 * (e.g. AMDGPU_PTE_IS_PTE on GFX12) 724 */ 725 flags |= AMDGPU_PTE_EXECUTABLE | adev->gmc.init_pte_flags; 726 } 727 728 /* 729 * Update no-retry flags to use the no-retry flag combination 730 * with TF enabled. The AMDGPU_VM_NORETRY_FLAGS flag combination 731 * does not work when TF is enabled. So, replace them with 732 * AMDGPU_VM_NORETRY_FLAGS_TF flag combination which works for 733 * all cases. 734 */ 735 if (level == AMDGPU_VM_PTB) 736 amdgpu_vm_pte_update_noretry_flags(adev, &flags); 737 738 params->vm->update_funcs->update(params, pt, pe, addr, count, incr, 739 flags); 740 } 741 742 /** 743 * amdgpu_vm_pte_fragment - get fragment for PTEs 744 * 745 * @params: see amdgpu_vm_update_params definition 746 * @start: first PTE to handle 747 * @end: last PTE to handle 748 * @flags: hw mapping flags 749 * @frag: resulting fragment size 750 * @frag_end: end of this fragment 751 * 752 * Returns the first possible fragment for the start and end address. 753 */ 754 static void amdgpu_vm_pte_fragment(struct amdgpu_vm_update_params *params, 755 uint64_t start, uint64_t end, uint64_t flags, 756 unsigned int *frag, uint64_t *frag_end) 757 { 758 /** 759 * The MC L1 TLB supports variable sized pages, based on a fragment 760 * field in the PTE. When this field is set to a non-zero value, page 761 * granularity is increased from 4KB to (1 << (12 + frag)). The PTE 762 * flags are considered valid for all PTEs within the fragment range 763 * and corresponding mappings are assumed to be physically contiguous. 764 * 765 * The L1 TLB can store a single PTE for the whole fragment, 766 * significantly increasing the space available for translation 767 * caching. This leads to large improvements in throughput when the 768 * TLB is under pressure. 769 * 770 * The L2 TLB distributes small and large fragments into two 771 * asymmetric partitions. The large fragment cache is significantly 772 * larger. Thus, we try to use large fragments wherever possible. 773 * Userspace can support this by aligning virtual base address and 774 * allocation size to the fragment size. 775 * 776 * Starting with Vega10 the fragment size only controls the L1. The L2 777 * is now directly feed with small/huge/giant pages from the walker. 778 */ 779 unsigned int max_frag; 780 781 if (params->adev->asic_type < CHIP_VEGA10) 782 max_frag = params->adev->vm_manager.fragment_size; 783 else 784 max_frag = 31; 785 786 /* system pages are non continuously */ 787 if (params->pages_addr) { 788 *frag = 0; 789 *frag_end = end; 790 return; 791 } 792 793 /* This intentionally wraps around if no bit is set */ 794 *frag = min_t(unsigned int, ffs(start) - 1, fls64(end - start) - 1); 795 if (*frag >= max_frag) { 796 *frag = max_frag; 797 *frag_end = end & ~((1ULL << max_frag) - 1); 798 } else { 799 *frag_end = start + (1 << *frag); 800 } 801 } 802 803 /** 804 * amdgpu_vm_ptes_update - make sure that page tables are valid 805 * 806 * @params: see amdgpu_vm_update_params definition 807 * @start: start of GPU address range 808 * @end: end of GPU address range 809 * @dst: destination address to map to, the next dst inside the function 810 * @flags: mapping flags 811 * 812 * Update the page tables in the range @start - @end. 813 * 814 * Returns: 815 * 0 for success, -EINVAL for failure. 816 */ 817 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params, 818 uint64_t start, uint64_t end, 819 uint64_t dst, uint64_t flags) 820 { 821 struct amdgpu_device *adev = params->adev; 822 struct amdgpu_vm_pt_cursor cursor; 823 uint64_t frag_start = start, frag_end; 824 unsigned int frag; 825 int r; 826 827 /* figure out the initial fragment */ 828 amdgpu_vm_pte_fragment(params, frag_start, end, flags, &frag, 829 &frag_end); 830 831 /* walk over the address space and update the PTs */ 832 amdgpu_vm_pt_start(adev, params->vm, start, &cursor); 833 while (cursor.pfn < end) { 834 unsigned int shift, parent_shift, mask; 835 uint64_t incr, entry_end, pe_start; 836 struct amdgpu_bo *pt; 837 838 if (!params->unlocked) { 839 /* make sure that the page tables covering the 840 * address range are actually allocated 841 */ 842 r = amdgpu_vm_pt_alloc(params, &cursor); 843 if (r) 844 return r; 845 } 846 847 shift = amdgpu_vm_pt_level_shift(adev, cursor.level); 848 parent_shift = amdgpu_vm_pt_level_shift(adev, cursor.level - 1); 849 if (params->unlocked) { 850 /* Unlocked updates are only allowed on the leaves */ 851 if (amdgpu_vm_pt_descendant(adev, &cursor)) 852 continue; 853 } else if (adev->asic_type < CHIP_VEGA10 && 854 (flags & AMDGPU_PTE_VALID)) { 855 /* No huge page support before GMC v9 */ 856 if (cursor.level != AMDGPU_VM_PTB) { 857 if (!amdgpu_vm_pt_descendant(adev, &cursor)) 858 return -ENOENT; 859 continue; 860 } 861 } else if (frag < shift) { 862 /* We can't use this level when the fragment size is 863 * smaller than the address shift. Go to the next 864 * child entry and try again. 865 */ 866 if (amdgpu_vm_pt_descendant(adev, &cursor)) 867 continue; 868 } else if (frag >= parent_shift) { 869 /* If the fragment size is even larger than the parent 870 * shift we should go up one level and check it again. 871 */ 872 if (!amdgpu_vm_pt_ancestor(&cursor)) 873 return -EINVAL; 874 continue; 875 } 876 877 pt = cursor.entry->bo; 878 if (!pt) { 879 /* We need all PDs and PTs for mapping something, */ 880 if (flags & AMDGPU_PTE_VALID) 881 return -ENOENT; 882 883 /* but unmapping something can happen at a higher 884 * level. 885 */ 886 if (!amdgpu_vm_pt_ancestor(&cursor)) 887 return -EINVAL; 888 889 pt = cursor.entry->bo; 890 shift = parent_shift; 891 frag_end = max(frag_end, ALIGN(frag_start + 1, 892 1ULL << shift)); 893 } 894 895 /* Looks good so far, calculate parameters for the update */ 896 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift; 897 mask = amdgpu_vm_pt_entries_mask(adev, cursor.level); 898 pe_start = ((cursor.pfn >> shift) & mask) * 8; 899 900 if (cursor.level < AMDGPU_VM_PTB && params->unlocked) 901 /* 902 * MMU notifier callback unlocked unmap huge page, leave is PDE entry, 903 * only clear one entry. Next entry search again for PDE or PTE leave. 904 */ 905 entry_end = 1ULL << shift; 906 else 907 entry_end = ((uint64_t)mask + 1) << shift; 908 entry_end += cursor.pfn & ~(entry_end - 1); 909 entry_end = min(entry_end, end); 910 911 do { 912 uint64_t upd_end = min(entry_end, frag_end); 913 unsigned int nptes = (upd_end - frag_start) >> shift; 914 uint64_t upd_flags = flags | AMDGPU_PTE_FRAG(frag); 915 916 /* This can happen when we set higher level PDs to 917 * silent to stop fault floods. 918 */ 919 nptes = max(nptes, 1u); 920 921 trace_amdgpu_vm_update_ptes(params, frag_start, upd_end, 922 min(nptes, 32u), dst, incr, 923 upd_flags); 924 amdgpu_vm_pte_update_flags(params, to_amdgpu_bo_vm(pt), 925 cursor.level, pe_start, dst, 926 nptes, incr, upd_flags); 927 928 pe_start += nptes * 8; 929 dst += nptes * incr; 930 931 frag_start = upd_end; 932 if (frag_start >= frag_end) { 933 /* figure out the next fragment */ 934 amdgpu_vm_pte_fragment(params, frag_start, end, 935 flags, &frag, &frag_end); 936 if (frag < shift) 937 break; 938 } 939 } while (frag_start < entry_end); 940 941 if (amdgpu_vm_pt_descendant(adev, &cursor)) { 942 /* Free all child entries. 943 * Update the tables with the flags and addresses and free up subsequent 944 * tables in the case of huge pages or freed up areas. 945 * This is the maximum you can free, because all other page tables are not 946 * completely covered by the range and so potentially still in use. 947 */ 948 while (cursor.pfn < frag_start) { 949 /* Make sure previous mapping is freed */ 950 if (cursor.entry->bo) { 951 params->needs_flush = true; 952 amdgpu_vm_pt_add_list(params, &cursor); 953 } 954 amdgpu_vm_pt_next(adev, &cursor); 955 } 956 957 } else if (frag >= shift) { 958 /* or just move on to the next on the same level. */ 959 amdgpu_vm_pt_next(adev, &cursor); 960 } 961 } 962 963 return 0; 964 } 965 966 /** 967 * amdgpu_vm_pt_map_tables - have bo of root PD cpu accessible 968 * @adev: amdgpu device structure 969 * @vm: amdgpu vm structure 970 * 971 * make root page directory and everything below it cpu accessible. 972 */ 973 int amdgpu_vm_pt_map_tables(struct amdgpu_device *adev, struct amdgpu_vm *vm) 974 { 975 struct amdgpu_vm_pt_cursor cursor; 976 struct amdgpu_vm_bo_base *entry; 977 978 for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry) { 979 980 struct amdgpu_bo_vm *bo; 981 int r; 982 983 if (entry->bo) { 984 bo = to_amdgpu_bo_vm(entry->bo); 985 r = vm->update_funcs->map_table(bo); 986 if (r) 987 return r; 988 } 989 } 990 991 return 0; 992 } 993