1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 27 #include <linux/io-64-nonatomic-lo-hi.h> 28 #ifdef CONFIG_X86 29 #include <asm/hypervisor.h> 30 #endif 31 32 #include "amdgpu.h" 33 #include "amdgpu_gmc.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_reset.h" 36 #include "amdgpu_xgmi.h" 37 #include "amdgpu_atomfirmware.h" 38 39 #include <drm/drm_drv.h> 40 #include <drm/ttm/ttm_tt.h> 41 42 static const u64 four_gb = 0x100000000ULL; 43 44 bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev) 45 { 46 return adev->gmc.xgmi.connected_to_cpu || amdgpu_virt_xgmi_migrate_enabled(adev); 47 } 48 49 /** 50 * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0 51 * 52 * @adev: amdgpu_device pointer 53 * 54 * Allocate video memory for pdb0 and map it for CPU access 55 * Returns 0 for success, error for failure. 56 */ 57 int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev) 58 { 59 int r; 60 struct amdgpu_bo_param bp; 61 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 62 uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21; 63 uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >> pde0_page_shift; 64 65 memset(&bp, 0, sizeof(bp)); 66 bp.size = PAGE_ALIGN((npdes + 1) * 8); 67 bp.byte_align = PAGE_SIZE; 68 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 69 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 70 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 71 bp.type = ttm_bo_type_kernel; 72 bp.resv = NULL; 73 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 74 75 r = amdgpu_bo_create(adev, &bp, &adev->gmc.pdb0_bo); 76 if (r) 77 return r; 78 79 r = amdgpu_bo_reserve(adev->gmc.pdb0_bo, false); 80 if (unlikely(r != 0)) 81 goto bo_reserve_failure; 82 83 r = amdgpu_bo_pin(adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM); 84 if (r) 85 goto bo_pin_failure; 86 r = amdgpu_bo_kmap(adev->gmc.pdb0_bo, &adev->gmc.ptr_pdb0); 87 if (r) 88 goto bo_kmap_failure; 89 90 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 91 return 0; 92 93 bo_kmap_failure: 94 amdgpu_bo_unpin(adev->gmc.pdb0_bo); 95 bo_pin_failure: 96 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 97 bo_reserve_failure: 98 amdgpu_bo_unref(&adev->gmc.pdb0_bo); 99 return r; 100 } 101 102 /** 103 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO 104 * 105 * @bo: the BO to get the PDE for 106 * @level: the level in the PD hirarchy 107 * @addr: resulting addr 108 * @flags: resulting flags 109 * 110 * Get the address and flags to be used for a PDE (Page Directory Entry). 111 */ 112 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, 113 uint64_t *addr, uint64_t *flags) 114 { 115 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 116 117 switch (bo->tbo.resource->mem_type) { 118 case TTM_PL_TT: 119 *addr = bo->tbo.ttm->dma_address[0]; 120 break; 121 case TTM_PL_VRAM: 122 *addr = amdgpu_bo_gpu_offset(bo); 123 break; 124 default: 125 *addr = 0; 126 break; 127 } 128 *flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, bo->tbo.resource); 129 amdgpu_gmc_get_vm_pde(adev, level, addr, flags); 130 } 131 132 /* 133 * amdgpu_gmc_pd_addr - return the address of the root directory 134 */ 135 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo) 136 { 137 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 138 uint64_t pd_addr; 139 140 /* TODO: move that into ASIC specific code */ 141 if (adev->asic_type >= CHIP_VEGA10) { 142 uint64_t flags = AMDGPU_PTE_VALID; 143 144 amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags); 145 pd_addr |= flags; 146 } else { 147 pd_addr = amdgpu_bo_gpu_offset(bo); 148 } 149 return pd_addr; 150 } 151 152 /** 153 * amdgpu_gmc_set_pte_pde - update the page tables using CPU 154 * 155 * @adev: amdgpu_device pointer 156 * @cpu_pt_addr: cpu address of the page table 157 * @gpu_page_idx: entry in the page table to update 158 * @addr: dst addr to write into pte/pde 159 * @flags: access flags 160 * 161 * Update the page tables using CPU. 162 */ 163 int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 164 uint32_t gpu_page_idx, uint64_t addr, 165 uint64_t flags) 166 { 167 void __iomem *ptr = (void *)cpu_pt_addr; 168 uint64_t value; 169 170 /* 171 * The following is for PTE only. GART does not have PDEs. 172 */ 173 value = addr & 0x0000FFFFFFFFF000ULL; 174 value |= flags; 175 writeq(value, ptr + (gpu_page_idx * 8)); 176 177 return 0; 178 } 179 180 /** 181 * amdgpu_gmc_agp_addr - return the address in the AGP address space 182 * 183 * @bo: TTM BO which needs the address, must be in GTT domain 184 * 185 * Tries to figure out how to access the BO through the AGP aperture. Returns 186 * AMDGPU_BO_INVALID_OFFSET if that is not possible. 187 */ 188 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) 189 { 190 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 191 192 if (!bo->ttm) 193 return AMDGPU_BO_INVALID_OFFSET; 194 195 if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached) 196 return AMDGPU_BO_INVALID_OFFSET; 197 198 if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) 199 return AMDGPU_BO_INVALID_OFFSET; 200 201 return adev->gmc.agp_start + bo->ttm->dma_address[0]; 202 } 203 204 /** 205 * amdgpu_gmc_vram_location - try to find VRAM location 206 * 207 * @adev: amdgpu device structure holding all necessary information 208 * @mc: memory controller structure holding memory information 209 * @base: base address at which to put VRAM 210 * 211 * Function will try to place VRAM at base address provided 212 * as parameter. 213 */ 214 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 215 u64 base) 216 { 217 uint64_t vis_limit = (uint64_t)amdgpu_vis_vram_limit << 20; 218 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; 219 220 mc->vram_start = base; 221 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 222 if (limit < mc->real_vram_size) 223 mc->real_vram_size = limit; 224 225 if (vis_limit && vis_limit < mc->visible_vram_size) 226 mc->visible_vram_size = vis_limit; 227 228 if (mc->real_vram_size < mc->visible_vram_size) 229 mc->visible_vram_size = mc->real_vram_size; 230 231 if (mc->xgmi.num_physical_nodes == 0) { 232 mc->fb_start = mc->vram_start; 233 mc->fb_end = mc->vram_end; 234 } 235 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 236 mc->mc_vram_size >> 20, mc->vram_start, 237 mc->vram_end, mc->real_vram_size >> 20); 238 } 239 240 /** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture 241 * 242 * @adev: amdgpu device structure holding all necessary information 243 * @mc: memory controller structure holding memory information 244 * 245 * This function is only used if use GART for FB translation. In such 246 * case, we use sysvm aperture (vmid0 page tables) for both vram 247 * and gart (aka system memory) access. 248 * 249 * GPUVM (and our organization of vmid0 page tables) require sysvm 250 * aperture to be placed at a location aligned with 8 times of native 251 * page size. For example, if vm_context0_cntl.page_table_block_size 252 * is 12, then native page size is 8G (2M*2^12), sysvm should start 253 * with a 64G aligned address. For simplicity, we just put sysvm at 254 * address 0. So vram start at address 0 and gart is right after vram. 255 */ 256 void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 257 { 258 u64 hive_vram_start = 0; 259 u64 hive_vram_end = mc->xgmi.node_segment_size * mc->xgmi.num_physical_nodes - 1; 260 mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id; 261 mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1; 262 /* node_segment_size may not 4GB aligned on SRIOV, align up is needed. */ 263 mc->gart_start = ALIGN(hive_vram_end + 1, four_gb); 264 mc->gart_end = mc->gart_start + mc->gart_size - 1; 265 if (amdgpu_virt_xgmi_migrate_enabled(adev)) { 266 /* set mc->vram_start to 0 to switch the returned GPU address of 267 * amdgpu_bo_create_reserved() from FB aperture to GART aperture. 268 */ 269 mc->vram_start = 0; 270 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 271 mc->visible_vram_size = min(mc->visible_vram_size, mc->real_vram_size); 272 } else { 273 mc->fb_start = hive_vram_start; 274 mc->fb_end = hive_vram_end; 275 } 276 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 277 mc->mc_vram_size >> 20, mc->vram_start, 278 mc->vram_end, mc->real_vram_size >> 20); 279 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 280 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 281 } 282 283 /** 284 * amdgpu_gmc_gart_location - try to find GART location 285 * 286 * @adev: amdgpu device structure holding all necessary information 287 * @mc: memory controller structure holding memory information 288 * @gart_placement: GART placement policy with respect to VRAM 289 * 290 * Function will try to place GART before or after VRAM. 291 * If GART size is bigger than space left then we ajust GART size. 292 * Thus function will never fails. 293 */ 294 void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 295 enum amdgpu_gart_placement gart_placement) 296 { 297 u64 size_af, size_bf; 298 /*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/ 299 u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1); 300 301 /* VCE doesn't like it when BOs cross a 4GB segment, so align 302 * the GART base on a 4GB boundary as well. 303 */ 304 size_bf = mc->fb_start; 305 size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb); 306 307 if (mc->gart_size > max(size_bf, size_af)) { 308 dev_warn(adev->dev, "limiting GART\n"); 309 mc->gart_size = max(size_bf, size_af); 310 } 311 312 switch (gart_placement) { 313 case AMDGPU_GART_PLACEMENT_HIGH: 314 mc->gart_start = max_mc_address - mc->gart_size + 1; 315 break; 316 case AMDGPU_GART_PLACEMENT_LOW: 317 if (size_bf >= mc->gart_size) 318 mc->gart_start = 0; 319 else 320 mc->gart_start = ALIGN(mc->fb_end, four_gb); 321 break; 322 case AMDGPU_GART_PLACEMENT_BEST_FIT: 323 default: 324 if ((size_bf >= mc->gart_size && size_bf < size_af) || 325 (size_af < mc->gart_size)) 326 mc->gart_start = 0; 327 else 328 mc->gart_start = max_mc_address - mc->gart_size + 1; 329 break; 330 } 331 332 mc->gart_start &= ~(four_gb - 1); 333 mc->gart_end = mc->gart_start + mc->gart_size - 1; 334 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 335 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 336 } 337 338 /** 339 * amdgpu_gmc_agp_location - try to find AGP location 340 * @adev: amdgpu device structure holding all necessary information 341 * @mc: memory controller structure holding memory information 342 * 343 * Function will place try to find a place for the AGP BAR in the MC address 344 * space. 345 * 346 * AGP BAR will be assigned the largest available hole in the address space. 347 * Should be called after VRAM and GART locations are setup. 348 */ 349 void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 350 { 351 const uint64_t sixteen_gb = 1ULL << 34; 352 const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1); 353 u64 size_af, size_bf; 354 355 if (mc->fb_start > mc->gart_start) { 356 size_bf = (mc->fb_start & sixteen_gb_mask) - 357 ALIGN(mc->gart_end + 1, sixteen_gb); 358 size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb); 359 } else { 360 size_bf = mc->fb_start & sixteen_gb_mask; 361 size_af = (mc->gart_start & sixteen_gb_mask) - 362 ALIGN(mc->fb_end + 1, sixteen_gb); 363 } 364 365 if (size_bf > size_af) { 366 mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask; 367 mc->agp_size = size_bf; 368 } else { 369 mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb); 370 mc->agp_size = size_af; 371 } 372 373 mc->agp_end = mc->agp_start + mc->agp_size - 1; 374 dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n", 375 mc->agp_size >> 20, mc->agp_start, mc->agp_end); 376 } 377 378 /** 379 * amdgpu_gmc_set_agp_default - Set the default AGP aperture value. 380 * @adev: amdgpu device structure holding all necessary information 381 * @mc: memory controller structure holding memory information 382 * 383 * To disable the AGP aperture, you need to set the start to a larger 384 * value than the end. This function sets the default value which 385 * can then be overridden using amdgpu_gmc_agp_location() if you want 386 * to enable the AGP aperture on a specific chip. 387 * 388 */ 389 void amdgpu_gmc_set_agp_default(struct amdgpu_device *adev, 390 struct amdgpu_gmc *mc) 391 { 392 mc->agp_start = 0xffffffffffff; 393 mc->agp_end = 0; 394 mc->agp_size = 0; 395 } 396 397 /** 398 * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid 399 * 400 * @addr: 48 bit physical address, page aligned (36 significant bits) 401 * @pasid: 16 bit process address space identifier 402 */ 403 static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid) 404 { 405 return addr << 4 | pasid; 406 } 407 408 /** 409 * amdgpu_gmc_filter_faults - filter VM faults 410 * 411 * @adev: amdgpu device structure 412 * @ih: interrupt ring that the fault received from 413 * @addr: address of the VM fault 414 * @pasid: PASID of the process causing the fault 415 * @timestamp: timestamp of the fault 416 * 417 * Returns: 418 * True if the fault was filtered and should not be processed further. 419 * False if the fault is a new one and needs to be handled. 420 */ 421 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, 422 struct amdgpu_ih_ring *ih, uint64_t addr, 423 uint16_t pasid, uint64_t timestamp) 424 { 425 struct amdgpu_gmc *gmc = &adev->gmc; 426 uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid); 427 struct amdgpu_gmc_fault *fault; 428 uint32_t hash; 429 430 /* Stale retry fault if timestamp goes backward */ 431 if (amdgpu_ih_ts_after(timestamp, ih->processed_timestamp)) 432 return true; 433 434 /* If we don't have space left in the ring buffer return immediately */ 435 stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) - 436 AMDGPU_GMC_FAULT_TIMEOUT; 437 if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp) 438 return true; 439 440 /* Try to find the fault in the hash */ 441 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 442 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 443 while (fault->timestamp >= stamp) { 444 uint64_t tmp; 445 446 if (atomic64_read(&fault->key) == key) { 447 /* 448 * if we get a fault which is already present in 449 * the fault_ring and the timestamp of 450 * the fault is after the expired timestamp, 451 * then this is a new fault that needs to be added 452 * into the fault ring. 453 */ 454 if (fault->timestamp_expiry != 0 && 455 amdgpu_ih_ts_after(fault->timestamp_expiry, 456 timestamp)) 457 break; 458 else 459 return true; 460 } 461 462 tmp = fault->timestamp; 463 fault = &gmc->fault_ring[fault->next]; 464 465 /* Check if the entry was reused */ 466 if (fault->timestamp >= tmp) 467 break; 468 } 469 470 /* Add the fault to the ring */ 471 fault = &gmc->fault_ring[gmc->last_fault]; 472 atomic64_set(&fault->key, key); 473 fault->timestamp = timestamp; 474 475 /* And update the hash */ 476 fault->next = gmc->fault_hash[hash].idx; 477 gmc->fault_hash[hash].idx = gmc->last_fault++; 478 return false; 479 } 480 481 /** 482 * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter 483 * 484 * @adev: amdgpu device structure 485 * @addr: address of the VM fault 486 * @pasid: PASID of the process causing the fault 487 * 488 * Remove the address from fault filter, then future vm fault on this address 489 * will pass to retry fault handler to recover. 490 */ 491 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr, 492 uint16_t pasid) 493 { 494 struct amdgpu_gmc *gmc = &adev->gmc; 495 uint64_t key = amdgpu_gmc_fault_key(addr, pasid); 496 struct amdgpu_ih_ring *ih; 497 struct amdgpu_gmc_fault *fault; 498 uint32_t last_wptr; 499 uint64_t last_ts; 500 uint32_t hash; 501 uint64_t tmp; 502 503 if (adev->irq.retry_cam_enabled) 504 return; 505 else if (adev->irq.ih1.ring_size) 506 ih = &adev->irq.ih1; 507 else if (adev->irq.ih_soft.enabled) 508 ih = &adev->irq.ih_soft; 509 else 510 return; 511 512 /* Get the WPTR of the last entry in IH ring */ 513 last_wptr = amdgpu_ih_get_wptr(adev, ih); 514 /* Order wptr with ring data. */ 515 rmb(); 516 /* Get the timetamp of the last entry in IH ring */ 517 last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1); 518 519 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 520 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 521 do { 522 if (atomic64_read(&fault->key) == key) { 523 /* 524 * Update the timestamp when this fault 525 * expired. 526 */ 527 fault->timestamp_expiry = last_ts; 528 break; 529 } 530 531 tmp = fault->timestamp; 532 fault = &gmc->fault_ring[fault->next]; 533 } while (fault->timestamp < tmp); 534 } 535 536 int amdgpu_gmc_handle_retry_fault(struct amdgpu_device *adev, 537 struct amdgpu_iv_entry *entry, 538 u64 addr, 539 u32 cam_index, 540 u32 node_id, 541 bool write_fault) 542 { 543 int ret; 544 545 if (adev->irq.retry_cam_enabled) { 546 /* Delegate it to a different ring if the hardware hasn't 547 * already done it. 548 */ 549 if (entry->ih == &adev->irq.ih) { 550 amdgpu_irq_delegate(adev, entry, 8); 551 return 1; 552 } 553 554 ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 555 addr, entry->timestamp, write_fault); 556 WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); 557 if (ret) 558 return 1; 559 } else { 560 /* Process it only if it's the first fault for this address */ 561 if (entry->ih != &adev->irq.ih_soft && 562 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 563 entry->timestamp)) 564 return 1; 565 566 /* Delegate it to a different ring if the hardware hasn't 567 * already done it. 568 */ 569 if (entry->ih == &adev->irq.ih) { 570 amdgpu_irq_delegate(adev, entry, 8); 571 return 1; 572 } 573 574 /* Try to handle the recoverable page faults by filling page 575 * tables 576 */ 577 if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 578 addr, entry->timestamp, write_fault)) 579 return 1; 580 } 581 return 0; 582 } 583 584 int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev) 585 { 586 int r; 587 588 /* umc ras block */ 589 r = amdgpu_umc_ras_sw_init(adev); 590 if (r) 591 return r; 592 593 /* mmhub ras block */ 594 r = amdgpu_mmhub_ras_sw_init(adev); 595 if (r) 596 return r; 597 598 /* hdp ras block */ 599 r = amdgpu_hdp_ras_sw_init(adev); 600 if (r) 601 return r; 602 603 /* mca.x ras block */ 604 r = amdgpu_mca_mp0_ras_sw_init(adev); 605 if (r) 606 return r; 607 608 r = amdgpu_mca_mp1_ras_sw_init(adev); 609 if (r) 610 return r; 611 612 r = amdgpu_mca_mpio_ras_sw_init(adev); 613 if (r) 614 return r; 615 616 /* xgmi ras block */ 617 r = amdgpu_xgmi_ras_sw_init(adev); 618 if (r) 619 return r; 620 621 return 0; 622 } 623 624 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev) 625 { 626 return 0; 627 } 628 629 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev) 630 { 631 632 } 633 634 /* 635 * The latest engine allocation on gfx9/10 is: 636 * Engine 2, 3: firmware 637 * Engine 0, 1, 4~16: amdgpu ring, 638 * subject to change when ring number changes 639 * Engine 17: Gart flushes 640 */ 641 #define AMDGPU_VMHUB_INV_ENG_BITMAP 0x1FFF3 642 643 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) 644 { 645 struct amdgpu_ring *ring; 646 unsigned vm_inv_engs[AMDGPU_MAX_VMHUBS] = {0}; 647 unsigned i; 648 unsigned vmhub, inv_eng; 649 struct amdgpu_ring *shared_ring; 650 651 /* init the vm inv eng for all vmhubs */ 652 for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 653 vm_inv_engs[i] = AMDGPU_VMHUB_INV_ENG_BITMAP; 654 /* reserve engine 5 for firmware */ 655 if (adev->enable_mes) 656 vm_inv_engs[i] &= ~(1 << 5); 657 /* reserve engine 6 for uni mes */ 658 if (adev->enable_uni_mes) 659 vm_inv_engs[i] &= ~(1 << 6); 660 /* reserve mmhub engine 3 for firmware */ 661 if (adev->enable_umsch_mm) 662 vm_inv_engs[i] &= ~(1 << 3); 663 } 664 665 for (i = 0; i < adev->num_rings; ++i) { 666 ring = adev->rings[i]; 667 vmhub = ring->vm_hub; 668 669 if (ring == &adev->mes.ring[0] || 670 ring == &adev->mes.ring[1] || 671 ring == &adev->umsch_mm.ring || 672 ring == &adev->cper.ring_buf) 673 continue; 674 675 /* Skip if the ring is a shared ring */ 676 if (amdgpu_sdma_is_shared_inv_eng(adev, ring)) 677 continue; 678 679 inv_eng = ffs(vm_inv_engs[vmhub]); 680 if (!inv_eng) { 681 dev_err(adev->dev, "no VM inv eng for ring %s\n", 682 ring->name); 683 return -EINVAL; 684 } 685 686 ring->vm_inv_eng = inv_eng - 1; 687 vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng); 688 689 dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n", 690 ring->name, ring->vm_inv_eng, ring->vm_hub); 691 /* SDMA has a special packet which allows it to use the same 692 * invalidation engine for all the rings in one instance. 693 * Therefore, we do not allocate a separate VM invalidation engine 694 * for SDMA page rings. Instead, they share the VM invalidation 695 * engine with the SDMA gfx ring. This change ensures efficient 696 * resource management and avoids the issue of insufficient VM 697 * invalidation engines. 698 */ 699 shared_ring = amdgpu_sdma_get_shared_ring(adev, ring); 700 if (shared_ring) { 701 shared_ring->vm_inv_eng = ring->vm_inv_eng; 702 dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n", 703 ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub); 704 continue; 705 } 706 } 707 708 return 0; 709 } 710 711 void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 712 uint32_t vmhub, uint32_t flush_type) 713 { 714 struct amdgpu_ring *ring; 715 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 716 struct dma_fence *fence; 717 struct amdgpu_job *job; 718 int r; 719 720 ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); 721 722 if (!hub->sdma_invalidation_workaround || vmid || 723 !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || 724 !ring->sched.ready) { 725 /* 726 * A GPU reset should flush all TLBs anyway, so no need to do 727 * this while one is ongoing. 728 */ 729 if (!down_read_trylock(&adev->reset_domain->sem)) 730 return; 731 732 if (adev->gmc.flush_tlb_needs_extra_type_2) 733 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 734 vmhub, 2); 735 736 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 737 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 738 vmhub, 0); 739 740 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub, 741 flush_type); 742 up_read(&adev->reset_domain->sem); 743 return; 744 } 745 746 /* The SDMA on Navi 1x has a bug which can theoretically result in memory 747 * corruption if an invalidation happens at the same time as an VA 748 * translation. Avoid this by doing the invalidation from the SDMA 749 * itself at least for GART. 750 */ 751 mutex_lock(&adev->mman.default_entity.lock); 752 r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base, 753 AMDGPU_FENCE_OWNER_UNDEFINED, 754 16 * 4, AMDGPU_IB_POOL_IMMEDIATE, 755 &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB); 756 if (r) 757 goto error_alloc; 758 759 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); 760 job->vm_needs_flush = true; 761 job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop; 762 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 763 fence = amdgpu_job_submit(job); 764 mutex_unlock(&adev->mman.default_entity.lock); 765 766 dma_fence_wait(fence, false); 767 dma_fence_put(fence); 768 769 return; 770 771 error_alloc: 772 mutex_unlock(&adev->mman.default_entity.lock); 773 dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r); 774 } 775 776 int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, 777 uint32_t flush_type, bool all_hub, 778 uint32_t inst) 779 { 780 struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; 781 struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; 782 unsigned int ndw; 783 int r, cnt = 0; 784 uint32_t seq; 785 786 /* 787 * A GPU reset should flush all TLBs anyway, so no need to do 788 * this while one is ongoing. 789 */ 790 if (!down_read_trylock(&adev->reset_domain->sem)) 791 return 0; 792 793 if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) { 794 795 if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) { 796 r = 0; 797 goto error_unlock_reset; 798 } 799 800 if (adev->gmc.flush_tlb_needs_extra_type_2) 801 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 802 2, all_hub, 803 inst); 804 805 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 806 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 807 0, all_hub, 808 inst); 809 810 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 811 flush_type, all_hub, 812 inst); 813 r = 0; 814 } else { 815 /* 2 dwords flush + 8 dwords fence */ 816 ndw = kiq->pmf->invalidate_tlbs_size + 8; 817 818 if (adev->gmc.flush_tlb_needs_extra_type_2) 819 ndw += kiq->pmf->invalidate_tlbs_size; 820 821 if (adev->gmc.flush_tlb_needs_extra_type_0) 822 ndw += kiq->pmf->invalidate_tlbs_size; 823 824 spin_lock(&adev->gfx.kiq[inst].ring_lock); 825 r = amdgpu_ring_alloc(ring, ndw); 826 if (r) { 827 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 828 goto error_unlock_reset; 829 } 830 if (adev->gmc.flush_tlb_needs_extra_type_2) 831 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); 832 833 if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) 834 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); 835 836 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); 837 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 838 if (r) { 839 amdgpu_ring_undo(ring); 840 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 841 goto error_unlock_reset; 842 } 843 844 amdgpu_ring_commit(ring); 845 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 846 847 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 848 849 might_sleep(); 850 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 851 !amdgpu_reset_pending(adev->reset_domain)) { 852 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 853 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 854 } 855 856 if (cnt > MAX_KIQ_REG_TRY) { 857 dev_err(adev->dev, "timeout waiting for kiq fence\n"); 858 r = -ETIME; 859 } else 860 r = 0; 861 } 862 863 error_unlock_reset: 864 up_read(&adev->reset_domain->sem); 865 return r; 866 } 867 868 void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, 869 uint32_t reg0, uint32_t reg1, 870 uint32_t ref, uint32_t mask, 871 uint32_t xcc_inst) 872 { 873 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_inst]; 874 struct amdgpu_ring *ring = &kiq->ring; 875 signed long r, cnt = 0; 876 unsigned long flags; 877 uint32_t seq; 878 879 if (adev->mes.ring[MES_PIPE_INST(xcc_inst, 0)].sched.ready) { 880 amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1, 881 ref, mask, xcc_inst); 882 return; 883 } 884 885 spin_lock_irqsave(&kiq->ring_lock, flags); 886 amdgpu_ring_alloc(ring, 32); 887 amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 888 ref, mask); 889 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 890 if (r) 891 goto failed_undo; 892 893 amdgpu_ring_commit(ring); 894 spin_unlock_irqrestore(&kiq->ring_lock, flags); 895 896 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 897 898 /* don't wait anymore for IRQ context */ 899 if (r < 1 && in_interrupt()) 900 goto failed_kiq; 901 902 might_sleep(); 903 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 904 !amdgpu_reset_pending(adev->reset_domain)) { 905 906 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 907 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 908 } 909 910 if (cnt > MAX_KIQ_REG_TRY) 911 goto failed_kiq; 912 913 return; 914 915 failed_undo: 916 amdgpu_ring_undo(ring); 917 spin_unlock_irqrestore(&kiq->ring_lock, flags); 918 failed_kiq: 919 dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1); 920 } 921 922 /** 923 * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ 924 * @adev: amdgpu_device pointer 925 * 926 * Check and set if an the device @adev supports Trusted Memory 927 * Zones (TMZ). 928 */ 929 void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) 930 { 931 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 932 /* RAVEN */ 933 case IP_VERSION(9, 2, 2): 934 case IP_VERSION(9, 1, 0): 935 /* RENOIR looks like RAVEN */ 936 case IP_VERSION(9, 3, 0): 937 /* GC 10.3.7 */ 938 case IP_VERSION(10, 3, 7): 939 /* GC 11.0.1 */ 940 case IP_VERSION(11, 0, 1): 941 if (amdgpu_tmz == 0) { 942 adev->gmc.tmz_enabled = false; 943 dev_info(adev->dev, 944 "Trusted Memory Zone (TMZ) feature disabled (cmd line)\n"); 945 } else { 946 adev->gmc.tmz_enabled = true; 947 dev_info(adev->dev, 948 "Trusted Memory Zone (TMZ) feature enabled\n"); 949 } 950 break; 951 case IP_VERSION(10, 1, 10): 952 case IP_VERSION(10, 1, 1): 953 case IP_VERSION(10, 1, 2): 954 case IP_VERSION(10, 1, 3): 955 case IP_VERSION(10, 3, 0): 956 case IP_VERSION(10, 3, 2): 957 case IP_VERSION(10, 3, 4): 958 case IP_VERSION(10, 3, 5): 959 case IP_VERSION(10, 3, 6): 960 /* VANGOGH */ 961 case IP_VERSION(10, 3, 1): 962 /* YELLOW_CARP*/ 963 case IP_VERSION(10, 3, 3): 964 case IP_VERSION(11, 0, 4): 965 case IP_VERSION(11, 5, 0): 966 case IP_VERSION(11, 5, 1): 967 case IP_VERSION(11, 5, 2): 968 case IP_VERSION(11, 5, 3): 969 case IP_VERSION(11, 5, 4): 970 /* Don't enable it by default yet. 971 */ 972 if (amdgpu_tmz < 1) { 973 adev->gmc.tmz_enabled = false; 974 dev_info(adev->dev, 975 "Trusted Memory Zone (TMZ) feature disabled as experimental (default)\n"); 976 } else { 977 adev->gmc.tmz_enabled = true; 978 dev_info(adev->dev, 979 "Trusted Memory Zone (TMZ) feature enabled as experimental (cmd line)\n"); 980 } 981 break; 982 default: 983 adev->gmc.tmz_enabled = false; 984 dev_info(adev->dev, 985 "Trusted Memory Zone (TMZ) feature not supported\n"); 986 break; 987 } 988 } 989 990 /** 991 * amdgpu_gmc_noretry_set -- set per asic noretry defaults 992 * @adev: amdgpu_device pointer 993 * 994 * Set a per asic default for the no-retry parameter. 995 * 996 */ 997 void amdgpu_gmc_noretry_set(struct amdgpu_device *adev) 998 { 999 struct amdgpu_gmc *gmc = &adev->gmc; 1000 uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); 1001 bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) || 1002 gc_ver == IP_VERSION(9, 4, 0) || 1003 gc_ver == IP_VERSION(9, 4, 1) || 1004 gc_ver == IP_VERSION(9, 4, 2) || 1005 gc_ver == IP_VERSION(9, 4, 3) || 1006 gc_ver == IP_VERSION(9, 4, 4) || 1007 gc_ver == IP_VERSION(9, 5, 0) || 1008 gc_ver >= IP_VERSION(10, 3, 0)); 1009 1010 /* For GFX12.1 B0, set xnack (retry) on as default */ 1011 if (gc_ver == IP_VERSION(12, 1, 0) && (adev->rev_id & 0xf) == 0x1) 1012 noretry_default = false; 1013 if (!amdgpu_sriov_xnack_support(adev)) 1014 gmc->noretry = 1; 1015 else 1016 gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry; 1017 } 1018 1019 void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, 1020 bool enable) 1021 { 1022 struct amdgpu_vmhub *hub; 1023 u32 tmp, reg, i; 1024 1025 hub = &adev->vmhub[hub_type]; 1026 for (i = 0; i < 16; i++) { 1027 reg = hub->vm_context0_cntl + hub->ctx_distance * i; 1028 1029 tmp = (hub_type == AMDGPU_GFXHUB(0)) ? 1030 RREG32_SOC15_IP(GC, reg) : 1031 RREG32_SOC15_IP(MMHUB, reg); 1032 1033 if (enable) 1034 tmp |= hub->vm_cntx_cntl_vm_fault; 1035 else 1036 tmp &= ~hub->vm_cntx_cntl_vm_fault; 1037 1038 (hub_type == AMDGPU_GFXHUB(0)) ? 1039 WREG32_SOC15_IP(GC, reg, tmp) : 1040 WREG32_SOC15_IP(MMHUB, reg, tmp); 1041 } 1042 } 1043 1044 void amdgpu_gmc_init_vga_resv_regions(struct amdgpu_device *adev) 1045 { 1046 unsigned size; 1047 1048 if (adev->gmc.is_app_apu) 1049 return; 1050 1051 /* 1052 * Some ASICs need to reserve a region of video memory to avoid access 1053 * from driver 1054 */ 1055 /* 1056 * TODO: 1057 * Currently there is a bug where some memory client outside 1058 * of the driver writes to first 8M of VRAM on S3 resume, 1059 * this overrides GART which by default gets placed in first 8M and 1060 * causes VM_FAULTS once GTT is accessed. 1061 * Keep the stolen memory reservation until the while this is not solved. 1062 */ 1063 switch (adev->asic_type) { 1064 case CHIP_VEGA10: 1065 adev->mman.keep_stolen_vga_memory = true; 1066 /* 1067 * VEGA10 SRIOV VF with MS_HYPERV host needs some firmware reserved area. 1068 */ 1069 #ifdef CONFIG_X86 1070 if (amdgpu_sriov_vf(adev) && hypervisor_is_type(X86_HYPER_MS_HYPERV)) { 1071 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_RESERVED, 1072 0x500000, 0x200000, false); 1073 } 1074 #endif 1075 break; 1076 case CHIP_RAVEN: 1077 case CHIP_RENOIR: 1078 adev->mman.keep_stolen_vga_memory = true; 1079 break; 1080 case CHIP_POLARIS10: 1081 case CHIP_POLARIS11: 1082 case CHIP_POLARIS12: 1083 /* MacBookPros with switchable graphics put VRAM at 0 when 1084 * the iGPU is enabled which results in cursor issues if 1085 * the cursor ends up at 0. Reserve vram at 0 in that case. 1086 */ 1087 if (adev->gmc.vram_start == 0) 1088 adev->mman.keep_stolen_vga_memory = true; 1089 break; 1090 default: 1091 adev->mman.keep_stolen_vga_memory = false; 1092 break; 1093 } 1094 1095 if (amdgpu_sriov_vf(adev) || 1096 !amdgpu_device_has_display_hardware(adev)) { 1097 size = 0; 1098 } else { 1099 size = amdgpu_gmc_get_vbios_fb_size(adev); 1100 1101 if (adev->mman.keep_stolen_vga_memory) 1102 size = max(size, (unsigned)AMDGPU_VBIOS_VGA_ALLOCATION); 1103 } 1104 1105 /* set to 0 if the pre-OS buffer uses up most of vram */ 1106 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 1107 size = 0; 1108 1109 if (size > AMDGPU_VBIOS_VGA_ALLOCATION) { 1110 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1111 0, AMDGPU_VBIOS_VGA_ALLOCATION, false); 1112 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_EXTENDED, 1113 AMDGPU_VBIOS_VGA_ALLOCATION, 1114 size - AMDGPU_VBIOS_VGA_ALLOCATION, false); 1115 } else { 1116 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1117 0, size, false); 1118 } 1119 } 1120 1121 /** 1122 * amdgpu_gmc_init_pdb0 - initialize PDB0 1123 * 1124 * @adev: amdgpu_device pointer 1125 * 1126 * This function is only used when GART page table is used 1127 * for FB address translatioin. In such a case, we construct 1128 * a 2-level system VM page table: PDB0->PTB, to cover both 1129 * VRAM of the hive and system memory. 1130 * 1131 * PDB0 is static, initialized once on driver initialization. 1132 * The first n entries of PDB0 are used as PTE by setting 1133 * P bit to 1, pointing to VRAM. The n+1'th entry points 1134 * to a big PTB covering system memory. 1135 * 1136 */ 1137 void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev) 1138 { 1139 int i; 1140 uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW? 1141 /* Each PDE0 (used as PTE) covers (2^vmid0_page_table_block_size)*2M 1142 */ 1143 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 1144 u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21; 1145 u64 vram_addr, vram_end; 1146 u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo); 1147 int idx; 1148 1149 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 1150 return; 1151 1152 flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE; 1153 flags |= AMDGPU_PTE_WRITEABLE; 1154 flags |= AMDGPU_PTE_SNOOPED; 1155 flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1)); 1156 flags |= AMDGPU_PDE_PTE_FLAG(adev); 1157 1158 vram_addr = adev->vm_manager.vram_base_offset; 1159 if (!amdgpu_virt_xgmi_migrate_enabled(adev)) 1160 vram_addr -= adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1161 vram_end = vram_addr + vram_size; 1162 1163 /* The first n PDE0 entries are used as PTE, 1164 * pointing to vram 1165 */ 1166 for (i = 0; vram_addr < vram_end; i++, vram_addr += pde0_page_size) 1167 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, vram_addr, flags); 1168 1169 /* The n+1'th PDE0 entry points to a huge 1170 * PTB who has more than 512 entries each 1171 * pointing to a 4K system page 1172 */ 1173 flags = AMDGPU_PTE_VALID; 1174 flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0); 1175 /* Requires gart_ptb_gpu_pa to be 4K aligned */ 1176 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags); 1177 drm_dev_exit(idx); 1178 } 1179 1180 /** 1181 * amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC 1182 * address 1183 * 1184 * @adev: amdgpu_device pointer 1185 * @mc_addr: MC address of buffer 1186 */ 1187 uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr) 1188 { 1189 return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset; 1190 } 1191 1192 /** 1193 * amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from 1194 * GPU's view 1195 * 1196 * @adev: amdgpu_device pointer 1197 * @bo: amdgpu buffer object 1198 */ 1199 uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo) 1200 { 1201 return amdgpu_gmc_vram_mc2pa(adev, amdgpu_bo_gpu_offset(bo)); 1202 } 1203 1204 int amdgpu_gmc_vram_checking(struct amdgpu_device *adev) 1205 { 1206 struct amdgpu_bo *vram_bo = NULL; 1207 uint64_t vram_gpu = 0; 1208 void *vram_ptr = NULL; 1209 1210 int ret, size = 0x100000; 1211 uint8_t cptr[10]; 1212 1213 ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, 1214 AMDGPU_GEM_DOMAIN_VRAM, 1215 &vram_bo, 1216 &vram_gpu, 1217 &vram_ptr); 1218 if (ret) 1219 return ret; 1220 1221 memset(vram_ptr, 0x86, size); 1222 memset(cptr, 0x86, 10); 1223 1224 /** 1225 * Check the start, the mid, and the end of the memory if the content of 1226 * each byte is the pattern "0x86". If yes, we suppose the vram bo is 1227 * workable. 1228 * 1229 * Note: If check the each byte of whole 1M bo, it will cost too many 1230 * seconds, so here, we just pick up three parts for emulation. 1231 */ 1232 ret = memcmp(vram_ptr, cptr, 10); 1233 if (ret) { 1234 ret = -EIO; 1235 goto release_buffer; 1236 } 1237 1238 ret = memcmp(vram_ptr + (size / 2), cptr, 10); 1239 if (ret) { 1240 ret = -EIO; 1241 goto release_buffer; 1242 } 1243 1244 ret = memcmp(vram_ptr + size - 10, cptr, 10); 1245 if (ret) { 1246 ret = -EIO; 1247 goto release_buffer; 1248 } 1249 1250 release_buffer: 1251 amdgpu_bo_free_kernel(&vram_bo, &vram_gpu, 1252 &vram_ptr); 1253 1254 return ret; 1255 } 1256 1257 static const char *nps_desc[] = { 1258 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 1259 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 1260 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 1261 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 1262 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 1263 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 1264 }; 1265 1266 static ssize_t available_memory_partition_show(struct device *dev, 1267 struct device_attribute *addr, 1268 char *buf) 1269 { 1270 struct drm_device *ddev = dev_get_drvdata(dev); 1271 struct amdgpu_device *adev = drm_to_adev(ddev); 1272 int size = 0, mode; 1273 char *sep = ""; 1274 1275 for_each_inst(mode, adev->gmc.supported_nps_modes) { 1276 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 1277 sep = ", "; 1278 } 1279 size += sysfs_emit_at(buf, size, "\n"); 1280 1281 return size; 1282 } 1283 1284 static ssize_t current_memory_partition_store(struct device *dev, 1285 struct device_attribute *attr, 1286 const char *buf, size_t count) 1287 { 1288 struct drm_device *ddev = dev_get_drvdata(dev); 1289 struct amdgpu_device *adev = drm_to_adev(ddev); 1290 enum amdgpu_memory_partition mode; 1291 struct amdgpu_hive_info *hive; 1292 int i; 1293 1294 mode = UNKNOWN_MEMORY_PARTITION_MODE; 1295 for_each_inst(i, adev->gmc.supported_nps_modes) { 1296 if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) { 1297 mode = i; 1298 break; 1299 } 1300 } 1301 1302 if (mode == UNKNOWN_MEMORY_PARTITION_MODE) 1303 return -EINVAL; 1304 1305 if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) { 1306 dev_info( 1307 adev->dev, 1308 "requested NPS mode is same as current NPS mode, skipping\n"); 1309 return count; 1310 } 1311 1312 /* If device is part of hive, all devices in the hive should request the 1313 * same mode. Hence store the requested mode in hive. 1314 */ 1315 hive = amdgpu_get_xgmi_hive(adev); 1316 if (hive) { 1317 atomic_set(&hive->requested_nps_mode, mode); 1318 amdgpu_put_xgmi_hive(hive); 1319 } else { 1320 adev->gmc.requested_nps_mode = mode; 1321 } 1322 1323 dev_info( 1324 adev->dev, 1325 "NPS mode change requested, please remove and reload the driver\n"); 1326 1327 return count; 1328 } 1329 1330 static ssize_t current_memory_partition_show( 1331 struct device *dev, struct device_attribute *addr, char *buf) 1332 { 1333 struct drm_device *ddev = dev_get_drvdata(dev); 1334 struct amdgpu_device *adev = drm_to_adev(ddev); 1335 enum amdgpu_memory_partition mode; 1336 1337 /* Only minimal precaution taken to reject requests while in reset */ 1338 if (amdgpu_in_reset(adev)) 1339 return -EPERM; 1340 1341 mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1342 if ((mode >= ARRAY_SIZE(nps_desc)) || 1343 (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode)) 1344 return sysfs_emit(buf, "UNKNOWN\n"); 1345 1346 return sysfs_emit(buf, "%s\n", nps_desc[mode]); 1347 } 1348 1349 static DEVICE_ATTR_RW(current_memory_partition); 1350 static DEVICE_ATTR_RO(available_memory_partition); 1351 1352 int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev) 1353 { 1354 bool nps_switch_support; 1355 int r = 0; 1356 1357 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1358 return 0; 1359 1360 nps_switch_support = (hweight32(adev->gmc.supported_nps_modes & 1361 AMDGPU_ALL_NPS_MASK) > 1); 1362 if (!nps_switch_support) 1363 dev_attr_current_memory_partition.attr.mode &= 1364 ~(S_IWUSR | S_IWGRP | S_IWOTH); 1365 else 1366 r = device_create_file(adev->dev, 1367 &dev_attr_available_memory_partition); 1368 1369 if (r) 1370 return r; 1371 1372 return device_create_file(adev->dev, 1373 &dev_attr_current_memory_partition); 1374 } 1375 1376 void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev) 1377 { 1378 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1379 return; 1380 1381 device_remove_file(adev->dev, &dev_attr_current_memory_partition); 1382 device_remove_file(adev->dev, &dev_attr_available_memory_partition); 1383 } 1384 1385 int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, 1386 struct amdgpu_mem_partition_info *mem_ranges, 1387 uint8_t *exp_ranges) 1388 { 1389 struct amdgpu_gmc_memrange ranges[AMDGPU_MAX_MEM_RANGES]; 1390 int range_cnt, ret, i, j; 1391 uint32_t nps_type; 1392 bool refresh; 1393 1394 if (!mem_ranges || !exp_ranges) 1395 return -EINVAL; 1396 range_cnt = AMDGPU_MAX_MEM_RANGES; 1397 refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) && 1398 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS); 1399 ret = amdgpu_discovery_get_nps_info(adev, &nps_type, ranges, &range_cnt, 1400 refresh); 1401 1402 if (ret) 1403 return ret; 1404 1405 /* TODO: For now, expect ranges and partition count to be the same. 1406 * Adjust if there are holes expected in any NPS domain. 1407 */ 1408 if (*exp_ranges && (range_cnt != *exp_ranges)) { 1409 dev_warn( 1410 adev->dev, 1411 "NPS config mismatch - expected ranges: %d discovery - nps mode: %d, nps ranges: %d", 1412 *exp_ranges, nps_type, range_cnt); 1413 ret = -EINVAL; 1414 goto err; 1415 } 1416 1417 for (i = 0; i < range_cnt; ++i) { 1418 if (ranges[i].base_address >= ranges[i].limit_address) { 1419 dev_warn( 1420 adev->dev, 1421 "Invalid NPS range - nps mode: %d, range[%d]: base: %llx limit: %llx", 1422 nps_type, i, ranges[i].base_address, 1423 ranges[i].limit_address); 1424 ret = -EINVAL; 1425 goto err; 1426 } 1427 1428 /* Check for overlaps, not expecting any now */ 1429 for (j = i - 1; j >= 0; j--) { 1430 if (max(ranges[j].base_address, 1431 ranges[i].base_address) <= 1432 min(ranges[j].limit_address, 1433 ranges[i].limit_address)) { 1434 dev_warn( 1435 adev->dev, 1436 "overlapping ranges detected [ %llx - %llx ] | [%llx - %llx]", 1437 ranges[j].base_address, 1438 ranges[j].limit_address, 1439 ranges[i].base_address, 1440 ranges[i].limit_address); 1441 ret = -EINVAL; 1442 goto err; 1443 } 1444 } 1445 1446 mem_ranges[i].range.fpfn = 1447 (ranges[i].base_address - 1448 adev->vm_manager.vram_base_offset) >> 1449 AMDGPU_GPU_PAGE_SHIFT; 1450 mem_ranges[i].range.lpfn = 1451 (ranges[i].limit_address - 1452 adev->vm_manager.vram_base_offset) >> 1453 AMDGPU_GPU_PAGE_SHIFT; 1454 mem_ranges[i].size = 1455 ranges[i].limit_address - ranges[i].base_address + 1; 1456 } 1457 1458 if (!*exp_ranges) 1459 *exp_ranges = range_cnt; 1460 err: 1461 return ret; 1462 } 1463 1464 int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev, 1465 int nps_mode) 1466 { 1467 /* Not supported on VF devices and APUs */ 1468 if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU)) 1469 return -EOPNOTSUPP; 1470 1471 if (!adev->psp.funcs) { 1472 dev_err(adev->dev, 1473 "PSP interface not available for nps mode change request"); 1474 return -EINVAL; 1475 } 1476 1477 return psp_memory_partition(&adev->psp, nps_mode); 1478 } 1479 1480 static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev, 1481 int req_nps_mode, 1482 int cur_nps_mode) 1483 { 1484 return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) == 1485 BIT(req_nps_mode)) && 1486 req_nps_mode != cur_nps_mode); 1487 } 1488 1489 void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev) 1490 { 1491 int req_nps_mode, cur_nps_mode, r; 1492 struct amdgpu_hive_info *hive; 1493 1494 if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes || 1495 !adev->gmc.gmc_funcs->request_mem_partition_mode) 1496 return; 1497 1498 cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1499 hive = amdgpu_get_xgmi_hive(adev); 1500 if (hive) { 1501 req_nps_mode = atomic_read(&hive->requested_nps_mode); 1502 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, 1503 cur_nps_mode)) { 1504 amdgpu_put_xgmi_hive(hive); 1505 return; 1506 } 1507 r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode); 1508 amdgpu_put_xgmi_hive(hive); 1509 goto out; 1510 } 1511 1512 req_nps_mode = adev->gmc.requested_nps_mode; 1513 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode)) 1514 return; 1515 1516 /* even if this fails, we should let driver unload w/o blocking */ 1517 r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode); 1518 out: 1519 if (r) 1520 dev_err(adev->dev, "NPS mode change request failed\n"); 1521 else 1522 dev_info( 1523 adev->dev, 1524 "NPS mode change request done, reload driver to complete the change\n"); 1525 } 1526 1527 bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) 1528 { 1529 if (adev->gmc.gmc_funcs->need_reset_on_init) 1530 return adev->gmc.gmc_funcs->need_reset_on_init(adev); 1531 1532 return false; 1533 } 1534 1535 enum amdgpu_memory_partition 1536 amdgpu_gmc_get_vf_memory_partition(struct amdgpu_device *adev) 1537 { 1538 switch (adev->gmc.num_mem_partitions) { 1539 case 0: 1540 return UNKNOWN_MEMORY_PARTITION_MODE; 1541 case 1: 1542 return AMDGPU_NPS1_PARTITION_MODE; 1543 case 2: 1544 return AMDGPU_NPS2_PARTITION_MODE; 1545 case 4: 1546 return AMDGPU_NPS4_PARTITION_MODE; 1547 case 8: 1548 return AMDGPU_NPS8_PARTITION_MODE; 1549 default: 1550 return AMDGPU_NPS1_PARTITION_MODE; 1551 } 1552 } 1553 1554 enum amdgpu_memory_partition 1555 amdgpu_gmc_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes) 1556 { 1557 enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE; 1558 1559 if (adev->nbio.funcs && 1560 adev->nbio.funcs->get_memory_partition_mode) 1561 mode = adev->nbio.funcs->get_memory_partition_mode(adev, 1562 supp_modes); 1563 else 1564 dev_warn(adev->dev, "memory partition mode query is not supported\n"); 1565 1566 return mode; 1567 } 1568 1569 enum amdgpu_memory_partition 1570 amdgpu_gmc_query_memory_partition(struct amdgpu_device *adev) 1571 { 1572 if (amdgpu_sriov_vf(adev)) 1573 return amdgpu_gmc_get_vf_memory_partition(adev); 1574 else 1575 return amdgpu_gmc_get_memory_partition(adev, NULL); 1576 } 1577 1578 static bool amdgpu_gmc_validate_partition_info(struct amdgpu_device *adev) 1579 { 1580 enum amdgpu_memory_partition mode; 1581 u32 supp_modes; 1582 bool valid; 1583 1584 mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes); 1585 1586 /* Mode detected by hardware not present in supported modes */ 1587 if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && 1588 !(BIT(mode - 1) & supp_modes)) 1589 return false; 1590 1591 switch (mode) { 1592 case UNKNOWN_MEMORY_PARTITION_MODE: 1593 case AMDGPU_NPS1_PARTITION_MODE: 1594 valid = (adev->gmc.num_mem_partitions == 1); 1595 break; 1596 case AMDGPU_NPS2_PARTITION_MODE: 1597 valid = (adev->gmc.num_mem_partitions == 2); 1598 break; 1599 case AMDGPU_NPS4_PARTITION_MODE: 1600 valid = (adev->gmc.num_mem_partitions == 3 || 1601 adev->gmc.num_mem_partitions == 4); 1602 break; 1603 case AMDGPU_NPS8_PARTITION_MODE: 1604 valid = (adev->gmc.num_mem_partitions == 8); 1605 break; 1606 default: 1607 valid = false; 1608 } 1609 1610 return valid; 1611 } 1612 1613 static bool amdgpu_gmc_is_node_present(int *node_ids, int num_ids, int nid) 1614 { 1615 int i; 1616 1617 /* Check if node with id 'nid' is present in 'node_ids' array */ 1618 for (i = 0; i < num_ids; ++i) 1619 if (node_ids[i] == nid) 1620 return true; 1621 1622 return false; 1623 } 1624 1625 static void 1626 amdgpu_gmc_init_acpi_mem_ranges(struct amdgpu_device *adev, 1627 struct amdgpu_mem_partition_info *mem_ranges) 1628 { 1629 struct amdgpu_numa_info numa_info; 1630 int node_ids[AMDGPU_MAX_MEM_RANGES]; 1631 int num_ranges = 0, ret; 1632 int num_xcc, xcc_id; 1633 uint32_t xcc_mask; 1634 1635 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1636 xcc_mask = (1U << num_xcc) - 1; 1637 1638 for_each_inst(xcc_id, xcc_mask) { 1639 ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info); 1640 if (ret) 1641 continue; 1642 1643 if (numa_info.nid == NUMA_NO_NODE) { 1644 mem_ranges[0].size = numa_info.size; 1645 mem_ranges[0].numa.node = numa_info.nid; 1646 num_ranges = 1; 1647 break; 1648 } 1649 1650 if (amdgpu_gmc_is_node_present(node_ids, num_ranges, 1651 numa_info.nid)) 1652 continue; 1653 1654 node_ids[num_ranges] = numa_info.nid; 1655 mem_ranges[num_ranges].numa.node = numa_info.nid; 1656 mem_ranges[num_ranges].size = numa_info.size; 1657 ++num_ranges; 1658 } 1659 1660 adev->gmc.num_mem_partitions = num_ranges; 1661 } 1662 1663 void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev, 1664 struct amdgpu_mem_partition_info *mem_ranges) 1665 { 1666 enum amdgpu_memory_partition mode; 1667 u32 start_addr = 0, size; 1668 int i, r, l; 1669 1670 mode = amdgpu_gmc_query_memory_partition(adev); 1671 1672 switch (mode) { 1673 case UNKNOWN_MEMORY_PARTITION_MODE: 1674 adev->gmc.num_mem_partitions = 0; 1675 break; 1676 case AMDGPU_NPS1_PARTITION_MODE: 1677 adev->gmc.num_mem_partitions = 1; 1678 break; 1679 case AMDGPU_NPS2_PARTITION_MODE: 1680 adev->gmc.num_mem_partitions = 2; 1681 break; 1682 case AMDGPU_NPS4_PARTITION_MODE: 1683 if (adev->flags & AMD_IS_APU) 1684 adev->gmc.num_mem_partitions = 3; 1685 else 1686 adev->gmc.num_mem_partitions = 4; 1687 break; 1688 case AMDGPU_NPS8_PARTITION_MODE: 1689 adev->gmc.num_mem_partitions = 8; 1690 break; 1691 default: 1692 adev->gmc.num_mem_partitions = 1; 1693 break; 1694 } 1695 1696 /* Use NPS range info, if populated */ 1697 r = amdgpu_gmc_get_nps_memranges(adev, mem_ranges, 1698 &adev->gmc.num_mem_partitions); 1699 if (!r) { 1700 l = 0; 1701 for (i = 1; i < adev->gmc.num_mem_partitions; ++i) { 1702 if (mem_ranges[i].range.lpfn > 1703 mem_ranges[i - 1].range.lpfn) 1704 l = i; 1705 } 1706 1707 } else { 1708 if (!adev->gmc.num_mem_partitions) { 1709 dev_warn(adev->dev, 1710 "Not able to detect NPS mode, fall back to NPS1\n"); 1711 adev->gmc.num_mem_partitions = 1; 1712 } 1713 /* Fallback to sw based calculation */ 1714 size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT; 1715 size /= adev->gmc.num_mem_partitions; 1716 1717 for (i = 0; i < adev->gmc.num_mem_partitions; ++i) { 1718 mem_ranges[i].range.fpfn = start_addr; 1719 mem_ranges[i].size = 1720 ((u64)size << AMDGPU_GPU_PAGE_SHIFT); 1721 mem_ranges[i].range.lpfn = start_addr + size - 1; 1722 start_addr += size; 1723 } 1724 1725 l = adev->gmc.num_mem_partitions - 1; 1726 } 1727 1728 /* Adjust the last one */ 1729 mem_ranges[l].range.lpfn = 1730 (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1; 1731 mem_ranges[l].size = 1732 adev->gmc.real_vram_size - 1733 ((u64)mem_ranges[l].range.fpfn << AMDGPU_GPU_PAGE_SHIFT); 1734 } 1735 1736 int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev) 1737 { 1738 bool valid; 1739 1740 adev->gmc.mem_partitions = kzalloc_objs(struct amdgpu_mem_partition_info, 1741 AMDGPU_MAX_MEM_RANGES); 1742 if (!adev->gmc.mem_partitions) 1743 return -ENOMEM; 1744 1745 if (adev->gmc.is_app_apu) 1746 amdgpu_gmc_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions); 1747 else 1748 amdgpu_gmc_init_sw_mem_ranges(adev, adev->gmc.mem_partitions); 1749 1750 if (amdgpu_sriov_vf(adev)) 1751 valid = true; 1752 else 1753 valid = amdgpu_gmc_validate_partition_info(adev); 1754 if (!valid) { 1755 /* TODO: handle invalid case */ 1756 dev_warn(adev->dev, 1757 "Mem ranges not matching with hardware config\n"); 1758 } 1759 1760 return 0; 1761 } 1762 1763 int amdgpu_gmc_get_vram_info(struct amdgpu_device *adev, 1764 int *vram_width, int *vram_type, int *vram_vendor) 1765 { 1766 int ret = 0; 1767 1768 if (adev->flags & AMD_IS_APU) 1769 return amdgpu_atomfirmware_get_integrated_system_info(adev, 1770 vram_width, vram_type, vram_vendor); 1771 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1772 case IP_VERSION(12, 0, 0): 1773 case IP_VERSION(12, 0, 1): 1774 return amdgpu_atomfirmware_get_umc_info(adev, 1775 vram_width, vram_type, vram_vendor); 1776 case IP_VERSION(9, 5, 0): 1777 case IP_VERSION(9, 4, 4): 1778 case IP_VERSION(9, 4, 3): 1779 ret = amdgpu_atomfirmware_get_umc_info(adev, 1780 vram_width, vram_type, vram_vendor); 1781 if (vram_width && !ret) 1782 *vram_width *= hweight32(adev->aid_mask); 1783 return ret; 1784 default: 1785 return amdgpu_atomfirmware_get_vram_info(adev, 1786 vram_width, vram_type, vram_vendor); 1787 } 1788 return 0; 1789 } 1790