1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 27 #include <linux/io-64-nonatomic-lo-hi.h> 28 #ifdef CONFIG_X86 29 #include <asm/hypervisor.h> 30 #endif 31 32 #include "amdgpu.h" 33 #include "amdgpu_gmc.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_reset.h" 36 #include "amdgpu_xgmi.h" 37 #include "amdgpu_atomfirmware.h" 38 39 #include <drm/drm_drv.h> 40 #include <drm/ttm/ttm_tt.h> 41 42 static const u64 four_gb = 0x100000000ULL; 43 44 bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev) 45 { 46 return adev->gmc.xgmi.connected_to_cpu || amdgpu_virt_xgmi_migrate_enabled(adev); 47 } 48 49 /** 50 * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0 51 * 52 * @adev: amdgpu_device pointer 53 * 54 * Allocate video memory for pdb0 and map it for CPU access 55 * Returns 0 for success, error for failure. 56 */ 57 int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev) 58 { 59 int r; 60 struct amdgpu_bo_param bp; 61 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 62 uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21; 63 uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >> pde0_page_shift; 64 65 memset(&bp, 0, sizeof(bp)); 66 bp.size = PAGE_ALIGN((npdes + 1) * 8); 67 bp.byte_align = PAGE_SIZE; 68 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 69 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 70 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 71 bp.type = ttm_bo_type_kernel; 72 bp.resv = NULL; 73 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 74 75 r = amdgpu_bo_create(adev, &bp, &adev->gmc.pdb0_bo); 76 if (r) 77 return r; 78 79 r = amdgpu_bo_reserve(adev->gmc.pdb0_bo, false); 80 if (unlikely(r != 0)) 81 goto bo_reserve_failure; 82 83 r = amdgpu_bo_pin(adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM); 84 if (r) 85 goto bo_pin_failure; 86 r = amdgpu_bo_kmap(adev->gmc.pdb0_bo, &adev->gmc.ptr_pdb0); 87 if (r) 88 goto bo_kmap_failure; 89 90 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 91 return 0; 92 93 bo_kmap_failure: 94 amdgpu_bo_unpin(adev->gmc.pdb0_bo); 95 bo_pin_failure: 96 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 97 bo_reserve_failure: 98 amdgpu_bo_unref(&adev->gmc.pdb0_bo); 99 return r; 100 } 101 102 /** 103 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO 104 * 105 * @bo: the BO to get the PDE for 106 * @level: the level in the PD hirarchy 107 * @addr: resulting addr 108 * @flags: resulting flags 109 * 110 * Get the address and flags to be used for a PDE (Page Directory Entry). 111 */ 112 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, 113 uint64_t *addr, uint64_t *flags) 114 { 115 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 116 117 switch (bo->tbo.resource->mem_type) { 118 case TTM_PL_TT: 119 *addr = bo->tbo.ttm->dma_address[0]; 120 break; 121 case TTM_PL_VRAM: 122 *addr = amdgpu_bo_gpu_offset(bo); 123 break; 124 default: 125 *addr = 0; 126 break; 127 } 128 *flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, bo->tbo.resource); 129 amdgpu_gmc_get_vm_pde(adev, level, addr, flags); 130 } 131 132 /* 133 * amdgpu_gmc_pd_addr - return the address of the root directory 134 */ 135 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo) 136 { 137 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 138 uint64_t pd_addr; 139 140 /* TODO: move that into ASIC specific code */ 141 if (adev->asic_type >= CHIP_VEGA10) { 142 uint64_t flags = AMDGPU_PTE_VALID; 143 144 amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags); 145 pd_addr |= flags; 146 } else { 147 pd_addr = amdgpu_bo_gpu_offset(bo); 148 } 149 return pd_addr; 150 } 151 152 /** 153 * amdgpu_gmc_set_pte_pde - update the page tables using CPU 154 * 155 * @adev: amdgpu_device pointer 156 * @cpu_pt_addr: cpu address of the page table 157 * @gpu_page_idx: entry in the page table to update 158 * @addr: dst addr to write into pte/pde 159 * @flags: access flags 160 * 161 * Update the page tables using CPU. 162 */ 163 int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 164 uint32_t gpu_page_idx, uint64_t addr, 165 uint64_t flags) 166 { 167 void __iomem *ptr = (void *)cpu_pt_addr; 168 uint64_t value; 169 170 /* 171 * The following is for PTE only. GART does not have PDEs. 172 */ 173 value = addr & 0x0000FFFFFFFFF000ULL; 174 value |= flags; 175 writeq(value, ptr + (gpu_page_idx * 8)); 176 177 return 0; 178 } 179 180 /** 181 * amdgpu_gmc_agp_addr - return the address in the AGP address space 182 * 183 * @bo: TTM BO which needs the address, must be in GTT domain 184 * 185 * Tries to figure out how to access the BO through the AGP aperture. Returns 186 * AMDGPU_BO_INVALID_OFFSET if that is not possible. 187 */ 188 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) 189 { 190 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 191 192 if (!bo->ttm) 193 return AMDGPU_BO_INVALID_OFFSET; 194 195 if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached) 196 return AMDGPU_BO_INVALID_OFFSET; 197 198 if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) 199 return AMDGPU_BO_INVALID_OFFSET; 200 201 return adev->gmc.agp_start + bo->ttm->dma_address[0]; 202 } 203 204 /** 205 * amdgpu_gmc_vram_location - try to find VRAM location 206 * 207 * @adev: amdgpu device structure holding all necessary information 208 * @mc: memory controller structure holding memory information 209 * @base: base address at which to put VRAM 210 * 211 * Function will try to place VRAM at base address provided 212 * as parameter. 213 */ 214 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 215 u64 base) 216 { 217 uint64_t vis_limit = (uint64_t)amdgpu_vis_vram_limit << 20; 218 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; 219 220 mc->vram_start = base; 221 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 222 if (limit < mc->real_vram_size) 223 mc->real_vram_size = limit; 224 225 if (vis_limit && vis_limit < mc->visible_vram_size) 226 mc->visible_vram_size = vis_limit; 227 228 if (mc->real_vram_size < mc->visible_vram_size) 229 mc->visible_vram_size = mc->real_vram_size; 230 231 if (mc->xgmi.num_physical_nodes == 0) { 232 mc->fb_start = mc->vram_start; 233 mc->fb_end = mc->vram_end; 234 } 235 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 236 mc->mc_vram_size >> 20, mc->vram_start, 237 mc->vram_end, mc->real_vram_size >> 20); 238 } 239 240 /** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture 241 * 242 * @adev: amdgpu device structure holding all necessary information 243 * @mc: memory controller structure holding memory information 244 * 245 * This function is only used if use GART for FB translation. In such 246 * case, we use sysvm aperture (vmid0 page tables) for both vram 247 * and gart (aka system memory) access. 248 * 249 * GPUVM (and our organization of vmid0 page tables) require sysvm 250 * aperture to be placed at a location aligned with 8 times of native 251 * page size. For example, if vm_context0_cntl.page_table_block_size 252 * is 12, then native page size is 8G (2M*2^12), sysvm should start 253 * with a 64G aligned address. For simplicity, we just put sysvm at 254 * address 0. So vram start at address 0 and gart is right after vram. 255 */ 256 void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 257 { 258 u64 hive_vram_start = 0; 259 u64 hive_vram_end = mc->xgmi.node_segment_size * mc->xgmi.num_physical_nodes - 1; 260 mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id; 261 mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1; 262 /* node_segment_size may not 4GB aligned on SRIOV, align up is needed. */ 263 mc->gart_start = ALIGN(hive_vram_end + 1, four_gb); 264 mc->gart_end = mc->gart_start + mc->gart_size - 1; 265 if (amdgpu_virt_xgmi_migrate_enabled(adev)) { 266 /* set mc->vram_start to 0 to switch the returned GPU address of 267 * amdgpu_bo_create_reserved() from FB aperture to GART aperture. 268 */ 269 mc->vram_start = 0; 270 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 271 mc->visible_vram_size = min(mc->visible_vram_size, mc->real_vram_size); 272 } else { 273 mc->fb_start = hive_vram_start; 274 mc->fb_end = hive_vram_end; 275 } 276 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 277 mc->mc_vram_size >> 20, mc->vram_start, 278 mc->vram_end, mc->real_vram_size >> 20); 279 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 280 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 281 } 282 283 /** 284 * amdgpu_gmc_gart_location - try to find GART location 285 * 286 * @adev: amdgpu device structure holding all necessary information 287 * @mc: memory controller structure holding memory information 288 * @gart_placement: GART placement policy with respect to VRAM 289 * 290 * Function will try to place GART before or after VRAM. 291 * If GART size is bigger than space left then we ajust GART size. 292 * Thus function will never fails. 293 */ 294 void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 295 enum amdgpu_gart_placement gart_placement) 296 { 297 u64 size_af, size_bf; 298 /*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/ 299 u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1); 300 301 /* VCE doesn't like it when BOs cross a 4GB segment, so align 302 * the GART base on a 4GB boundary as well. 303 */ 304 size_bf = mc->fb_start; 305 size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb); 306 307 if (mc->gart_size > max(size_bf, size_af)) { 308 dev_warn(adev->dev, "limiting GART\n"); 309 mc->gart_size = max(size_bf, size_af); 310 } 311 312 switch (gart_placement) { 313 case AMDGPU_GART_PLACEMENT_HIGH: 314 mc->gart_start = max_mc_address - mc->gart_size + 1; 315 break; 316 case AMDGPU_GART_PLACEMENT_LOW: 317 if (size_bf >= mc->gart_size) 318 mc->gart_start = 0; 319 else 320 mc->gart_start = ALIGN(mc->fb_end, four_gb); 321 break; 322 case AMDGPU_GART_PLACEMENT_BEST_FIT: 323 default: 324 if ((size_bf >= mc->gart_size && size_bf < size_af) || 325 (size_af < mc->gart_size)) 326 mc->gart_start = 0; 327 else 328 mc->gart_start = max_mc_address - mc->gart_size + 1; 329 break; 330 } 331 332 mc->gart_start &= ~(four_gb - 1); 333 mc->gart_end = mc->gart_start + mc->gart_size - 1; 334 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 335 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 336 } 337 338 /** 339 * amdgpu_gmc_agp_location - try to find AGP location 340 * @adev: amdgpu device structure holding all necessary information 341 * @mc: memory controller structure holding memory information 342 * 343 * Function will place try to find a place for the AGP BAR in the MC address 344 * space. 345 * 346 * AGP BAR will be assigned the largest available hole in the address space. 347 * Should be called after VRAM and GART locations are setup. 348 */ 349 void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 350 { 351 const uint64_t sixteen_gb = 1ULL << 34; 352 const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1); 353 u64 size_af, size_bf; 354 355 if (mc->fb_start > mc->gart_start) { 356 size_bf = (mc->fb_start & sixteen_gb_mask) - 357 ALIGN(mc->gart_end + 1, sixteen_gb); 358 size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb); 359 } else { 360 size_bf = mc->fb_start & sixteen_gb_mask; 361 size_af = (mc->gart_start & sixteen_gb_mask) - 362 ALIGN(mc->fb_end + 1, sixteen_gb); 363 } 364 365 if (size_bf > size_af) { 366 mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask; 367 mc->agp_size = size_bf; 368 } else { 369 mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb); 370 mc->agp_size = size_af; 371 } 372 373 mc->agp_end = mc->agp_start + mc->agp_size - 1; 374 dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n", 375 mc->agp_size >> 20, mc->agp_start, mc->agp_end); 376 } 377 378 /** 379 * amdgpu_gmc_set_agp_default - Set the default AGP aperture value. 380 * @adev: amdgpu device structure holding all necessary information 381 * @mc: memory controller structure holding memory information 382 * 383 * To disable the AGP aperture, you need to set the start to a larger 384 * value than the end. This function sets the default value which 385 * can then be overridden using amdgpu_gmc_agp_location() if you want 386 * to enable the AGP aperture on a specific chip. 387 * 388 */ 389 void amdgpu_gmc_set_agp_default(struct amdgpu_device *adev, 390 struct amdgpu_gmc *mc) 391 { 392 mc->agp_start = 0xffffffffffff; 393 mc->agp_end = 0; 394 mc->agp_size = 0; 395 } 396 397 /** 398 * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid 399 * 400 * @addr: 48 bit physical address, page aligned (36 significant bits) 401 * @pasid: 16 bit process address space identifier 402 */ 403 static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid) 404 { 405 return addr << 4 | pasid; 406 } 407 408 /** 409 * amdgpu_gmc_filter_faults - filter VM faults 410 * 411 * @adev: amdgpu device structure 412 * @ih: interrupt ring that the fault received from 413 * @addr: address of the VM fault 414 * @pasid: PASID of the process causing the fault 415 * @timestamp: timestamp of the fault 416 * 417 * Returns: 418 * True if the fault was filtered and should not be processed further. 419 * False if the fault is a new one and needs to be handled. 420 */ 421 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, 422 struct amdgpu_ih_ring *ih, uint64_t addr, 423 uint16_t pasid, uint64_t timestamp) 424 { 425 struct amdgpu_gmc *gmc = &adev->gmc; 426 uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid); 427 struct amdgpu_gmc_fault *fault; 428 uint32_t hash; 429 430 /* Stale retry fault if timestamp goes backward */ 431 if (amdgpu_ih_ts_after(timestamp, ih->processed_timestamp)) 432 return true; 433 434 /* If we don't have space left in the ring buffer return immediately */ 435 stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) - 436 AMDGPU_GMC_FAULT_TIMEOUT; 437 if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp) 438 return true; 439 440 /* Try to find the fault in the hash */ 441 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 442 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 443 while (fault->timestamp >= stamp) { 444 uint64_t tmp; 445 446 if (atomic64_read(&fault->key) == key) { 447 /* 448 * if we get a fault which is already present in 449 * the fault_ring and the timestamp of 450 * the fault is after the expired timestamp, 451 * then this is a new fault that needs to be added 452 * into the fault ring. 453 */ 454 if (fault->timestamp_expiry != 0 && 455 amdgpu_ih_ts_after(fault->timestamp_expiry, 456 timestamp)) 457 break; 458 else 459 return true; 460 } 461 462 tmp = fault->timestamp; 463 fault = &gmc->fault_ring[fault->next]; 464 465 /* Check if the entry was reused */ 466 if (fault->timestamp >= tmp) 467 break; 468 } 469 470 /* Add the fault to the ring */ 471 fault = &gmc->fault_ring[gmc->last_fault]; 472 atomic64_set(&fault->key, key); 473 fault->timestamp = timestamp; 474 475 /* And update the hash */ 476 fault->next = gmc->fault_hash[hash].idx; 477 gmc->fault_hash[hash].idx = gmc->last_fault++; 478 return false; 479 } 480 481 /** 482 * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter 483 * 484 * @adev: amdgpu device structure 485 * @addr: address of the VM fault 486 * @pasid: PASID of the process causing the fault 487 * 488 * Remove the address from fault filter, then future vm fault on this address 489 * will pass to retry fault handler to recover. 490 */ 491 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr, 492 uint16_t pasid) 493 { 494 struct amdgpu_gmc *gmc = &adev->gmc; 495 uint64_t key = amdgpu_gmc_fault_key(addr, pasid); 496 struct amdgpu_ih_ring *ih; 497 struct amdgpu_gmc_fault *fault; 498 uint32_t last_wptr; 499 uint64_t last_ts; 500 uint32_t hash; 501 uint64_t tmp; 502 503 if (adev->irq.retry_cam_enabled) 504 return; 505 else if (adev->irq.ih1.ring_size) 506 ih = &adev->irq.ih1; 507 else if (adev->irq.ih_soft.enabled) 508 ih = &adev->irq.ih_soft; 509 else 510 return; 511 512 /* Get the WPTR of the last entry in IH ring */ 513 last_wptr = amdgpu_ih_get_wptr(adev, ih); 514 /* Order wptr with ring data. */ 515 rmb(); 516 /* Get the timetamp of the last entry in IH ring */ 517 last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1); 518 519 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 520 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 521 do { 522 if (atomic64_read(&fault->key) == key) { 523 /* 524 * Update the timestamp when this fault 525 * expired. 526 */ 527 fault->timestamp_expiry = last_ts; 528 break; 529 } 530 531 tmp = fault->timestamp; 532 fault = &gmc->fault_ring[fault->next]; 533 } while (fault->timestamp < tmp); 534 } 535 536 int amdgpu_gmc_handle_retry_fault(struct amdgpu_device *adev, 537 struct amdgpu_iv_entry *entry, 538 u64 addr, 539 u32 cam_index, 540 u32 node_id, 541 bool write_fault) 542 { 543 int ret; 544 545 if (adev->irq.retry_cam_enabled) { 546 /* Delegate it to a different ring if the hardware hasn't 547 * already done it. 548 */ 549 if (entry->ih == &adev->irq.ih) { 550 amdgpu_irq_delegate(adev, entry, 8); 551 return 1; 552 } 553 554 ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 555 addr, entry->timestamp, write_fault); 556 WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); 557 if (ret) 558 return 1; 559 } else { 560 /* Process it only if it's the first fault for this address */ 561 if (entry->ih != &adev->irq.ih_soft && 562 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 563 entry->timestamp)) 564 return 1; 565 566 /* Delegate it to a different ring if the hardware hasn't 567 * already done it. 568 */ 569 if (entry->ih == &adev->irq.ih) { 570 amdgpu_irq_delegate(adev, entry, 8); 571 return 1; 572 } 573 574 /* Try to handle the recoverable page faults by filling page 575 * tables 576 */ 577 if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 578 addr, entry->timestamp, write_fault)) 579 return 1; 580 } 581 return 0; 582 } 583 584 int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev) 585 { 586 int r; 587 588 /* umc ras block */ 589 r = amdgpu_umc_ras_sw_init(adev); 590 if (r) 591 return r; 592 593 /* mmhub ras block */ 594 r = amdgpu_mmhub_ras_sw_init(adev); 595 if (r) 596 return r; 597 598 /* hdp ras block */ 599 r = amdgpu_hdp_ras_sw_init(adev); 600 if (r) 601 return r; 602 603 /* mca.x ras block */ 604 r = amdgpu_mca_mp0_ras_sw_init(adev); 605 if (r) 606 return r; 607 608 r = amdgpu_mca_mp1_ras_sw_init(adev); 609 if (r) 610 return r; 611 612 r = amdgpu_mca_mpio_ras_sw_init(adev); 613 if (r) 614 return r; 615 616 /* xgmi ras block */ 617 r = amdgpu_xgmi_ras_sw_init(adev); 618 if (r) 619 return r; 620 621 return 0; 622 } 623 624 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev) 625 { 626 return 0; 627 } 628 629 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev) 630 { 631 632 } 633 634 /* 635 * The latest engine allocation on gfx9/10 is: 636 * Engine 2, 3: firmware 637 * Engine 0, 1, 4~16: amdgpu ring, 638 * subject to change when ring number changes 639 * Engine 17: Gart flushes 640 */ 641 #define AMDGPU_VMHUB_INV_ENG_BITMAP 0x1FFF3 642 643 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) 644 { 645 struct amdgpu_ring *ring; 646 unsigned vm_inv_engs[AMDGPU_MAX_VMHUBS] = {0}; 647 unsigned i; 648 unsigned vmhub, inv_eng; 649 struct amdgpu_ring *shared_ring; 650 651 /* init the vm inv eng for all vmhubs */ 652 for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 653 vm_inv_engs[i] = AMDGPU_VMHUB_INV_ENG_BITMAP; 654 /* reserve engine 5 for firmware */ 655 if (adev->enable_mes) 656 vm_inv_engs[i] &= ~(1 << 5); 657 /* reserve engine 6 for uni mes */ 658 if (adev->enable_uni_mes) 659 vm_inv_engs[i] &= ~(1 << 6); 660 /* reserve mmhub engine 3 for firmware */ 661 if (adev->enable_umsch_mm) 662 vm_inv_engs[i] &= ~(1 << 3); 663 } 664 665 for (i = 0; i < adev->num_rings; ++i) { 666 ring = adev->rings[i]; 667 vmhub = ring->vm_hub; 668 669 if (ring == &adev->mes.ring[0] || 670 ring == &adev->mes.ring[1] || 671 ring == &adev->umsch_mm.ring || 672 ring == &adev->cper.ring_buf) 673 continue; 674 675 /* Skip if the ring is a shared ring */ 676 if (amdgpu_sdma_is_shared_inv_eng(adev, ring)) 677 continue; 678 679 inv_eng = ffs(vm_inv_engs[vmhub]); 680 if (!inv_eng) { 681 dev_err(adev->dev, "no VM inv eng for ring %s\n", 682 ring->name); 683 return -EINVAL; 684 } 685 686 ring->vm_inv_eng = inv_eng - 1; 687 vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng); 688 689 dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n", 690 ring->name, ring->vm_inv_eng, ring->vm_hub); 691 /* SDMA has a special packet which allows it to use the same 692 * invalidation engine for all the rings in one instance. 693 * Therefore, we do not allocate a separate VM invalidation engine 694 * for SDMA page rings. Instead, they share the VM invalidation 695 * engine with the SDMA gfx ring. This change ensures efficient 696 * resource management and avoids the issue of insufficient VM 697 * invalidation engines. 698 */ 699 shared_ring = amdgpu_sdma_get_shared_ring(adev, ring); 700 if (shared_ring) { 701 shared_ring->vm_inv_eng = ring->vm_inv_eng; 702 dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n", 703 ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub); 704 continue; 705 } 706 } 707 708 return 0; 709 } 710 711 void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 712 uint32_t vmhub, uint32_t flush_type) 713 { 714 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 715 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 716 struct dma_fence *fence; 717 struct amdgpu_job *job; 718 int r; 719 720 if (!hub->sdma_invalidation_workaround || vmid || 721 !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || 722 !ring->sched.ready) { 723 /* 724 * A GPU reset should flush all TLBs anyway, so no need to do 725 * this while one is ongoing. 726 */ 727 if (!down_read_trylock(&adev->reset_domain->sem)) 728 return; 729 730 if (adev->gmc.flush_tlb_needs_extra_type_2) 731 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 732 vmhub, 2); 733 734 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 735 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 736 vmhub, 0); 737 738 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub, 739 flush_type); 740 up_read(&adev->reset_domain->sem); 741 return; 742 } 743 744 /* The SDMA on Navi 1x has a bug which can theoretically result in memory 745 * corruption if an invalidation happens at the same time as an VA 746 * translation. Avoid this by doing the invalidation from the SDMA 747 * itself at least for GART. 748 */ 749 mutex_lock(&adev->mman.default_entity.lock); 750 r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base, 751 AMDGPU_FENCE_OWNER_UNDEFINED, 752 16 * 4, AMDGPU_IB_POOL_IMMEDIATE, 753 &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB); 754 if (r) 755 goto error_alloc; 756 757 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); 758 job->vm_needs_flush = true; 759 job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop; 760 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 761 fence = amdgpu_job_submit(job); 762 mutex_unlock(&adev->mman.default_entity.lock); 763 764 dma_fence_wait(fence, false); 765 dma_fence_put(fence); 766 767 return; 768 769 error_alloc: 770 mutex_unlock(&adev->mman.default_entity.lock); 771 dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r); 772 } 773 774 int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, 775 uint32_t flush_type, bool all_hub, 776 uint32_t inst) 777 { 778 struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; 779 struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; 780 unsigned int ndw; 781 int r, cnt = 0; 782 uint32_t seq; 783 784 /* 785 * A GPU reset should flush all TLBs anyway, so no need to do 786 * this while one is ongoing. 787 */ 788 if (!down_read_trylock(&adev->reset_domain->sem)) 789 return 0; 790 791 if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) { 792 793 if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) { 794 r = 0; 795 goto error_unlock_reset; 796 } 797 798 if (adev->gmc.flush_tlb_needs_extra_type_2) 799 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 800 2, all_hub, 801 inst); 802 803 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 804 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 805 0, all_hub, 806 inst); 807 808 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 809 flush_type, all_hub, 810 inst); 811 r = 0; 812 } else { 813 /* 2 dwords flush + 8 dwords fence */ 814 ndw = kiq->pmf->invalidate_tlbs_size + 8; 815 816 if (adev->gmc.flush_tlb_needs_extra_type_2) 817 ndw += kiq->pmf->invalidate_tlbs_size; 818 819 if (adev->gmc.flush_tlb_needs_extra_type_0) 820 ndw += kiq->pmf->invalidate_tlbs_size; 821 822 spin_lock(&adev->gfx.kiq[inst].ring_lock); 823 r = amdgpu_ring_alloc(ring, ndw); 824 if (r) { 825 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 826 goto error_unlock_reset; 827 } 828 if (adev->gmc.flush_tlb_needs_extra_type_2) 829 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); 830 831 if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) 832 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); 833 834 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); 835 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 836 if (r) { 837 amdgpu_ring_undo(ring); 838 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 839 goto error_unlock_reset; 840 } 841 842 amdgpu_ring_commit(ring); 843 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 844 845 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 846 847 might_sleep(); 848 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 849 !amdgpu_reset_pending(adev->reset_domain)) { 850 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 851 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 852 } 853 854 if (cnt > MAX_KIQ_REG_TRY) { 855 dev_err(adev->dev, "timeout waiting for kiq fence\n"); 856 r = -ETIME; 857 } else 858 r = 0; 859 } 860 861 error_unlock_reset: 862 up_read(&adev->reset_domain->sem); 863 return r; 864 } 865 866 void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, 867 uint32_t reg0, uint32_t reg1, 868 uint32_t ref, uint32_t mask, 869 uint32_t xcc_inst) 870 { 871 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_inst]; 872 struct amdgpu_ring *ring = &kiq->ring; 873 signed long r, cnt = 0; 874 unsigned long flags; 875 uint32_t seq; 876 877 if (adev->mes.ring[MES_PIPE_INST(xcc_inst, 0)].sched.ready) { 878 amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1, 879 ref, mask, xcc_inst); 880 return; 881 } 882 883 spin_lock_irqsave(&kiq->ring_lock, flags); 884 amdgpu_ring_alloc(ring, 32); 885 amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 886 ref, mask); 887 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 888 if (r) 889 goto failed_undo; 890 891 amdgpu_ring_commit(ring); 892 spin_unlock_irqrestore(&kiq->ring_lock, flags); 893 894 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 895 896 /* don't wait anymore for IRQ context */ 897 if (r < 1 && in_interrupt()) 898 goto failed_kiq; 899 900 might_sleep(); 901 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 902 !amdgpu_reset_pending(adev->reset_domain)) { 903 904 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 905 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 906 } 907 908 if (cnt > MAX_KIQ_REG_TRY) 909 goto failed_kiq; 910 911 return; 912 913 failed_undo: 914 amdgpu_ring_undo(ring); 915 spin_unlock_irqrestore(&kiq->ring_lock, flags); 916 failed_kiq: 917 dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1); 918 } 919 920 /** 921 * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ 922 * @adev: amdgpu_device pointer 923 * 924 * Check and set if an the device @adev supports Trusted Memory 925 * Zones (TMZ). 926 */ 927 void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) 928 { 929 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 930 /* RAVEN */ 931 case IP_VERSION(9, 2, 2): 932 case IP_VERSION(9, 1, 0): 933 /* RENOIR looks like RAVEN */ 934 case IP_VERSION(9, 3, 0): 935 /* GC 10.3.7 */ 936 case IP_VERSION(10, 3, 7): 937 /* GC 11.0.1 */ 938 case IP_VERSION(11, 0, 1): 939 if (amdgpu_tmz == 0) { 940 adev->gmc.tmz_enabled = false; 941 dev_info(adev->dev, 942 "Trusted Memory Zone (TMZ) feature disabled (cmd line)\n"); 943 } else { 944 adev->gmc.tmz_enabled = true; 945 dev_info(adev->dev, 946 "Trusted Memory Zone (TMZ) feature enabled\n"); 947 } 948 break; 949 case IP_VERSION(10, 1, 10): 950 case IP_VERSION(10, 1, 1): 951 case IP_VERSION(10, 1, 2): 952 case IP_VERSION(10, 1, 3): 953 case IP_VERSION(10, 3, 0): 954 case IP_VERSION(10, 3, 2): 955 case IP_VERSION(10, 3, 4): 956 case IP_VERSION(10, 3, 5): 957 case IP_VERSION(10, 3, 6): 958 /* VANGOGH */ 959 case IP_VERSION(10, 3, 1): 960 /* YELLOW_CARP*/ 961 case IP_VERSION(10, 3, 3): 962 case IP_VERSION(11, 0, 4): 963 case IP_VERSION(11, 5, 0): 964 case IP_VERSION(11, 5, 1): 965 case IP_VERSION(11, 5, 2): 966 case IP_VERSION(11, 5, 3): 967 case IP_VERSION(11, 5, 4): 968 /* Don't enable it by default yet. 969 */ 970 if (amdgpu_tmz < 1) { 971 adev->gmc.tmz_enabled = false; 972 dev_info(adev->dev, 973 "Trusted Memory Zone (TMZ) feature disabled as experimental (default)\n"); 974 } else { 975 adev->gmc.tmz_enabled = true; 976 dev_info(adev->dev, 977 "Trusted Memory Zone (TMZ) feature enabled as experimental (cmd line)\n"); 978 } 979 break; 980 default: 981 adev->gmc.tmz_enabled = false; 982 dev_info(adev->dev, 983 "Trusted Memory Zone (TMZ) feature not supported\n"); 984 break; 985 } 986 } 987 988 /** 989 * amdgpu_gmc_noretry_set -- set per asic noretry defaults 990 * @adev: amdgpu_device pointer 991 * 992 * Set a per asic default for the no-retry parameter. 993 * 994 */ 995 void amdgpu_gmc_noretry_set(struct amdgpu_device *adev) 996 { 997 struct amdgpu_gmc *gmc = &adev->gmc; 998 uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); 999 bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) || 1000 gc_ver == IP_VERSION(9, 4, 0) || 1001 gc_ver == IP_VERSION(9, 4, 1) || 1002 gc_ver == IP_VERSION(9, 4, 2) || 1003 gc_ver == IP_VERSION(9, 4, 3) || 1004 gc_ver == IP_VERSION(9, 4, 4) || 1005 gc_ver == IP_VERSION(9, 5, 0) || 1006 gc_ver >= IP_VERSION(10, 3, 0)); 1007 1008 if (!amdgpu_sriov_xnack_support(adev)) 1009 gmc->noretry = 1; 1010 else 1011 gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry; 1012 } 1013 1014 void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, 1015 bool enable) 1016 { 1017 struct amdgpu_vmhub *hub; 1018 u32 tmp, reg, i; 1019 1020 hub = &adev->vmhub[hub_type]; 1021 for (i = 0; i < 16; i++) { 1022 reg = hub->vm_context0_cntl + hub->ctx_distance * i; 1023 1024 tmp = (hub_type == AMDGPU_GFXHUB(0)) ? 1025 RREG32_SOC15_IP(GC, reg) : 1026 RREG32_SOC15_IP(MMHUB, reg); 1027 1028 if (enable) 1029 tmp |= hub->vm_cntx_cntl_vm_fault; 1030 else 1031 tmp &= ~hub->vm_cntx_cntl_vm_fault; 1032 1033 (hub_type == AMDGPU_GFXHUB(0)) ? 1034 WREG32_SOC15_IP(GC, reg, tmp) : 1035 WREG32_SOC15_IP(MMHUB, reg, tmp); 1036 } 1037 } 1038 1039 void amdgpu_gmc_init_vga_resv_regions(struct amdgpu_device *adev) 1040 { 1041 unsigned size; 1042 1043 if (adev->gmc.is_app_apu) 1044 return; 1045 1046 /* 1047 * Some ASICs need to reserve a region of video memory to avoid access 1048 * from driver 1049 */ 1050 /* 1051 * TODO: 1052 * Currently there is a bug where some memory client outside 1053 * of the driver writes to first 8M of VRAM on S3 resume, 1054 * this overrides GART which by default gets placed in first 8M and 1055 * causes VM_FAULTS once GTT is accessed. 1056 * Keep the stolen memory reservation until the while this is not solved. 1057 */ 1058 switch (adev->asic_type) { 1059 case CHIP_VEGA10: 1060 adev->mman.keep_stolen_vga_memory = true; 1061 /* 1062 * VEGA10 SRIOV VF with MS_HYPERV host needs some firmware reserved area. 1063 */ 1064 #ifdef CONFIG_X86 1065 if (amdgpu_sriov_vf(adev) && hypervisor_is_type(X86_HYPER_MS_HYPERV)) { 1066 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_RESERVED, 1067 0x500000, 0x200000, false); 1068 } 1069 #endif 1070 break; 1071 case CHIP_RAVEN: 1072 case CHIP_RENOIR: 1073 adev->mman.keep_stolen_vga_memory = true; 1074 break; 1075 case CHIP_POLARIS10: 1076 case CHIP_POLARIS11: 1077 case CHIP_POLARIS12: 1078 /* MacBookPros with switchable graphics put VRAM at 0 when 1079 * the iGPU is enabled which results in cursor issues if 1080 * the cursor ends up at 0. Reserve vram at 0 in that case. 1081 */ 1082 if (adev->gmc.vram_start == 0) 1083 adev->mman.keep_stolen_vga_memory = true; 1084 break; 1085 default: 1086 adev->mman.keep_stolen_vga_memory = false; 1087 break; 1088 } 1089 1090 if (amdgpu_sriov_vf(adev) || 1091 !amdgpu_device_has_display_hardware(adev)) { 1092 size = 0; 1093 } else { 1094 size = amdgpu_gmc_get_vbios_fb_size(adev); 1095 1096 if (adev->mman.keep_stolen_vga_memory) 1097 size = max(size, (unsigned)AMDGPU_VBIOS_VGA_ALLOCATION); 1098 } 1099 1100 /* set to 0 if the pre-OS buffer uses up most of vram */ 1101 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 1102 size = 0; 1103 1104 if (size > AMDGPU_VBIOS_VGA_ALLOCATION) { 1105 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1106 0, AMDGPU_VBIOS_VGA_ALLOCATION, false); 1107 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_EXTENDED, 1108 AMDGPU_VBIOS_VGA_ALLOCATION, 1109 size - AMDGPU_VBIOS_VGA_ALLOCATION, false); 1110 } else { 1111 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1112 0, size, false); 1113 } 1114 } 1115 1116 /** 1117 * amdgpu_gmc_init_pdb0 - initialize PDB0 1118 * 1119 * @adev: amdgpu_device pointer 1120 * 1121 * This function is only used when GART page table is used 1122 * for FB address translatioin. In such a case, we construct 1123 * a 2-level system VM page table: PDB0->PTB, to cover both 1124 * VRAM of the hive and system memory. 1125 * 1126 * PDB0 is static, initialized once on driver initialization. 1127 * The first n entries of PDB0 are used as PTE by setting 1128 * P bit to 1, pointing to VRAM. The n+1'th entry points 1129 * to a big PTB covering system memory. 1130 * 1131 */ 1132 void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev) 1133 { 1134 int i; 1135 uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW? 1136 /* Each PDE0 (used as PTE) covers (2^vmid0_page_table_block_size)*2M 1137 */ 1138 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 1139 u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21; 1140 u64 vram_addr, vram_end; 1141 u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo); 1142 int idx; 1143 1144 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 1145 return; 1146 1147 flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE; 1148 flags |= AMDGPU_PTE_WRITEABLE; 1149 flags |= AMDGPU_PTE_SNOOPED; 1150 flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1)); 1151 flags |= AMDGPU_PDE_PTE_FLAG(adev); 1152 1153 vram_addr = adev->vm_manager.vram_base_offset; 1154 if (!amdgpu_virt_xgmi_migrate_enabled(adev)) 1155 vram_addr -= adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1156 vram_end = vram_addr + vram_size; 1157 1158 /* The first n PDE0 entries are used as PTE, 1159 * pointing to vram 1160 */ 1161 for (i = 0; vram_addr < vram_end; i++, vram_addr += pde0_page_size) 1162 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, vram_addr, flags); 1163 1164 /* The n+1'th PDE0 entry points to a huge 1165 * PTB who has more than 512 entries each 1166 * pointing to a 4K system page 1167 */ 1168 flags = AMDGPU_PTE_VALID; 1169 flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0); 1170 /* Requires gart_ptb_gpu_pa to be 4K aligned */ 1171 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags); 1172 drm_dev_exit(idx); 1173 } 1174 1175 /** 1176 * amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC 1177 * address 1178 * 1179 * @adev: amdgpu_device pointer 1180 * @mc_addr: MC address of buffer 1181 */ 1182 uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr) 1183 { 1184 return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset; 1185 } 1186 1187 /** 1188 * amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from 1189 * GPU's view 1190 * 1191 * @adev: amdgpu_device pointer 1192 * @bo: amdgpu buffer object 1193 */ 1194 uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo) 1195 { 1196 return amdgpu_gmc_vram_mc2pa(adev, amdgpu_bo_gpu_offset(bo)); 1197 } 1198 1199 int amdgpu_gmc_vram_checking(struct amdgpu_device *adev) 1200 { 1201 struct amdgpu_bo *vram_bo = NULL; 1202 uint64_t vram_gpu = 0; 1203 void *vram_ptr = NULL; 1204 1205 int ret, size = 0x100000; 1206 uint8_t cptr[10]; 1207 1208 ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, 1209 AMDGPU_GEM_DOMAIN_VRAM, 1210 &vram_bo, 1211 &vram_gpu, 1212 &vram_ptr); 1213 if (ret) 1214 return ret; 1215 1216 memset(vram_ptr, 0x86, size); 1217 memset(cptr, 0x86, 10); 1218 1219 /** 1220 * Check the start, the mid, and the end of the memory if the content of 1221 * each byte is the pattern "0x86". If yes, we suppose the vram bo is 1222 * workable. 1223 * 1224 * Note: If check the each byte of whole 1M bo, it will cost too many 1225 * seconds, so here, we just pick up three parts for emulation. 1226 */ 1227 ret = memcmp(vram_ptr, cptr, 10); 1228 if (ret) { 1229 ret = -EIO; 1230 goto release_buffer; 1231 } 1232 1233 ret = memcmp(vram_ptr + (size / 2), cptr, 10); 1234 if (ret) { 1235 ret = -EIO; 1236 goto release_buffer; 1237 } 1238 1239 ret = memcmp(vram_ptr + size - 10, cptr, 10); 1240 if (ret) { 1241 ret = -EIO; 1242 goto release_buffer; 1243 } 1244 1245 release_buffer: 1246 amdgpu_bo_free_kernel(&vram_bo, &vram_gpu, 1247 &vram_ptr); 1248 1249 return ret; 1250 } 1251 1252 static const char *nps_desc[] = { 1253 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 1254 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 1255 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 1256 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 1257 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 1258 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 1259 }; 1260 1261 static ssize_t available_memory_partition_show(struct device *dev, 1262 struct device_attribute *addr, 1263 char *buf) 1264 { 1265 struct drm_device *ddev = dev_get_drvdata(dev); 1266 struct amdgpu_device *adev = drm_to_adev(ddev); 1267 int size = 0, mode; 1268 char *sep = ""; 1269 1270 for_each_inst(mode, adev->gmc.supported_nps_modes) { 1271 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 1272 sep = ", "; 1273 } 1274 size += sysfs_emit_at(buf, size, "\n"); 1275 1276 return size; 1277 } 1278 1279 static ssize_t current_memory_partition_store(struct device *dev, 1280 struct device_attribute *attr, 1281 const char *buf, size_t count) 1282 { 1283 struct drm_device *ddev = dev_get_drvdata(dev); 1284 struct amdgpu_device *adev = drm_to_adev(ddev); 1285 enum amdgpu_memory_partition mode; 1286 struct amdgpu_hive_info *hive; 1287 int i; 1288 1289 mode = UNKNOWN_MEMORY_PARTITION_MODE; 1290 for_each_inst(i, adev->gmc.supported_nps_modes) { 1291 if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) { 1292 mode = i; 1293 break; 1294 } 1295 } 1296 1297 if (mode == UNKNOWN_MEMORY_PARTITION_MODE) 1298 return -EINVAL; 1299 1300 if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) { 1301 dev_info( 1302 adev->dev, 1303 "requested NPS mode is same as current NPS mode, skipping\n"); 1304 return count; 1305 } 1306 1307 /* If device is part of hive, all devices in the hive should request the 1308 * same mode. Hence store the requested mode in hive. 1309 */ 1310 hive = amdgpu_get_xgmi_hive(adev); 1311 if (hive) { 1312 atomic_set(&hive->requested_nps_mode, mode); 1313 amdgpu_put_xgmi_hive(hive); 1314 } else { 1315 adev->gmc.requested_nps_mode = mode; 1316 } 1317 1318 dev_info( 1319 adev->dev, 1320 "NPS mode change requested, please remove and reload the driver\n"); 1321 1322 return count; 1323 } 1324 1325 static ssize_t current_memory_partition_show( 1326 struct device *dev, struct device_attribute *addr, char *buf) 1327 { 1328 struct drm_device *ddev = dev_get_drvdata(dev); 1329 struct amdgpu_device *adev = drm_to_adev(ddev); 1330 enum amdgpu_memory_partition mode; 1331 1332 /* Only minimal precaution taken to reject requests while in reset */ 1333 if (amdgpu_in_reset(adev)) 1334 return -EPERM; 1335 1336 mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1337 if ((mode >= ARRAY_SIZE(nps_desc)) || 1338 (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode)) 1339 return sysfs_emit(buf, "UNKNOWN\n"); 1340 1341 return sysfs_emit(buf, "%s\n", nps_desc[mode]); 1342 } 1343 1344 static DEVICE_ATTR_RW(current_memory_partition); 1345 static DEVICE_ATTR_RO(available_memory_partition); 1346 1347 int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev) 1348 { 1349 bool nps_switch_support; 1350 int r = 0; 1351 1352 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1353 return 0; 1354 1355 nps_switch_support = (hweight32(adev->gmc.supported_nps_modes & 1356 AMDGPU_ALL_NPS_MASK) > 1); 1357 if (!nps_switch_support) 1358 dev_attr_current_memory_partition.attr.mode &= 1359 ~(S_IWUSR | S_IWGRP | S_IWOTH); 1360 else 1361 r = device_create_file(adev->dev, 1362 &dev_attr_available_memory_partition); 1363 1364 if (r) 1365 return r; 1366 1367 return device_create_file(adev->dev, 1368 &dev_attr_current_memory_partition); 1369 } 1370 1371 void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev) 1372 { 1373 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1374 return; 1375 1376 device_remove_file(adev->dev, &dev_attr_current_memory_partition); 1377 device_remove_file(adev->dev, &dev_attr_available_memory_partition); 1378 } 1379 1380 int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, 1381 struct amdgpu_mem_partition_info *mem_ranges, 1382 uint8_t *exp_ranges) 1383 { 1384 struct amdgpu_gmc_memrange ranges[AMDGPU_MAX_MEM_RANGES]; 1385 int range_cnt, ret, i, j; 1386 uint32_t nps_type; 1387 bool refresh; 1388 1389 if (!mem_ranges || !exp_ranges) 1390 return -EINVAL; 1391 range_cnt = AMDGPU_MAX_MEM_RANGES; 1392 refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) && 1393 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS); 1394 ret = amdgpu_discovery_get_nps_info(adev, &nps_type, ranges, &range_cnt, 1395 refresh); 1396 1397 if (ret) 1398 return ret; 1399 1400 /* TODO: For now, expect ranges and partition count to be the same. 1401 * Adjust if there are holes expected in any NPS domain. 1402 */ 1403 if (*exp_ranges && (range_cnt != *exp_ranges)) { 1404 dev_warn( 1405 adev->dev, 1406 "NPS config mismatch - expected ranges: %d discovery - nps mode: %d, nps ranges: %d", 1407 *exp_ranges, nps_type, range_cnt); 1408 ret = -EINVAL; 1409 goto err; 1410 } 1411 1412 for (i = 0; i < range_cnt; ++i) { 1413 if (ranges[i].base_address >= ranges[i].limit_address) { 1414 dev_warn( 1415 adev->dev, 1416 "Invalid NPS range - nps mode: %d, range[%d]: base: %llx limit: %llx", 1417 nps_type, i, ranges[i].base_address, 1418 ranges[i].limit_address); 1419 ret = -EINVAL; 1420 goto err; 1421 } 1422 1423 /* Check for overlaps, not expecting any now */ 1424 for (j = i - 1; j >= 0; j--) { 1425 if (max(ranges[j].base_address, 1426 ranges[i].base_address) <= 1427 min(ranges[j].limit_address, 1428 ranges[i].limit_address)) { 1429 dev_warn( 1430 adev->dev, 1431 "overlapping ranges detected [ %llx - %llx ] | [%llx - %llx]", 1432 ranges[j].base_address, 1433 ranges[j].limit_address, 1434 ranges[i].base_address, 1435 ranges[i].limit_address); 1436 ret = -EINVAL; 1437 goto err; 1438 } 1439 } 1440 1441 mem_ranges[i].range.fpfn = 1442 (ranges[i].base_address - 1443 adev->vm_manager.vram_base_offset) >> 1444 AMDGPU_GPU_PAGE_SHIFT; 1445 mem_ranges[i].range.lpfn = 1446 (ranges[i].limit_address - 1447 adev->vm_manager.vram_base_offset) >> 1448 AMDGPU_GPU_PAGE_SHIFT; 1449 mem_ranges[i].size = 1450 ranges[i].limit_address - ranges[i].base_address + 1; 1451 } 1452 1453 if (!*exp_ranges) 1454 *exp_ranges = range_cnt; 1455 err: 1456 return ret; 1457 } 1458 1459 int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev, 1460 int nps_mode) 1461 { 1462 /* Not supported on VF devices and APUs */ 1463 if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU)) 1464 return -EOPNOTSUPP; 1465 1466 if (!adev->psp.funcs) { 1467 dev_err(adev->dev, 1468 "PSP interface not available for nps mode change request"); 1469 return -EINVAL; 1470 } 1471 1472 return psp_memory_partition(&adev->psp, nps_mode); 1473 } 1474 1475 static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev, 1476 int req_nps_mode, 1477 int cur_nps_mode) 1478 { 1479 return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) == 1480 BIT(req_nps_mode)) && 1481 req_nps_mode != cur_nps_mode); 1482 } 1483 1484 void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev) 1485 { 1486 int req_nps_mode, cur_nps_mode, r; 1487 struct amdgpu_hive_info *hive; 1488 1489 if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes || 1490 !adev->gmc.gmc_funcs->request_mem_partition_mode) 1491 return; 1492 1493 cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1494 hive = amdgpu_get_xgmi_hive(adev); 1495 if (hive) { 1496 req_nps_mode = atomic_read(&hive->requested_nps_mode); 1497 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, 1498 cur_nps_mode)) { 1499 amdgpu_put_xgmi_hive(hive); 1500 return; 1501 } 1502 r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode); 1503 amdgpu_put_xgmi_hive(hive); 1504 goto out; 1505 } 1506 1507 req_nps_mode = adev->gmc.requested_nps_mode; 1508 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode)) 1509 return; 1510 1511 /* even if this fails, we should let driver unload w/o blocking */ 1512 r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode); 1513 out: 1514 if (r) 1515 dev_err(adev->dev, "NPS mode change request failed\n"); 1516 else 1517 dev_info( 1518 adev->dev, 1519 "NPS mode change request done, reload driver to complete the change\n"); 1520 } 1521 1522 bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) 1523 { 1524 if (adev->gmc.gmc_funcs->need_reset_on_init) 1525 return adev->gmc.gmc_funcs->need_reset_on_init(adev); 1526 1527 return false; 1528 } 1529 1530 enum amdgpu_memory_partition 1531 amdgpu_gmc_get_vf_memory_partition(struct amdgpu_device *adev) 1532 { 1533 switch (adev->gmc.num_mem_partitions) { 1534 case 0: 1535 return UNKNOWN_MEMORY_PARTITION_MODE; 1536 case 1: 1537 return AMDGPU_NPS1_PARTITION_MODE; 1538 case 2: 1539 return AMDGPU_NPS2_PARTITION_MODE; 1540 case 4: 1541 return AMDGPU_NPS4_PARTITION_MODE; 1542 case 8: 1543 return AMDGPU_NPS8_PARTITION_MODE; 1544 default: 1545 return AMDGPU_NPS1_PARTITION_MODE; 1546 } 1547 } 1548 1549 enum amdgpu_memory_partition 1550 amdgpu_gmc_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes) 1551 { 1552 enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE; 1553 1554 if (adev->nbio.funcs && 1555 adev->nbio.funcs->get_memory_partition_mode) 1556 mode = adev->nbio.funcs->get_memory_partition_mode(adev, 1557 supp_modes); 1558 else 1559 dev_warn(adev->dev, "memory partition mode query is not supported\n"); 1560 1561 return mode; 1562 } 1563 1564 enum amdgpu_memory_partition 1565 amdgpu_gmc_query_memory_partition(struct amdgpu_device *adev) 1566 { 1567 if (amdgpu_sriov_vf(adev)) 1568 return amdgpu_gmc_get_vf_memory_partition(adev); 1569 else 1570 return amdgpu_gmc_get_memory_partition(adev, NULL); 1571 } 1572 1573 static bool amdgpu_gmc_validate_partition_info(struct amdgpu_device *adev) 1574 { 1575 enum amdgpu_memory_partition mode; 1576 u32 supp_modes; 1577 bool valid; 1578 1579 mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes); 1580 1581 /* Mode detected by hardware not present in supported modes */ 1582 if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && 1583 !(BIT(mode - 1) & supp_modes)) 1584 return false; 1585 1586 switch (mode) { 1587 case UNKNOWN_MEMORY_PARTITION_MODE: 1588 case AMDGPU_NPS1_PARTITION_MODE: 1589 valid = (adev->gmc.num_mem_partitions == 1); 1590 break; 1591 case AMDGPU_NPS2_PARTITION_MODE: 1592 valid = (adev->gmc.num_mem_partitions == 2); 1593 break; 1594 case AMDGPU_NPS4_PARTITION_MODE: 1595 valid = (adev->gmc.num_mem_partitions == 3 || 1596 adev->gmc.num_mem_partitions == 4); 1597 break; 1598 case AMDGPU_NPS8_PARTITION_MODE: 1599 valid = (adev->gmc.num_mem_partitions == 8); 1600 break; 1601 default: 1602 valid = false; 1603 } 1604 1605 return valid; 1606 } 1607 1608 static bool amdgpu_gmc_is_node_present(int *node_ids, int num_ids, int nid) 1609 { 1610 int i; 1611 1612 /* Check if node with id 'nid' is present in 'node_ids' array */ 1613 for (i = 0; i < num_ids; ++i) 1614 if (node_ids[i] == nid) 1615 return true; 1616 1617 return false; 1618 } 1619 1620 static void 1621 amdgpu_gmc_init_acpi_mem_ranges(struct amdgpu_device *adev, 1622 struct amdgpu_mem_partition_info *mem_ranges) 1623 { 1624 struct amdgpu_numa_info numa_info; 1625 int node_ids[AMDGPU_MAX_MEM_RANGES]; 1626 int num_ranges = 0, ret; 1627 int num_xcc, xcc_id; 1628 uint32_t xcc_mask; 1629 1630 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1631 xcc_mask = (1U << num_xcc) - 1; 1632 1633 for_each_inst(xcc_id, xcc_mask) { 1634 ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info); 1635 if (ret) 1636 continue; 1637 1638 if (numa_info.nid == NUMA_NO_NODE) { 1639 mem_ranges[0].size = numa_info.size; 1640 mem_ranges[0].numa.node = numa_info.nid; 1641 num_ranges = 1; 1642 break; 1643 } 1644 1645 if (amdgpu_gmc_is_node_present(node_ids, num_ranges, 1646 numa_info.nid)) 1647 continue; 1648 1649 node_ids[num_ranges] = numa_info.nid; 1650 mem_ranges[num_ranges].numa.node = numa_info.nid; 1651 mem_ranges[num_ranges].size = numa_info.size; 1652 ++num_ranges; 1653 } 1654 1655 adev->gmc.num_mem_partitions = num_ranges; 1656 } 1657 1658 void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev, 1659 struct amdgpu_mem_partition_info *mem_ranges) 1660 { 1661 enum amdgpu_memory_partition mode; 1662 u32 start_addr = 0, size; 1663 int i, r, l; 1664 1665 mode = amdgpu_gmc_query_memory_partition(adev); 1666 1667 switch (mode) { 1668 case UNKNOWN_MEMORY_PARTITION_MODE: 1669 adev->gmc.num_mem_partitions = 0; 1670 break; 1671 case AMDGPU_NPS1_PARTITION_MODE: 1672 adev->gmc.num_mem_partitions = 1; 1673 break; 1674 case AMDGPU_NPS2_PARTITION_MODE: 1675 adev->gmc.num_mem_partitions = 2; 1676 break; 1677 case AMDGPU_NPS4_PARTITION_MODE: 1678 if (adev->flags & AMD_IS_APU) 1679 adev->gmc.num_mem_partitions = 3; 1680 else 1681 adev->gmc.num_mem_partitions = 4; 1682 break; 1683 case AMDGPU_NPS8_PARTITION_MODE: 1684 adev->gmc.num_mem_partitions = 8; 1685 break; 1686 default: 1687 adev->gmc.num_mem_partitions = 1; 1688 break; 1689 } 1690 1691 /* Use NPS range info, if populated */ 1692 r = amdgpu_gmc_get_nps_memranges(adev, mem_ranges, 1693 &adev->gmc.num_mem_partitions); 1694 if (!r) { 1695 l = 0; 1696 for (i = 1; i < adev->gmc.num_mem_partitions; ++i) { 1697 if (mem_ranges[i].range.lpfn > 1698 mem_ranges[i - 1].range.lpfn) 1699 l = i; 1700 } 1701 1702 } else { 1703 if (!adev->gmc.num_mem_partitions) { 1704 dev_warn(adev->dev, 1705 "Not able to detect NPS mode, fall back to NPS1\n"); 1706 adev->gmc.num_mem_partitions = 1; 1707 } 1708 /* Fallback to sw based calculation */ 1709 size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT; 1710 size /= adev->gmc.num_mem_partitions; 1711 1712 for (i = 0; i < adev->gmc.num_mem_partitions; ++i) { 1713 mem_ranges[i].range.fpfn = start_addr; 1714 mem_ranges[i].size = 1715 ((u64)size << AMDGPU_GPU_PAGE_SHIFT); 1716 mem_ranges[i].range.lpfn = start_addr + size - 1; 1717 start_addr += size; 1718 } 1719 1720 l = adev->gmc.num_mem_partitions - 1; 1721 } 1722 1723 /* Adjust the last one */ 1724 mem_ranges[l].range.lpfn = 1725 (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1; 1726 mem_ranges[l].size = 1727 adev->gmc.real_vram_size - 1728 ((u64)mem_ranges[l].range.fpfn << AMDGPU_GPU_PAGE_SHIFT); 1729 } 1730 1731 int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev) 1732 { 1733 bool valid; 1734 1735 adev->gmc.mem_partitions = kzalloc_objs(struct amdgpu_mem_partition_info, 1736 AMDGPU_MAX_MEM_RANGES); 1737 if (!adev->gmc.mem_partitions) 1738 return -ENOMEM; 1739 1740 if (adev->gmc.is_app_apu) 1741 amdgpu_gmc_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions); 1742 else 1743 amdgpu_gmc_init_sw_mem_ranges(adev, adev->gmc.mem_partitions); 1744 1745 if (amdgpu_sriov_vf(adev)) 1746 valid = true; 1747 else 1748 valid = amdgpu_gmc_validate_partition_info(adev); 1749 if (!valid) { 1750 /* TODO: handle invalid case */ 1751 dev_warn(adev->dev, 1752 "Mem ranges not matching with hardware config\n"); 1753 } 1754 1755 return 0; 1756 } 1757 1758 int amdgpu_gmc_get_vram_info(struct amdgpu_device *adev, 1759 int *vram_width, int *vram_type, int *vram_vendor) 1760 { 1761 int ret = 0; 1762 1763 if (adev->flags & AMD_IS_APU) 1764 return amdgpu_atomfirmware_get_integrated_system_info(adev, 1765 vram_width, vram_type, vram_vendor); 1766 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1767 case IP_VERSION(12, 0, 0): 1768 case IP_VERSION(12, 0, 1): 1769 return amdgpu_atomfirmware_get_umc_info(adev, 1770 vram_width, vram_type, vram_vendor); 1771 case IP_VERSION(9, 5, 0): 1772 case IP_VERSION(9, 4, 4): 1773 case IP_VERSION(9, 4, 3): 1774 ret = amdgpu_atomfirmware_get_umc_info(adev, 1775 vram_width, vram_type, vram_vendor); 1776 if (vram_width && !ret) 1777 *vram_width *= hweight32(adev->aid_mask); 1778 return ret; 1779 default: 1780 return amdgpu_atomfirmware_get_vram_info(adev, 1781 vram_width, vram_type, vram_vendor); 1782 } 1783 return 0; 1784 } 1785