1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 27 #include <linux/io-64-nonatomic-lo-hi.h> 28 #ifdef CONFIG_X86 29 #include <asm/hypervisor.h> 30 #endif 31 32 #include "amdgpu.h" 33 #include "amdgpu_gmc.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_reset.h" 36 #include "amdgpu_ring.h" 37 #include "amdgpu_sdma.h" 38 #include "amdgpu_xgmi.h" 39 #include "amdgpu_atomfirmware.h" 40 41 #include <drm/drm_drv.h> 42 #include <drm/ttm/ttm_tt.h> 43 44 static const u64 four_gb = 0x100000000ULL; 45 46 bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev) 47 { 48 return adev->gmc.xgmi.connected_to_cpu || amdgpu_virt_xgmi_migrate_enabled(adev); 49 } 50 51 /** 52 * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0 53 * 54 * @adev: amdgpu_device pointer 55 * 56 * Allocate video memory for pdb0 and map it for CPU access 57 * Returns 0 for success, error for failure. 58 */ 59 int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev) 60 { 61 int r; 62 struct amdgpu_bo_param bp; 63 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 64 uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21; 65 uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >> pde0_page_shift; 66 67 memset(&bp, 0, sizeof(bp)); 68 bp.size = PAGE_ALIGN((npdes + 1) * 8); 69 bp.byte_align = PAGE_SIZE; 70 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 71 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 72 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 73 bp.type = ttm_bo_type_kernel; 74 bp.resv = NULL; 75 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 76 77 r = amdgpu_bo_create(adev, &bp, &adev->gmc.pdb0_bo); 78 if (r) 79 return r; 80 81 r = amdgpu_bo_reserve(adev->gmc.pdb0_bo, false); 82 if (unlikely(r != 0)) 83 goto bo_reserve_failure; 84 85 r = amdgpu_bo_pin(adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM); 86 if (r) 87 goto bo_pin_failure; 88 r = amdgpu_bo_kmap(adev->gmc.pdb0_bo, &adev->gmc.ptr_pdb0); 89 if (r) 90 goto bo_kmap_failure; 91 92 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 93 return 0; 94 95 bo_kmap_failure: 96 amdgpu_bo_unpin(adev->gmc.pdb0_bo); 97 bo_pin_failure: 98 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 99 bo_reserve_failure: 100 amdgpu_bo_unref(&adev->gmc.pdb0_bo); 101 return r; 102 } 103 104 /** 105 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO 106 * 107 * @bo: the BO to get the PDE for 108 * @level: the level in the PD hirarchy 109 * @addr: resulting addr 110 * @flags: resulting flags 111 * 112 * Get the address and flags to be used for a PDE (Page Directory Entry). 113 */ 114 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, 115 uint64_t *addr, uint64_t *flags) 116 { 117 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 118 119 switch (bo->tbo.resource->mem_type) { 120 case TTM_PL_TT: 121 *addr = bo->tbo.ttm->dma_address[0]; 122 break; 123 case TTM_PL_VRAM: 124 *addr = amdgpu_bo_gpu_offset(bo); 125 /* For SPM-backed VRAM, convert the MC address to a physical 126 * address for system memory path access. 127 */ 128 if (amdgpu_virt_vram_is_spm(adev)) 129 *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); 130 break; 131 default: 132 *addr = 0; 133 break; 134 } 135 *flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, bo->tbo.resource); 136 amdgpu_gmc_get_vm_pde(adev, level, addr, flags); 137 } 138 139 /* 140 * amdgpu_gmc_pd_addr - return the address of the root directory 141 */ 142 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo) 143 { 144 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 145 uint64_t pd_addr; 146 147 /* TODO: move that into ASIC specific code */ 148 if (adev->asic_type >= CHIP_VEGA10) { 149 uint64_t flags = AMDGPU_PTE_VALID; 150 151 amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags); 152 pd_addr |= flags; 153 } else { 154 pd_addr = amdgpu_bo_gpu_offset(bo); 155 } 156 return pd_addr; 157 } 158 159 /** 160 * amdgpu_gmc_set_pte_pde - update the page tables using CPU 161 * 162 * @adev: amdgpu_device pointer 163 * @cpu_pt_addr: cpu address of the page table 164 * @gpu_page_idx: entry in the page table to update 165 * @addr: dst addr to write into pte/pde 166 * @flags: access flags 167 * 168 * Update the page tables using CPU. 169 */ 170 int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 171 uint32_t gpu_page_idx, uint64_t addr, 172 uint64_t flags) 173 { 174 void __iomem *ptr = (void *)cpu_pt_addr; 175 uint64_t value; 176 177 /* 178 * The following is for PTE only. GART does not have PDEs. 179 */ 180 value = addr & adev->gmc.pte_addr_mask; 181 value |= flags; 182 writeq(value, ptr + (gpu_page_idx * 8)); 183 184 return 0; 185 } 186 187 /** 188 * amdgpu_gmc_agp_addr - return the address in the AGP address space 189 * 190 * @bo: TTM BO which needs the address, must be in GTT domain 191 * 192 * Tries to figure out how to access the BO through the AGP aperture. Returns 193 * AMDGPU_BO_INVALID_OFFSET if that is not possible. 194 */ 195 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) 196 { 197 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 198 199 if (!bo->ttm) 200 return AMDGPU_BO_INVALID_OFFSET; 201 202 if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached) 203 return AMDGPU_BO_INVALID_OFFSET; 204 205 if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) 206 return AMDGPU_BO_INVALID_OFFSET; 207 208 return adev->gmc.agp_start + bo->ttm->dma_address[0]; 209 } 210 211 /** 212 * amdgpu_gmc_vram_location - try to find VRAM location 213 * 214 * @adev: amdgpu device structure holding all necessary information 215 * @mc: memory controller structure holding memory information 216 * @base: base address at which to put VRAM 217 * 218 * Function will try to place VRAM at base address provided 219 * as parameter. 220 */ 221 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 222 u64 base) 223 { 224 uint64_t vis_limit = (uint64_t)amdgpu_vis_vram_limit << 20; 225 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; 226 227 mc->vram_start = base; 228 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 229 if (limit < mc->real_vram_size) 230 mc->real_vram_size = limit; 231 232 if (vis_limit && vis_limit < mc->visible_vram_size) 233 mc->visible_vram_size = vis_limit; 234 235 if (mc->real_vram_size < mc->visible_vram_size) 236 mc->visible_vram_size = mc->real_vram_size; 237 238 if (mc->xgmi.num_physical_nodes == 0) { 239 mc->fb_start = mc->vram_start; 240 mc->fb_end = mc->vram_end; 241 } 242 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 243 mc->mc_vram_size >> 20, mc->vram_start, 244 mc->vram_end, mc->real_vram_size >> 20); 245 } 246 247 /** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture 248 * 249 * @adev: amdgpu device structure holding all necessary information 250 * @mc: memory controller structure holding memory information 251 * 252 * This function is only used if use GART for FB translation. In such 253 * case, we use sysvm aperture (vmid0 page tables) for both vram 254 * and gart (aka system memory) access. 255 * 256 * GPUVM (and our organization of vmid0 page tables) require sysvm 257 * aperture to be placed at a location aligned with 8 times of native 258 * page size. For example, if vm_context0_cntl.page_table_block_size 259 * is 12, then native page size is 8G (2M*2^12), sysvm should start 260 * with a 64G aligned address. For simplicity, we just put sysvm at 261 * address 0. So vram start at address 0 and gart is right after vram. 262 */ 263 void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 264 { 265 u64 hive_vram_start = 0; 266 u64 hive_vram_end = mc->xgmi.node_segment_size * mc->xgmi.num_physical_nodes - 1; 267 mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id; 268 mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1; 269 /* node_segment_size may not 4GB aligned on SRIOV, align up is needed. */ 270 mc->gart_start = ALIGN(hive_vram_end + 1, four_gb); 271 mc->gart_end = mc->gart_start + mc->gart_size - 1; 272 if (amdgpu_virt_xgmi_migrate_enabled(adev)) { 273 /* set mc->vram_start to 0 to switch the returned GPU address of 274 * amdgpu_bo_create_reserved() from FB aperture to GART aperture. 275 */ 276 mc->vram_start = 0; 277 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 278 mc->visible_vram_size = min(mc->visible_vram_size, mc->real_vram_size); 279 } else { 280 mc->fb_start = hive_vram_start; 281 mc->fb_end = hive_vram_end; 282 } 283 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 284 mc->mc_vram_size >> 20, mc->vram_start, 285 mc->vram_end, mc->real_vram_size >> 20); 286 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 287 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 288 } 289 290 void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size) 291 { 292 if (amdgpu_gart_size == -1) 293 adev->gmc.gart_size = 294 default_size + adev->pm.smu_prv_buffer_size; 295 else 296 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 297 } 298 299 /** 300 * amdgpu_gmc_gart_location - try to find GART location 301 * 302 * @adev: amdgpu device structure holding all necessary information 303 * @mc: memory controller structure holding memory information 304 * @gart_placement: GART placement policy with respect to VRAM 305 * 306 * Function will try to place GART before or after VRAM. 307 * If GART size is bigger than space left then we ajust GART size. 308 * Thus function will never fails. 309 */ 310 void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 311 enum amdgpu_gart_placement gart_placement) 312 { 313 u64 size_af, size_bf; 314 /*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/ 315 u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1); 316 317 /* VCE doesn't like it when BOs cross a 4GB segment, so align 318 * the GART base on a 4GB boundary as well. 319 */ 320 size_bf = mc->fb_start; 321 size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb); 322 323 if (mc->gart_size > max(size_bf, size_af)) { 324 dev_warn(adev->dev, "limiting GART\n"); 325 mc->gart_size = max(size_bf, size_af); 326 } 327 328 switch (gart_placement) { 329 case AMDGPU_GART_PLACEMENT_HIGH: 330 mc->gart_start = max_mc_address - mc->gart_size + 1; 331 break; 332 case AMDGPU_GART_PLACEMENT_LOW: 333 if (size_bf >= mc->gart_size) 334 mc->gart_start = 0; 335 else 336 mc->gart_start = ALIGN(mc->fb_end, four_gb); 337 break; 338 case AMDGPU_GART_PLACEMENT_BEST_FIT: 339 default: 340 if ((size_bf >= mc->gart_size && size_bf < size_af) || 341 (size_af < mc->gart_size)) 342 mc->gart_start = 0; 343 else 344 mc->gart_start = max_mc_address - mc->gart_size + 1; 345 break; 346 } 347 348 mc->gart_start &= ~(four_gb - 1); 349 mc->gart_end = mc->gart_start + mc->gart_size - 1; 350 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 351 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 352 } 353 354 /** 355 * amdgpu_gmc_agp_location - try to find AGP location 356 * @adev: amdgpu device structure holding all necessary information 357 * @mc: memory controller structure holding memory information 358 * 359 * Function will place try to find a place for the AGP BAR in the MC address 360 * space. 361 * 362 * AGP BAR will be assigned the largest available hole in the address space. 363 * Should be called after VRAM and GART locations are setup. 364 */ 365 void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 366 { 367 const uint64_t sixteen_gb = 1ULL << 34; 368 const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1); 369 u64 size_af, size_bf; 370 371 if (mc->fb_start > mc->gart_start) { 372 size_bf = (mc->fb_start & sixteen_gb_mask) - 373 ALIGN(mc->gart_end + 1, sixteen_gb); 374 size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb); 375 } else { 376 size_bf = mc->fb_start & sixteen_gb_mask; 377 size_af = (mc->gart_start & sixteen_gb_mask) - 378 ALIGN(mc->fb_end + 1, sixteen_gb); 379 } 380 381 if (size_bf > size_af) { 382 mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask; 383 mc->agp_size = size_bf; 384 } else { 385 mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb); 386 mc->agp_size = size_af; 387 } 388 389 mc->agp_end = mc->agp_start + mc->agp_size - 1; 390 dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n", 391 mc->agp_size >> 20, mc->agp_start, mc->agp_end); 392 } 393 394 /** 395 * amdgpu_gmc_set_agp_default - Set the default AGP aperture value. 396 * @adev: amdgpu device structure holding all necessary information 397 * @mc: memory controller structure holding memory information 398 * 399 * To disable the AGP aperture, you need to set the start to a larger 400 * value than the end. This function sets the default value which 401 * can then be overridden using amdgpu_gmc_agp_location() if you want 402 * to enable the AGP aperture on a specific chip. 403 * 404 */ 405 void amdgpu_gmc_set_agp_default(struct amdgpu_device *adev, 406 struct amdgpu_gmc *mc) 407 { 408 mc->agp_start = 0xffffffffffff; 409 mc->agp_end = 0; 410 mc->agp_size = 0; 411 } 412 413 /** 414 * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid 415 * 416 * @addr: 48 bit physical address, page aligned (36 significant bits) 417 * @pasid: 16 bit process address space identifier 418 */ 419 static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid) 420 { 421 return addr << 4 | pasid; 422 } 423 424 /** 425 * amdgpu_gmc_filter_faults - filter VM faults 426 * 427 * @adev: amdgpu device structure 428 * @ih: interrupt ring that the fault received from 429 * @addr: address of the VM fault 430 * @pasid: PASID of the process causing the fault 431 * @timestamp: timestamp of the fault 432 * 433 * Returns: 434 * True if the fault was filtered and should not be processed further. 435 * False if the fault is a new one and needs to be handled. 436 */ 437 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, 438 struct amdgpu_ih_ring *ih, uint64_t addr, 439 uint16_t pasid, uint64_t timestamp) 440 { 441 struct amdgpu_gmc *gmc = &adev->gmc; 442 uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid); 443 struct amdgpu_gmc_fault *fault; 444 uint32_t hash; 445 446 /* Stale retry fault if timestamp goes backward */ 447 if (amdgpu_ih_ts_after(timestamp, ih->processed_timestamp)) 448 return true; 449 450 /* If we don't have space left in the ring buffer return immediately */ 451 stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) - 452 AMDGPU_GMC_FAULT_TIMEOUT; 453 if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp) 454 return true; 455 456 /* Try to find the fault in the hash */ 457 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 458 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 459 while (fault->timestamp >= stamp) { 460 uint64_t tmp; 461 462 if (atomic64_read(&fault->key) == key) { 463 /* 464 * if we get a fault which is already present in 465 * the fault_ring and the timestamp of 466 * the fault is after the expired timestamp, 467 * then this is a new fault that needs to be added 468 * into the fault ring. 469 */ 470 if (fault->timestamp_expiry != 0 && 471 amdgpu_ih_ts_after(fault->timestamp_expiry, 472 timestamp)) 473 break; 474 else 475 return true; 476 } 477 478 tmp = fault->timestamp; 479 fault = &gmc->fault_ring[fault->next]; 480 481 /* Check if the entry was reused */ 482 if (fault->timestamp >= tmp) 483 break; 484 } 485 486 /* Add the fault to the ring */ 487 fault = &gmc->fault_ring[gmc->last_fault]; 488 atomic64_set(&fault->key, key); 489 fault->timestamp = timestamp; 490 491 /* And update the hash */ 492 fault->next = gmc->fault_hash[hash].idx; 493 gmc->fault_hash[hash].idx = gmc->last_fault++; 494 return false; 495 } 496 497 /** 498 * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter 499 * 500 * @adev: amdgpu device structure 501 * @addr: address of the VM fault 502 * @pasid: PASID of the process causing the fault 503 * 504 * Remove the address from fault filter, then future vm fault on this address 505 * will pass to retry fault handler to recover. 506 */ 507 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr, 508 uint16_t pasid) 509 { 510 struct amdgpu_gmc *gmc = &adev->gmc; 511 uint64_t key = amdgpu_gmc_fault_key(addr, pasid); 512 struct amdgpu_ih_ring *ih; 513 struct amdgpu_gmc_fault *fault; 514 uint32_t last_wptr; 515 uint64_t last_ts; 516 uint32_t hash; 517 uint64_t tmp; 518 519 if (adev->irq.retry_cam_enabled) 520 return; 521 else if (adev->irq.ih1.ring_size) 522 ih = &adev->irq.ih1; 523 else if (adev->irq.ih_soft.enabled) 524 ih = &adev->irq.ih_soft; 525 else 526 return; 527 528 /* Get the WPTR of the last entry in IH ring */ 529 last_wptr = amdgpu_ih_get_wptr(adev, ih); 530 /* Order wptr with ring data. */ 531 rmb(); 532 /* Get the timetamp of the last entry in IH ring */ 533 last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1); 534 535 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 536 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 537 do { 538 if (atomic64_read(&fault->key) == key) { 539 /* 540 * Update the timestamp when this fault 541 * expired. 542 */ 543 fault->timestamp_expiry = last_ts; 544 break; 545 } 546 547 tmp = fault->timestamp; 548 fault = &gmc->fault_ring[fault->next]; 549 } while (fault->timestamp < tmp); 550 } 551 552 int amdgpu_gmc_handle_retry_fault(struct amdgpu_device *adev, 553 struct amdgpu_iv_entry *entry, 554 u64 addr, 555 u32 cam_index, 556 u32 node_id, 557 bool write_fault) 558 { 559 int ret; 560 561 if (adev->irq.retry_cam_enabled) { 562 /* Delegate it to a different ring if the hardware hasn't 563 * already done it. 564 */ 565 if (entry->ih == &adev->irq.ih) { 566 amdgpu_irq_delegate(adev, entry, 8); 567 return 1; 568 } 569 570 ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 571 addr, entry->timestamp, write_fault); 572 WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); 573 if (ret) 574 return 1; 575 } else { 576 /* Process it only if it's the first fault for this address */ 577 if (entry->ih != &adev->irq.ih_soft && 578 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 579 entry->timestamp)) 580 return 1; 581 582 /* Delegate it to a different ring if the hardware hasn't 583 * already done it. 584 */ 585 if (entry->ih == &adev->irq.ih) { 586 amdgpu_irq_delegate(adev, entry, 8); 587 return 1; 588 } 589 590 /* Try to handle the recoverable page faults by filling page 591 * tables 592 */ 593 if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 594 addr, entry->timestamp, write_fault)) 595 return 1; 596 } 597 return 0; 598 } 599 600 int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev) 601 { 602 int r; 603 604 /* umc ras block */ 605 r = amdgpu_umc_ras_sw_init(adev); 606 if (r) 607 return r; 608 609 /* mmhub ras block */ 610 r = amdgpu_mmhub_ras_sw_init(adev); 611 if (r) 612 return r; 613 614 /* hdp ras block */ 615 r = amdgpu_hdp_ras_sw_init(adev); 616 if (r) 617 return r; 618 619 /* mca.x ras block */ 620 r = amdgpu_mca_mp0_ras_sw_init(adev); 621 if (r) 622 return r; 623 624 r = amdgpu_mca_mp1_ras_sw_init(adev); 625 if (r) 626 return r; 627 628 r = amdgpu_mca_mpio_ras_sw_init(adev); 629 if (r) 630 return r; 631 632 /* xgmi ras block */ 633 r = amdgpu_xgmi_ras_sw_init(adev); 634 if (r) 635 return r; 636 637 return 0; 638 } 639 640 /* 641 * The latest engine allocation on gfx9/10 is: 642 * Engine 2, 3: firmware 643 * Engine 0, 1, 4~16: amdgpu ring, 644 * subject to change when ring number changes 645 * Engine 17: Gart flushes 646 */ 647 #define AMDGPU_VMHUB_INV_ENG_BITMAP 0x1FFF3 648 649 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) 650 { 651 struct amdgpu_ring *ring; 652 unsigned vm_inv_engs[AMDGPU_MAX_VMHUBS] = {0}; 653 unsigned i; 654 unsigned vmhub, inv_eng; 655 struct amdgpu_ring *shared_ring; 656 657 /* init the vm inv eng for all vmhubs */ 658 for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 659 vm_inv_engs[i] = AMDGPU_VMHUB_INV_ENG_BITMAP; 660 /* reserve engine 5 for firmware */ 661 if (adev->enable_mes) 662 vm_inv_engs[i] &= ~(1 << 5); 663 /* reserve engine 6 for uni mes */ 664 if (adev->enable_uni_mes) 665 vm_inv_engs[i] &= ~(1 << 6); 666 /* reserve mmhub engine 3 for firmware */ 667 if (adev->enable_umsch_mm) 668 vm_inv_engs[i] &= ~(1 << 3); 669 } 670 671 for (i = 0; i < adev->num_rings; ++i) { 672 ring = adev->rings[i]; 673 vmhub = ring->vm_hub; 674 675 if (ring == &adev->mes.ring[0] || 676 ring == &adev->mes.ring[1] || 677 ring == &adev->umsch_mm.ring || 678 ring == &adev->cper.ring_buf) 679 continue; 680 681 /* Skip if the ring is a shared ring */ 682 if (amdgpu_sdma_is_shared_inv_eng(adev, ring) || 683 amdgpu_jpeg_is_shared_inv_eng(adev, ring)) 684 continue; 685 686 inv_eng = ffs(vm_inv_engs[vmhub]); 687 if (!inv_eng) { 688 dev_err(adev->dev, "no VM inv eng for ring %s\n", 689 ring->name); 690 return -EINVAL; 691 } 692 693 ring->vm_inv_eng = inv_eng - 1; 694 vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng); 695 696 dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n", 697 ring->name, ring->vm_inv_eng, ring->vm_hub); 698 /* SDMA has a special packet which allows it to use the same 699 * invalidation engine for all the rings in one instance. 700 * Therefore, we do not allocate a separate VM invalidation engine 701 * for SDMA page rings. Instead, they share the VM invalidation 702 * engine with the SDMA gfx ring. This change ensures efficient 703 * resource management and avoids the issue of insufficient VM 704 * invalidation engines. 705 */ 706 shared_ring = amdgpu_sdma_get_shared_ring(adev, ring); 707 if (shared_ring) { 708 shared_ring->vm_inv_eng = ring->vm_inv_eng; 709 dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n", 710 ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub); 711 continue; 712 } 713 714 /* All decode rings within one JPEG instance share one VM 715 * invalidation engine (no-op unless ring is a JPEG instance's 716 * first ring). 717 */ 718 amdgpu_jpeg_set_shared_inv_eng(adev, ring); 719 } 720 721 return 0; 722 } 723 724 void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 725 uint32_t vmhub, uint32_t flush_type) 726 { 727 struct amdgpu_ring *ring; 728 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 729 struct dma_fence *fence; 730 struct amdgpu_job *job; 731 int r; 732 733 ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); 734 735 if (!hub->sdma_invalidation_workaround || vmid || 736 !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || 737 !ring->sched.ready) { 738 /* 739 * A GPU reset should flush all TLBs anyway, so no need to do 740 * this while one is ongoing. 741 */ 742 if (!down_read_trylock(&adev->reset_domain->sem)) 743 return; 744 745 if (adev->gmc.flush_tlb_needs_extra_type_2) 746 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 747 vmhub, 2); 748 749 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 750 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 751 vmhub, 0); 752 753 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub, 754 flush_type); 755 up_read(&adev->reset_domain->sem); 756 return; 757 } 758 759 /* The SDMA on Navi 1x has a bug which can theoretically result in memory 760 * corruption if an invalidation happens at the same time as an VA 761 * translation. Avoid this by doing the invalidation from the SDMA 762 * itself at least for GART. 763 */ 764 mutex_lock(&adev->mman.default_entity.lock); 765 r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base, 766 AMDGPU_FENCE_OWNER_UNDEFINED, 767 16 * 4, AMDGPU_IB_POOL_IMMEDIATE, 768 AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB, 769 &job); 770 if (r) 771 goto error_alloc; 772 773 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); 774 job->vm_needs_flush = true; 775 job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop; 776 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 777 fence = amdgpu_job_submit(job); 778 mutex_unlock(&adev->mman.default_entity.lock); 779 780 dma_fence_wait(fence, false); 781 dma_fence_put(fence); 782 783 return; 784 785 error_alloc: 786 mutex_unlock(&adev->mman.default_entity.lock); 787 dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r); 788 } 789 790 static void amdgpu_gmc_flush_pasid_regs(struct amdgpu_device *adev, u16 pasid, 791 u32 flush_type, bool all_hub, 792 u32 inst) 793 { 794 if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) 795 return; 796 797 if (adev->gmc.flush_tlb_needs_extra_type_2) 798 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 2, all_hub, 799 inst); 800 801 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 802 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 0, all_hub, 803 inst); 804 805 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, flush_type, all_hub, 806 inst); 807 } 808 809 /* 810 * Queue the invalidation on one XCC and return its fence without waiting, so 811 * that a caller flushing several XCCs can have them in flight together. The 812 * KIQ ring, its lock and its fence sequence are per XCC, so the submissions do 813 * not interfere with each other. 814 */ 815 static int amdgpu_gmc_flush_pasid_kiq_submit(struct amdgpu_device *adev, 816 u16 pasid, u32 flush_type, 817 bool all_hub, u32 inst, 818 u32 *seq) 819 { 820 struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; 821 struct amdgpu_ring *ring = &kiq->ring; 822 unsigned int ndw; 823 int r; 824 825 /* one flush + 8 dwords fence */ 826 ndw = kiq->pmf->invalidate_tlbs_size + 8; 827 828 if (adev->gmc.flush_tlb_needs_extra_type_2) 829 ndw += kiq->pmf->invalidate_tlbs_size; 830 831 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 832 ndw += kiq->pmf->invalidate_tlbs_size; 833 834 spin_lock(&kiq->ring_lock); 835 r = amdgpu_ring_alloc(ring, ndw); 836 if (r) { 837 spin_unlock(&kiq->ring_lock); 838 return r; 839 } 840 841 if (adev->gmc.flush_tlb_needs_extra_type_2) 842 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); 843 844 if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) 845 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); 846 847 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); 848 r = amdgpu_fence_emit_polling(ring, seq, MAX_KIQ_REG_WAIT); 849 if (r) { 850 amdgpu_ring_undo(ring); 851 spin_unlock(&kiq->ring_lock); 852 return r; 853 } 854 855 amdgpu_ring_commit(ring); 856 spin_unlock(&kiq->ring_lock); 857 858 return 0; 859 } 860 861 /* 862 * Wait for an invalidation queued by amdgpu_gmc_flush_pasid_kiq_submit(). 863 * 864 * Bailing out because a reset became pending is reported as success: the reset 865 * flushes all TLBs anyway, so the invalidation no longer has to complete. Only 866 * running out of tries is an error. 867 */ 868 static int amdgpu_gmc_flush_pasid_kiq_wait(struct amdgpu_device *adev, 869 u32 inst, u32 seq) 870 { 871 struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; 872 int cnt = 0; 873 signed long r; 874 875 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 876 877 might_sleep(); 878 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 879 !amdgpu_reset_pending(adev->reset_domain)) { 880 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 881 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 882 } 883 884 if (cnt > MAX_KIQ_REG_TRY) { 885 dev_err(adev->dev, "timeout waiting for kiq fence\n"); 886 return -ETIME; 887 } 888 889 return 0; 890 } 891 892 /** 893 * amdgpu_gmc_flush_gpu_tlb_pasid_xccs - flush a pasid on several XCCs at once 894 * 895 * @adev: amdgpu_device pointer 896 * @pasid: pasid to be flushed 897 * @flush_type: the flush type 898 * @all_hub: flush all hubs 899 * @xcc_mask: mask of the XCCs to flush 900 * 901 * Submits the invalidation to every XCC in @xcc_mask before waiting for any of 902 * them, so the round trips overlap instead of running back to back. Flushing 903 * one XCC at a time costs num_xcc times the latency of a single one, which on 904 * a partition holding every XCC of the device is most of the cost of a compute 905 * TLB flush. 906 * 907 * The XCCs are independent of each other here: the page tables are already 908 * updated before any invalidation is issued, and nothing in an invalidation 909 * depends on another XCC having completed its own. 910 * 911 * Returns: 912 * 0 for success, the first error otherwise. Every XCC is flushed even if one 913 * of them fails. 914 */ 915 int amdgpu_gmc_flush_gpu_tlb_pasid_xccs(struct amdgpu_device *adev, u16 pasid, 916 u32 flush_type, bool all_hub, 917 u32 xcc_mask) 918 { 919 u32 seq[AMDGPU_MAX_GC_INSTANCES]; 920 unsigned long pending = 0; 921 int xcc, r = 0, err; 922 923 /* 924 * A GPU reset should flush all TLBs anyway, so no need to do 925 * this while one is ongoing. 926 * 927 * Unlike the one XCC at a time flush this holds the read side across 928 * every submit and every wait, so a reset's down_write() can only get 929 * in once the whole mask is done. The waits are sequential, which 930 * bounds that at num_xcc * MAX_KIQ_REG_TRY * MAX_KIQ_REG_BAILOUT_INTERVAL 931 * in the worst case, and amdgpu_gmc_flush_pasid_kiq_wait() cuts each 932 * wait short as soon as a reset becomes pending. 933 */ 934 if (!down_read_trylock(&adev->reset_domain->sem)) 935 return 0; 936 937 for_each_inst(xcc, xcc_mask) { 938 struct amdgpu_ring *ring; 939 940 if (WARN_ON_ONCE(xcc >= AMDGPU_MAX_GC_INSTANCES)) { 941 if (!r) 942 r = -EINVAL; 943 break; 944 } 945 946 ring = &adev->gfx.kiq[xcc].ring; 947 if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) { 948 amdgpu_gmc_flush_pasid_regs(adev, pasid, flush_type, 949 all_hub, xcc); 950 continue; 951 } 952 953 err = amdgpu_gmc_flush_pasid_kiq_submit(adev, pasid, flush_type, 954 all_hub, xcc, &seq[xcc]); 955 if (err) { 956 if (!r) 957 r = err; 958 continue; 959 } 960 961 pending |= BIT(xcc); 962 } 963 964 for_each_set_bit(xcc, &pending, AMDGPU_MAX_GC_INSTANCES) { 965 err = amdgpu_gmc_flush_pasid_kiq_wait(adev, xcc, seq[xcc]); 966 if (err && !r) 967 r = err; 968 } 969 970 up_read(&adev->reset_domain->sem); 971 return r; 972 } 973 974 int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, u16 pasid, 975 u32 flush_type, bool all_hub, u32 inst) 976 { 977 return amdgpu_gmc_flush_gpu_tlb_pasid_xccs(adev, pasid, flush_type, 978 all_hub, BIT(inst)); 979 } 980 981 void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, 982 uint32_t reg0, uint32_t reg1, 983 uint32_t ref, uint32_t mask, 984 uint32_t xcc_inst) 985 { 986 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_inst]; 987 struct amdgpu_ring *ring = &kiq->ring; 988 signed long r, cnt = 0; 989 unsigned long flags; 990 uint32_t seq; 991 992 if (adev->mes.ring[MES_PIPE_INST(xcc_inst, 0)].sched.ready) { 993 amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1, 994 ref, mask, xcc_inst); 995 return; 996 } 997 998 spin_lock_irqsave(&kiq->ring_lock, flags); 999 amdgpu_ring_alloc(ring, 32); 1000 amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 1001 ref, mask); 1002 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 1003 if (r) 1004 goto failed_undo; 1005 1006 amdgpu_ring_commit(ring); 1007 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1008 1009 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1010 1011 /* don't wait anymore for IRQ context */ 1012 if (r < 1 && in_interrupt()) 1013 goto failed_kiq; 1014 1015 might_sleep(); 1016 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 1017 !amdgpu_reset_pending(adev->reset_domain)) { 1018 1019 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 1020 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 1021 } 1022 1023 if (cnt > MAX_KIQ_REG_TRY) 1024 goto failed_kiq; 1025 1026 return; 1027 1028 failed_undo: 1029 amdgpu_ring_undo(ring); 1030 spin_unlock_irqrestore(&kiq->ring_lock, flags); 1031 failed_kiq: 1032 dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1); 1033 } 1034 1035 /** 1036 * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ 1037 * @adev: amdgpu_device pointer 1038 * 1039 * Check and set if an the device @adev supports Trusted Memory 1040 * Zones (TMZ). 1041 */ 1042 void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) 1043 { 1044 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1045 /* RAVEN */ 1046 case IP_VERSION(9, 2, 2): 1047 case IP_VERSION(9, 1, 0): 1048 /* RENOIR looks like RAVEN */ 1049 case IP_VERSION(9, 3, 0): 1050 /* GC 10.3.7 */ 1051 case IP_VERSION(10, 3, 7): 1052 /* GC 11.0.1 */ 1053 case IP_VERSION(11, 0, 1): 1054 if (amdgpu_tmz == 0) { 1055 adev->gmc.tmz_enabled = false; 1056 dev_info(adev->dev, 1057 "Trusted Memory Zone (TMZ) feature disabled (cmd line)\n"); 1058 } else { 1059 adev->gmc.tmz_enabled = true; 1060 dev_info(adev->dev, 1061 "Trusted Memory Zone (TMZ) feature enabled\n"); 1062 } 1063 break; 1064 case IP_VERSION(10, 1, 10): 1065 case IP_VERSION(10, 1, 1): 1066 case IP_VERSION(10, 1, 2): 1067 case IP_VERSION(10, 1, 3): 1068 case IP_VERSION(10, 3, 0): 1069 case IP_VERSION(10, 3, 2): 1070 case IP_VERSION(10, 3, 4): 1071 case IP_VERSION(10, 3, 5): 1072 case IP_VERSION(10, 3, 6): 1073 /* VANGOGH */ 1074 case IP_VERSION(10, 3, 1): 1075 /* YELLOW_CARP*/ 1076 case IP_VERSION(10, 3, 3): 1077 case IP_VERSION(11, 0, 4): 1078 case IP_VERSION(11, 5, 0): 1079 case IP_VERSION(11, 5, 1): 1080 case IP_VERSION(11, 5, 2): 1081 case IP_VERSION(11, 5, 3): 1082 case IP_VERSION(11, 5, 4): 1083 case IP_VERSION(11, 5, 6): 1084 case IP_VERSION(11, 7, 0): 1085 case IP_VERSION(11, 7, 1): 1086 /* Don't enable it by default yet. 1087 */ 1088 if (amdgpu_tmz < 1) { 1089 adev->gmc.tmz_enabled = false; 1090 dev_info(adev->dev, 1091 "Trusted Memory Zone (TMZ) feature disabled as experimental (default)\n"); 1092 } else { 1093 adev->gmc.tmz_enabled = true; 1094 dev_info(adev->dev, 1095 "Trusted Memory Zone (TMZ) feature enabled as experimental (cmd line)\n"); 1096 } 1097 break; 1098 default: 1099 adev->gmc.tmz_enabled = false; 1100 dev_info(adev->dev, 1101 "Trusted Memory Zone (TMZ) feature not supported\n"); 1102 break; 1103 } 1104 } 1105 1106 /** 1107 * amdgpu_gmc_noretry_set -- set per asic noretry defaults 1108 * @adev: amdgpu_device pointer 1109 * 1110 * Set a per asic default for the no-retry parameter. 1111 * 1112 */ 1113 void amdgpu_gmc_noretry_set(struct amdgpu_device *adev) 1114 { 1115 struct amdgpu_gmc *gmc = &adev->gmc; 1116 uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); 1117 bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) || 1118 gc_ver == IP_VERSION(9, 4, 0) || 1119 gc_ver == IP_VERSION(9, 4, 1) || 1120 gc_ver == IP_VERSION(9, 4, 2) || 1121 gc_ver == IP_VERSION(9, 4, 3) || 1122 gc_ver == IP_VERSION(9, 4, 4) || 1123 gc_ver == IP_VERSION(9, 5, 0) || 1124 gc_ver >= IP_VERSION(10, 1, 0)); 1125 1126 if (!amdgpu_sriov_xnack_support(adev)) 1127 gmc->noretry = 1; 1128 else 1129 gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry; 1130 } 1131 1132 void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, 1133 bool enable) 1134 { 1135 struct amdgpu_vmhub *hub; 1136 u32 tmp, reg, i; 1137 1138 hub = &adev->vmhub[hub_type]; 1139 for (i = 0; i < 16; i++) { 1140 reg = hub->vm_context0_cntl + hub->ctx_distance * i; 1141 1142 tmp = (hub_type == AMDGPU_GFXHUB(0)) ? 1143 RREG32_SOC15_IP(GC, reg) : 1144 RREG32_SOC15_IP(MMHUB, reg); 1145 1146 if (enable) 1147 tmp |= hub->vm_cntx_cntl_vm_fault; 1148 else 1149 tmp &= ~hub->vm_cntx_cntl_vm_fault; 1150 1151 (hub_type == AMDGPU_GFXHUB(0)) ? 1152 WREG32_SOC15_IP(GC, reg, tmp) : 1153 WREG32_SOC15_IP(MMHUB, reg, tmp); 1154 } 1155 } 1156 1157 void amdgpu_gmc_init_vga_resv_regions(struct amdgpu_device *adev) 1158 { 1159 unsigned size; 1160 1161 if (adev->gmc.is_app_apu) 1162 return; 1163 1164 /* 1165 * Some ASICs need to reserve a region of video memory to avoid access 1166 * from driver 1167 */ 1168 /* 1169 * TODO: 1170 * Currently there is a bug where some memory client outside 1171 * of the driver writes to first 8M of VRAM on S3 resume, 1172 * this overrides GART which by default gets placed in first 8M and 1173 * causes VM_FAULTS once GTT is accessed. 1174 * Keep the stolen memory reservation until the while this is not solved. 1175 */ 1176 switch (adev->asic_type) { 1177 case CHIP_VEGA10: 1178 adev->mman.keep_stolen_vga_memory = true; 1179 /* 1180 * VEGA10 SRIOV VF with MS_HYPERV host needs some firmware reserved area. 1181 */ 1182 #ifdef CONFIG_X86 1183 if (amdgpu_sriov_vf(adev) && hypervisor_is_type(X86_HYPER_MS_HYPERV)) { 1184 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_RESERVED, 1185 0x500000, 0x200000, false); 1186 } 1187 #endif 1188 break; 1189 case CHIP_RAVEN: 1190 case CHIP_RENOIR: 1191 adev->mman.keep_stolen_vga_memory = true; 1192 break; 1193 default: 1194 adev->mman.keep_stolen_vga_memory = false; 1195 break; 1196 } 1197 /* MacBookPros with switchable graphics put VRAM at 0 when 1198 * the iGPU is enabled which results in cursor issues if 1199 * the cursor ends up at 0. Reserve vram at 0 in that case. 1200 */ 1201 if ((adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE) && 1202 (adev->gmc.vram_start == 0)) 1203 adev->mman.keep_stolen_vga_memory = true; 1204 1205 if (amdgpu_sriov_vf(adev) || 1206 !amdgpu_device_has_display_hardware(adev)) { 1207 size = 0; 1208 } else { 1209 size = amdgpu_gmc_get_vbios_fb_size(adev); 1210 1211 if (adev->mman.keep_stolen_vga_memory) 1212 size = max(size, (unsigned)AMDGPU_VBIOS_VGA_ALLOCATION); 1213 } 1214 1215 /* set to 0 if the pre-OS buffer uses up most of vram */ 1216 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 1217 size = 0; 1218 1219 if (size > AMDGPU_VBIOS_VGA_ALLOCATION) { 1220 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1221 0, AMDGPU_VBIOS_VGA_ALLOCATION, false); 1222 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_EXTENDED, 1223 AMDGPU_VBIOS_VGA_ALLOCATION, 1224 size - AMDGPU_VBIOS_VGA_ALLOCATION, false); 1225 } else { 1226 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1227 0, size, false); 1228 } 1229 } 1230 1231 /** 1232 * amdgpu_gmc_init_pdb0 - initialize PDB0 1233 * 1234 * @adev: amdgpu_device pointer 1235 * 1236 * This function is only used when GART page table is used 1237 * for FB address translatioin. In such a case, we construct 1238 * a 2-level system VM page table: PDB0->PTB, to cover both 1239 * VRAM of the hive and system memory. 1240 * 1241 * PDB0 is static, initialized once on driver initialization. 1242 * The first n entries of PDB0 are used as PTE by setting 1243 * P bit to 1, pointing to VRAM. The n+1'th entry points 1244 * to a big PTB covering system memory. 1245 * 1246 */ 1247 void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev) 1248 { 1249 int i; 1250 uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW? 1251 /* Each PDE0 (used as PTE) covers (2^vmid0_page_table_block_size)*2M 1252 */ 1253 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 1254 u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21; 1255 u64 vram_addr, vram_end; 1256 u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo); 1257 int idx; 1258 1259 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 1260 return; 1261 1262 flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE; 1263 flags |= AMDGPU_PTE_WRITEABLE; 1264 flags |= AMDGPU_PTE_SNOOPED; 1265 if (amdgpu_virt_vram_is_spm(adev)) 1266 flags |= AMDGPU_PTE_SYSTEM; 1267 flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1)); 1268 flags |= AMDGPU_PDE_PTE_FLAG(adev); 1269 1270 vram_addr = adev->vm_manager.vram_base_offset; 1271 if (!amdgpu_virt_xgmi_migrate_enabled(adev)) 1272 vram_addr -= adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1273 vram_end = vram_addr + vram_size; 1274 1275 /* The first n PDE0 entries are used as PTE, 1276 * pointing to vram 1277 */ 1278 for (i = 0; vram_addr < vram_end; i++, vram_addr += pde0_page_size) 1279 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, vram_addr, flags); 1280 1281 /* The n+1'th PDE0 entry points to a huge 1282 * PTB who has more than 512 entries each 1283 * pointing to a 4K system page 1284 */ 1285 flags = AMDGPU_PTE_VALID; 1286 if (amdgpu_virt_vram_is_spm(adev)) 1287 flags |= AMDGPU_PTE_SYSTEM; 1288 flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0); 1289 /* Requires gart_ptb_gpu_pa to be 4K aligned */ 1290 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags); 1291 drm_dev_exit(idx); 1292 } 1293 1294 /** 1295 * amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC 1296 * address 1297 * 1298 * @adev: amdgpu_device pointer 1299 * @mc_addr: MC address of buffer 1300 */ 1301 uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr) 1302 { 1303 return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset; 1304 } 1305 1306 /** 1307 * amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from 1308 * GPU's view 1309 * 1310 * @adev: amdgpu_device pointer 1311 * @bo: amdgpu buffer object 1312 */ 1313 uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo) 1314 { 1315 return amdgpu_gmc_vram_mc2pa(adev, amdgpu_bo_gpu_offset(bo)); 1316 } 1317 1318 int amdgpu_gmc_vram_checking(struct amdgpu_device *adev) 1319 { 1320 struct amdgpu_bo *vram_bo = NULL; 1321 uint64_t vram_gpu = 0; 1322 void *vram_ptr = NULL; 1323 1324 int ret, size = 0x100000; 1325 uint8_t cptr[10]; 1326 1327 ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, 1328 AMDGPU_GEM_DOMAIN_VRAM, 1329 &vram_bo, 1330 &vram_gpu, 1331 &vram_ptr); 1332 if (ret) 1333 return ret; 1334 1335 memset(vram_ptr, 0x86, size); 1336 memset(cptr, 0x86, 10); 1337 1338 /** 1339 * Check the start, the mid, and the end of the memory if the content of 1340 * each byte is the pattern "0x86". If yes, we suppose the vram bo is 1341 * workable. 1342 * 1343 * Note: If check the each byte of whole 1M bo, it will cost too many 1344 * seconds, so here, we just pick up three parts for emulation. 1345 */ 1346 ret = memcmp(vram_ptr, cptr, 10); 1347 if (ret) { 1348 ret = -EIO; 1349 goto release_buffer; 1350 } 1351 1352 ret = memcmp(vram_ptr + (size / 2), cptr, 10); 1353 if (ret) { 1354 ret = -EIO; 1355 goto release_buffer; 1356 } 1357 1358 ret = memcmp(vram_ptr + size - 10, cptr, 10); 1359 if (ret) { 1360 ret = -EIO; 1361 goto release_buffer; 1362 } 1363 1364 release_buffer: 1365 amdgpu_bo_free_kernel(&vram_bo, &vram_gpu, 1366 &vram_ptr); 1367 1368 return ret; 1369 } 1370 1371 static const char *nps_desc[] = { 1372 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 1373 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 1374 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 1375 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 1376 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 1377 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 1378 }; 1379 1380 static ssize_t available_memory_partition_show(struct device *dev, 1381 struct device_attribute *addr, 1382 char *buf) 1383 { 1384 struct drm_device *ddev = dev_get_drvdata(dev); 1385 struct amdgpu_device *adev = drm_to_adev(ddev); 1386 int size = 0, mode; 1387 char *sep = ""; 1388 1389 for_each_inst(mode, adev->gmc.supported_nps_modes) { 1390 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 1391 sep = ", "; 1392 } 1393 size += sysfs_emit_at(buf, size, "\n"); 1394 1395 return size; 1396 } 1397 1398 static ssize_t current_memory_partition_store(struct device *dev, 1399 struct device_attribute *attr, 1400 const char *buf, size_t count) 1401 { 1402 struct drm_device *ddev = dev_get_drvdata(dev); 1403 struct amdgpu_device *adev = drm_to_adev(ddev); 1404 enum amdgpu_memory_partition mode; 1405 struct amdgpu_hive_info *hive; 1406 int i; 1407 1408 mode = UNKNOWN_MEMORY_PARTITION_MODE; 1409 for_each_inst(i, adev->gmc.supported_nps_modes) { 1410 if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) { 1411 mode = i; 1412 break; 1413 } 1414 } 1415 1416 if (mode == UNKNOWN_MEMORY_PARTITION_MODE) 1417 return -EINVAL; 1418 1419 if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) { 1420 dev_info( 1421 adev->dev, 1422 "requested NPS mode is same as current NPS mode, skipping\n"); 1423 return count; 1424 } 1425 1426 /* If device is part of hive, all devices in the hive should request the 1427 * same mode. Hence store the requested mode in hive. 1428 */ 1429 hive = amdgpu_get_xgmi_hive(adev); 1430 if (hive) { 1431 atomic_set(&hive->requested_nps_mode, mode); 1432 amdgpu_put_xgmi_hive(hive); 1433 } else { 1434 adev->gmc.requested_nps_mode = mode; 1435 } 1436 1437 dev_info( 1438 adev->dev, 1439 "NPS mode change requested, please remove and reload the driver\n"); 1440 1441 return count; 1442 } 1443 1444 static ssize_t current_memory_partition_show( 1445 struct device *dev, struct device_attribute *addr, char *buf) 1446 { 1447 struct drm_device *ddev = dev_get_drvdata(dev); 1448 struct amdgpu_device *adev = drm_to_adev(ddev); 1449 enum amdgpu_memory_partition mode; 1450 1451 /* Only minimal precaution taken to reject requests while in reset */ 1452 if (amdgpu_in_reset(adev)) 1453 return -EPERM; 1454 1455 mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1456 if ((mode >= ARRAY_SIZE(nps_desc)) || 1457 (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode)) 1458 return sysfs_emit(buf, "UNKNOWN\n"); 1459 1460 return sysfs_emit(buf, "%s\n", nps_desc[mode]); 1461 } 1462 1463 static DEVICE_ATTR_RW(current_memory_partition); 1464 static DEVICE_ATTR_RO(available_memory_partition); 1465 1466 int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev) 1467 { 1468 bool nps_switch_support; 1469 int r = 0; 1470 1471 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1472 return 0; 1473 1474 nps_switch_support = !adev->gmc.xgmi.connected_to_cpu && 1475 (hweight32(adev->gmc.supported_nps_modes & 1476 AMDGPU_ALL_NPS_MASK) > 1); 1477 if (!nps_switch_support) 1478 dev_attr_current_memory_partition.attr.mode &= 1479 ~(S_IWUSR | S_IWGRP | S_IWOTH); 1480 else 1481 r = device_create_file(adev->dev, 1482 &dev_attr_available_memory_partition); 1483 1484 if (r) 1485 return r; 1486 1487 return device_create_file(adev->dev, 1488 &dev_attr_current_memory_partition); 1489 } 1490 1491 void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev) 1492 { 1493 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1494 return; 1495 1496 device_remove_file(adev->dev, &dev_attr_current_memory_partition); 1497 device_remove_file(adev->dev, &dev_attr_available_memory_partition); 1498 } 1499 1500 int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, 1501 struct amdgpu_mem_partition_info *mem_ranges, 1502 uint8_t *exp_ranges) 1503 { 1504 struct amdgpu_gmc_memrange ranges[AMDGPU_MAX_MEM_RANGES]; 1505 int range_cnt, ret, i, j; 1506 uint32_t nps_type; 1507 bool refresh; 1508 1509 if (!mem_ranges || !exp_ranges) 1510 return -EINVAL; 1511 range_cnt = AMDGPU_MAX_MEM_RANGES; 1512 refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) && 1513 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS); 1514 ret = amdgpu_discovery_get_nps_info(adev, &nps_type, ranges, &range_cnt, 1515 refresh); 1516 1517 if (ret) 1518 return ret; 1519 1520 /* TODO: For now, expect ranges and partition count to be the same. 1521 * Adjust if there are holes expected in any NPS domain. 1522 */ 1523 if (*exp_ranges && (range_cnt != *exp_ranges)) { 1524 dev_warn( 1525 adev->dev, 1526 "NPS config mismatch - expected ranges: %d discovery - nps mode: %d, nps ranges: %d", 1527 *exp_ranges, nps_type, range_cnt); 1528 ret = -EINVAL; 1529 goto err; 1530 } 1531 1532 for (i = 0; i < range_cnt; ++i) { 1533 if (ranges[i].base_address >= ranges[i].limit_address) { 1534 dev_warn( 1535 adev->dev, 1536 "Invalid NPS range - nps mode: %d, range[%d]: base: %llx limit: %llx", 1537 nps_type, i, ranges[i].base_address, 1538 ranges[i].limit_address); 1539 ret = -EINVAL; 1540 goto err; 1541 } 1542 1543 /* Check for overlaps, not expecting any now */ 1544 for (j = i - 1; j >= 0; j--) { 1545 if (max(ranges[j].base_address, 1546 ranges[i].base_address) <= 1547 min(ranges[j].limit_address, 1548 ranges[i].limit_address)) { 1549 dev_warn( 1550 adev->dev, 1551 "overlapping ranges detected [ %llx - %llx ] | [%llx - %llx]", 1552 ranges[j].base_address, 1553 ranges[j].limit_address, 1554 ranges[i].base_address, 1555 ranges[i].limit_address); 1556 ret = -EINVAL; 1557 goto err; 1558 } 1559 } 1560 1561 mem_ranges[i].range.fpfn = 1562 (ranges[i].base_address - 1563 adev->vm_manager.vram_base_offset) >> 1564 AMDGPU_GPU_PAGE_SHIFT; 1565 mem_ranges[i].range.lpfn = 1566 (ranges[i].limit_address - 1567 adev->vm_manager.vram_base_offset) >> 1568 AMDGPU_GPU_PAGE_SHIFT; 1569 mem_ranges[i].size = 1570 ranges[i].limit_address - ranges[i].base_address + 1; 1571 } 1572 1573 if (!*exp_ranges) 1574 *exp_ranges = range_cnt; 1575 err: 1576 return ret; 1577 } 1578 1579 int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev, 1580 int nps_mode) 1581 { 1582 /* Not supported on VF devices and APUs */ 1583 if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU)) 1584 return -EOPNOTSUPP; 1585 1586 if (!adev->psp.funcs) { 1587 dev_err(adev->dev, 1588 "PSP interface not available for nps mode change request"); 1589 return -EINVAL; 1590 } 1591 1592 return psp_memory_partition(&adev->psp, nps_mode); 1593 } 1594 1595 static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev, 1596 int req_nps_mode, 1597 int cur_nps_mode) 1598 { 1599 return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) == 1600 BIT(req_nps_mode)) && 1601 req_nps_mode != cur_nps_mode); 1602 } 1603 1604 void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev) 1605 { 1606 int req_nps_mode, cur_nps_mode, r; 1607 struct amdgpu_hive_info *hive; 1608 1609 if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes || 1610 !adev->gmc.gmc_funcs->request_mem_partition_mode) 1611 return; 1612 1613 cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1614 hive = amdgpu_get_xgmi_hive(adev); 1615 if (hive) { 1616 req_nps_mode = atomic_read(&hive->requested_nps_mode); 1617 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, 1618 cur_nps_mode)) { 1619 amdgpu_put_xgmi_hive(hive); 1620 return; 1621 } 1622 r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode); 1623 amdgpu_put_xgmi_hive(hive); 1624 goto out; 1625 } 1626 1627 req_nps_mode = adev->gmc.requested_nps_mode; 1628 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode)) 1629 return; 1630 1631 /* even if this fails, we should let driver unload w/o blocking */ 1632 r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode); 1633 out: 1634 if (r) 1635 dev_err(adev->dev, "NPS mode change request failed\n"); 1636 else 1637 dev_info( 1638 adev->dev, 1639 "NPS mode change request done, reload driver to complete the change\n"); 1640 } 1641 1642 bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) 1643 { 1644 if (adev->gmc.gmc_funcs->need_reset_on_init) 1645 return adev->gmc.gmc_funcs->need_reset_on_init(adev); 1646 1647 return false; 1648 } 1649 1650 enum amdgpu_memory_partition 1651 amdgpu_gmc_get_vf_memory_partition(struct amdgpu_device *adev) 1652 { 1653 switch (adev->gmc.num_mem_partitions) { 1654 case 0: 1655 return UNKNOWN_MEMORY_PARTITION_MODE; 1656 case 1: 1657 return AMDGPU_NPS1_PARTITION_MODE; 1658 case 2: 1659 return AMDGPU_NPS2_PARTITION_MODE; 1660 case 4: 1661 return AMDGPU_NPS4_PARTITION_MODE; 1662 case 8: 1663 return AMDGPU_NPS8_PARTITION_MODE; 1664 default: 1665 return AMDGPU_NPS1_PARTITION_MODE; 1666 } 1667 } 1668 1669 enum amdgpu_memory_partition 1670 amdgpu_gmc_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes) 1671 { 1672 enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE; 1673 1674 if (adev->nbio.funcs && 1675 adev->nbio.funcs->get_memory_partition_mode) 1676 mode = adev->nbio.funcs->get_memory_partition_mode(adev, 1677 supp_modes); 1678 else 1679 dev_warn(adev->dev, "memory partition mode query is not supported\n"); 1680 1681 return mode; 1682 } 1683 1684 enum amdgpu_memory_partition 1685 amdgpu_gmc_query_memory_partition(struct amdgpu_device *adev) 1686 { 1687 if (amdgpu_sriov_vf(adev)) 1688 return amdgpu_gmc_get_vf_memory_partition(adev); 1689 else 1690 return amdgpu_gmc_get_memory_partition(adev, NULL); 1691 } 1692 1693 static bool amdgpu_gmc_validate_partition_info(struct amdgpu_device *adev) 1694 { 1695 enum amdgpu_memory_partition mode; 1696 u32 supp_modes; 1697 bool valid; 1698 1699 mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes); 1700 1701 /* Mode detected by hardware not present in supported modes */ 1702 if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && 1703 !(BIT(mode - 1) & supp_modes)) 1704 return false; 1705 1706 switch (mode) { 1707 case UNKNOWN_MEMORY_PARTITION_MODE: 1708 case AMDGPU_NPS1_PARTITION_MODE: 1709 valid = (adev->gmc.num_mem_partitions == 1); 1710 break; 1711 case AMDGPU_NPS2_PARTITION_MODE: 1712 valid = (adev->gmc.num_mem_partitions == 2); 1713 break; 1714 case AMDGPU_NPS4_PARTITION_MODE: 1715 valid = (adev->gmc.num_mem_partitions == 3 || 1716 adev->gmc.num_mem_partitions == 4); 1717 break; 1718 case AMDGPU_NPS8_PARTITION_MODE: 1719 valid = (adev->gmc.num_mem_partitions == 8); 1720 break; 1721 default: 1722 valid = false; 1723 } 1724 1725 return valid; 1726 } 1727 1728 static bool amdgpu_gmc_is_node_present(int *node_ids, int num_ids, int nid) 1729 { 1730 int i; 1731 1732 /* Check if node with id 'nid' is present in 'node_ids' array */ 1733 for (i = 0; i < num_ids; ++i) 1734 if (node_ids[i] == nid) 1735 return true; 1736 1737 return false; 1738 } 1739 1740 static void 1741 amdgpu_gmc_init_acpi_mem_ranges(struct amdgpu_device *adev, 1742 struct amdgpu_mem_partition_info *mem_ranges) 1743 { 1744 struct amdgpu_numa_info numa_info; 1745 int node_ids[AMDGPU_MAX_MEM_RANGES]; 1746 int num_ranges = 0, ret; 1747 int num_xcc, xcc_id; 1748 uint32_t xcc_mask; 1749 1750 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1751 xcc_mask = (1U << num_xcc) - 1; 1752 1753 for_each_inst(xcc_id, xcc_mask) { 1754 ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info); 1755 if (ret) 1756 continue; 1757 1758 if (numa_info.nid == NUMA_NO_NODE) { 1759 mem_ranges[0].size = numa_info.size; 1760 mem_ranges[0].numa.node = numa_info.nid; 1761 num_ranges = 1; 1762 break; 1763 } 1764 1765 if (amdgpu_gmc_is_node_present(node_ids, num_ranges, 1766 numa_info.nid)) 1767 continue; 1768 1769 node_ids[num_ranges] = numa_info.nid; 1770 mem_ranges[num_ranges].numa.node = numa_info.nid; 1771 mem_ranges[num_ranges].size = numa_info.size; 1772 ++num_ranges; 1773 } 1774 1775 adev->gmc.num_mem_partitions = num_ranges; 1776 } 1777 1778 void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev, 1779 struct amdgpu_mem_partition_info *mem_ranges) 1780 { 1781 enum amdgpu_memory_partition mode; 1782 u32 start_addr = 0, size; 1783 int i, r, l; 1784 1785 mode = amdgpu_gmc_query_memory_partition(adev); 1786 1787 switch (mode) { 1788 case UNKNOWN_MEMORY_PARTITION_MODE: 1789 adev->gmc.num_mem_partitions = 0; 1790 break; 1791 case AMDGPU_NPS1_PARTITION_MODE: 1792 adev->gmc.num_mem_partitions = 1; 1793 break; 1794 case AMDGPU_NPS2_PARTITION_MODE: 1795 adev->gmc.num_mem_partitions = 2; 1796 break; 1797 case AMDGPU_NPS4_PARTITION_MODE: 1798 if (adev->flags & AMD_IS_APU) 1799 adev->gmc.num_mem_partitions = 3; 1800 else 1801 adev->gmc.num_mem_partitions = 4; 1802 break; 1803 case AMDGPU_NPS8_PARTITION_MODE: 1804 adev->gmc.num_mem_partitions = 8; 1805 break; 1806 default: 1807 adev->gmc.num_mem_partitions = 1; 1808 break; 1809 } 1810 1811 /* Use NPS range info, if populated */ 1812 r = amdgpu_gmc_get_nps_memranges(adev, mem_ranges, 1813 &adev->gmc.num_mem_partitions); 1814 if (!r) { 1815 l = 0; 1816 for (i = 1; i < adev->gmc.num_mem_partitions; ++i) { 1817 if (mem_ranges[i].range.lpfn > 1818 mem_ranges[i - 1].range.lpfn) 1819 l = i; 1820 } 1821 1822 } else { 1823 if (!adev->gmc.num_mem_partitions) { 1824 dev_warn(adev->dev, 1825 "Not able to detect NPS mode, fall back to NPS1\n"); 1826 adev->gmc.num_mem_partitions = 1; 1827 } 1828 /* Fallback to sw based calculation */ 1829 size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT; 1830 size /= adev->gmc.num_mem_partitions; 1831 1832 for (i = 0; i < adev->gmc.num_mem_partitions; ++i) { 1833 mem_ranges[i].range.fpfn = start_addr; 1834 mem_ranges[i].size = 1835 ((u64)size << AMDGPU_GPU_PAGE_SHIFT); 1836 mem_ranges[i].range.lpfn = start_addr + size - 1; 1837 start_addr += size; 1838 } 1839 1840 l = adev->gmc.num_mem_partitions - 1; 1841 } 1842 1843 /* Adjust the last one */ 1844 mem_ranges[l].range.lpfn = 1845 (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1; 1846 mem_ranges[l].size = 1847 adev->gmc.real_vram_size - 1848 ((u64)mem_ranges[l].range.fpfn << AMDGPU_GPU_PAGE_SHIFT); 1849 } 1850 1851 int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev) 1852 { 1853 bool valid; 1854 1855 adev->gmc.mem_partitions = kzalloc_objs(struct amdgpu_mem_partition_info, 1856 AMDGPU_MAX_MEM_RANGES); 1857 if (!adev->gmc.mem_partitions) 1858 return -ENOMEM; 1859 1860 if (adev->gmc.is_app_apu) 1861 amdgpu_gmc_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions); 1862 else 1863 amdgpu_gmc_init_sw_mem_ranges(adev, adev->gmc.mem_partitions); 1864 1865 if (amdgpu_sriov_vf(adev)) 1866 valid = true; 1867 else 1868 valid = amdgpu_gmc_validate_partition_info(adev); 1869 if (!valid) 1870 dev_warn(adev->dev, 1871 "Mem ranges not matching with hardware config\n"); 1872 1873 if (!adev->gmc.num_mem_partitions) { 1874 dev_err(adev->dev, "num_mem_partitions is zero\n"); 1875 kfree(adev->gmc.mem_partitions); 1876 adev->gmc.mem_partitions = NULL; 1877 return -EINVAL; 1878 } 1879 1880 return 0; 1881 } 1882 1883 int amdgpu_gmc_get_vram_info(struct amdgpu_device *adev, 1884 int *vram_width, int *vram_type, int *vram_vendor) 1885 { 1886 int ret = 0; 1887 1888 if (adev->flags & AMD_IS_APU) 1889 return amdgpu_atomfirmware_get_integrated_system_info(adev, 1890 vram_width, vram_type, vram_vendor); 1891 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1892 case IP_VERSION(12, 0, 0): 1893 case IP_VERSION(12, 0, 1): 1894 return amdgpu_atomfirmware_get_umc_info(adev, 1895 vram_width, vram_type, vram_vendor); 1896 case IP_VERSION(9, 5, 0): 1897 case IP_VERSION(9, 4, 4): 1898 case IP_VERSION(9, 4, 3): 1899 ret = amdgpu_atomfirmware_get_umc_info(adev, 1900 vram_width, vram_type, vram_vendor); 1901 if (vram_width && !ret) 1902 *vram_width *= hweight32(adev->aid_mask); 1903 return ret; 1904 default: 1905 return amdgpu_atomfirmware_get_vram_info(adev, 1906 vram_width, vram_type, vram_vendor); 1907 } 1908 return 0; 1909 } 1910