1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 27 #include <linux/io-64-nonatomic-lo-hi.h> 28 #ifdef CONFIG_X86 29 #include <asm/hypervisor.h> 30 #endif 31 32 #include "amdgpu.h" 33 #include "amdgpu_gmc.h" 34 #include "amdgpu_ras.h" 35 #include "amdgpu_reset.h" 36 #include "amdgpu_xgmi.h" 37 #include "amdgpu_atomfirmware.h" 38 39 #include <drm/drm_drv.h> 40 #include <drm/ttm/ttm_tt.h> 41 42 static const u64 four_gb = 0x100000000ULL; 43 44 bool amdgpu_gmc_is_pdb0_enabled(struct amdgpu_device *adev) 45 { 46 return adev->gmc.xgmi.connected_to_cpu || amdgpu_virt_xgmi_migrate_enabled(adev); 47 } 48 49 /** 50 * amdgpu_gmc_pdb0_alloc - allocate vram for pdb0 51 * 52 * @adev: amdgpu_device pointer 53 * 54 * Allocate video memory for pdb0 and map it for CPU access 55 * Returns 0 for success, error for failure. 56 */ 57 int amdgpu_gmc_pdb0_alloc(struct amdgpu_device *adev) 58 { 59 int r; 60 struct amdgpu_bo_param bp; 61 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 62 uint32_t pde0_page_shift = adev->gmc.vmid0_page_table_block_size + 21; 63 uint32_t npdes = (vram_size + (1ULL << pde0_page_shift) - 1) >> pde0_page_shift; 64 65 memset(&bp, 0, sizeof(bp)); 66 bp.size = PAGE_ALIGN((npdes + 1) * 8); 67 bp.byte_align = PAGE_SIZE; 68 bp.domain = AMDGPU_GEM_DOMAIN_VRAM; 69 bp.flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 70 AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 71 bp.type = ttm_bo_type_kernel; 72 bp.resv = NULL; 73 bp.bo_ptr_size = sizeof(struct amdgpu_bo); 74 75 r = amdgpu_bo_create(adev, &bp, &adev->gmc.pdb0_bo); 76 if (r) 77 return r; 78 79 r = amdgpu_bo_reserve(adev->gmc.pdb0_bo, false); 80 if (unlikely(r != 0)) 81 goto bo_reserve_failure; 82 83 r = amdgpu_bo_pin(adev->gmc.pdb0_bo, AMDGPU_GEM_DOMAIN_VRAM); 84 if (r) 85 goto bo_pin_failure; 86 r = amdgpu_bo_kmap(adev->gmc.pdb0_bo, &adev->gmc.ptr_pdb0); 87 if (r) 88 goto bo_kmap_failure; 89 90 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 91 return 0; 92 93 bo_kmap_failure: 94 amdgpu_bo_unpin(adev->gmc.pdb0_bo); 95 bo_pin_failure: 96 amdgpu_bo_unreserve(adev->gmc.pdb0_bo); 97 bo_reserve_failure: 98 amdgpu_bo_unref(&adev->gmc.pdb0_bo); 99 return r; 100 } 101 102 /** 103 * amdgpu_gmc_get_pde_for_bo - get the PDE for a BO 104 * 105 * @bo: the BO to get the PDE for 106 * @level: the level in the PD hirarchy 107 * @addr: resulting addr 108 * @flags: resulting flags 109 * 110 * Get the address and flags to be used for a PDE (Page Directory Entry). 111 */ 112 void amdgpu_gmc_get_pde_for_bo(struct amdgpu_bo *bo, int level, 113 uint64_t *addr, uint64_t *flags) 114 { 115 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 116 117 switch (bo->tbo.resource->mem_type) { 118 case TTM_PL_TT: 119 *addr = bo->tbo.ttm->dma_address[0]; 120 break; 121 case TTM_PL_VRAM: 122 *addr = amdgpu_bo_gpu_offset(bo); 123 break; 124 default: 125 *addr = 0; 126 break; 127 } 128 *flags = amdgpu_ttm_tt_pde_flags(bo->tbo.ttm, bo->tbo.resource); 129 amdgpu_gmc_get_vm_pde(adev, level, addr, flags); 130 } 131 132 /* 133 * amdgpu_gmc_pd_addr - return the address of the root directory 134 */ 135 uint64_t amdgpu_gmc_pd_addr(struct amdgpu_bo *bo) 136 { 137 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 138 uint64_t pd_addr; 139 140 /* TODO: move that into ASIC specific code */ 141 if (adev->asic_type >= CHIP_VEGA10) { 142 uint64_t flags = AMDGPU_PTE_VALID; 143 144 amdgpu_gmc_get_pde_for_bo(bo, -1, &pd_addr, &flags); 145 pd_addr |= flags; 146 } else { 147 pd_addr = amdgpu_bo_gpu_offset(bo); 148 } 149 return pd_addr; 150 } 151 152 /** 153 * amdgpu_gmc_set_pte_pde - update the page tables using CPU 154 * 155 * @adev: amdgpu_device pointer 156 * @cpu_pt_addr: cpu address of the page table 157 * @gpu_page_idx: entry in the page table to update 158 * @addr: dst addr to write into pte/pde 159 * @flags: access flags 160 * 161 * Update the page tables using CPU. 162 */ 163 int amdgpu_gmc_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 164 uint32_t gpu_page_idx, uint64_t addr, 165 uint64_t flags) 166 { 167 void __iomem *ptr = (void *)cpu_pt_addr; 168 uint64_t value; 169 170 /* 171 * The following is for PTE only. GART does not have PDEs. 172 */ 173 value = addr & 0x0000FFFFFFFFF000ULL; 174 value |= flags; 175 writeq(value, ptr + (gpu_page_idx * 8)); 176 177 return 0; 178 } 179 180 /** 181 * amdgpu_gmc_agp_addr - return the address in the AGP address space 182 * 183 * @bo: TTM BO which needs the address, must be in GTT domain 184 * 185 * Tries to figure out how to access the BO through the AGP aperture. Returns 186 * AMDGPU_BO_INVALID_OFFSET if that is not possible. 187 */ 188 uint64_t amdgpu_gmc_agp_addr(struct ttm_buffer_object *bo) 189 { 190 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->bdev); 191 192 if (!bo->ttm) 193 return AMDGPU_BO_INVALID_OFFSET; 194 195 if (bo->ttm->num_pages != 1 || bo->ttm->caching == ttm_cached) 196 return AMDGPU_BO_INVALID_OFFSET; 197 198 if (bo->ttm->dma_address[0] + PAGE_SIZE >= adev->gmc.agp_size) 199 return AMDGPU_BO_INVALID_OFFSET; 200 201 return adev->gmc.agp_start + bo->ttm->dma_address[0]; 202 } 203 204 /** 205 * amdgpu_gmc_vram_location - try to find VRAM location 206 * 207 * @adev: amdgpu device structure holding all necessary information 208 * @mc: memory controller structure holding memory information 209 * @base: base address at which to put VRAM 210 * 211 * Function will try to place VRAM at base address provided 212 * as parameter. 213 */ 214 void amdgpu_gmc_vram_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 215 u64 base) 216 { 217 uint64_t vis_limit = (uint64_t)amdgpu_vis_vram_limit << 20; 218 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; 219 220 mc->vram_start = base; 221 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 222 if (limit < mc->real_vram_size) 223 mc->real_vram_size = limit; 224 225 if (vis_limit && vis_limit < mc->visible_vram_size) 226 mc->visible_vram_size = vis_limit; 227 228 if (mc->real_vram_size < mc->visible_vram_size) 229 mc->visible_vram_size = mc->real_vram_size; 230 231 if (mc->xgmi.num_physical_nodes == 0) { 232 mc->fb_start = mc->vram_start; 233 mc->fb_end = mc->vram_end; 234 } 235 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 236 mc->mc_vram_size >> 20, mc->vram_start, 237 mc->vram_end, mc->real_vram_size >> 20); 238 } 239 240 /** amdgpu_gmc_sysvm_location - place vram and gart in sysvm aperture 241 * 242 * @adev: amdgpu device structure holding all necessary information 243 * @mc: memory controller structure holding memory information 244 * 245 * This function is only used if use GART for FB translation. In such 246 * case, we use sysvm aperture (vmid0 page tables) for both vram 247 * and gart (aka system memory) access. 248 * 249 * GPUVM (and our organization of vmid0 page tables) require sysvm 250 * aperture to be placed at a location aligned with 8 times of native 251 * page size. For example, if vm_context0_cntl.page_table_block_size 252 * is 12, then native page size is 8G (2M*2^12), sysvm should start 253 * with a 64G aligned address. For simplicity, we just put sysvm at 254 * address 0. So vram start at address 0 and gart is right after vram. 255 */ 256 void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 257 { 258 u64 hive_vram_start = 0; 259 u64 hive_vram_end = mc->xgmi.node_segment_size * mc->xgmi.num_physical_nodes - 1; 260 mc->vram_start = mc->xgmi.node_segment_size * mc->xgmi.physical_node_id; 261 mc->vram_end = mc->vram_start + mc->xgmi.node_segment_size - 1; 262 /* node_segment_size may not 4GB aligned on SRIOV, align up is needed. */ 263 mc->gart_start = ALIGN(hive_vram_end + 1, four_gb); 264 mc->gart_end = mc->gart_start + mc->gart_size - 1; 265 if (amdgpu_virt_xgmi_migrate_enabled(adev)) { 266 /* set mc->vram_start to 0 to switch the returned GPU address of 267 * amdgpu_bo_create_reserved() from FB aperture to GART aperture. 268 */ 269 mc->vram_start = 0; 270 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 271 mc->visible_vram_size = min(mc->visible_vram_size, mc->real_vram_size); 272 } else { 273 mc->fb_start = hive_vram_start; 274 mc->fb_end = hive_vram_end; 275 } 276 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 277 mc->mc_vram_size >> 20, mc->vram_start, 278 mc->vram_end, mc->real_vram_size >> 20); 279 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 280 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 281 } 282 283 void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size) 284 { 285 if (amdgpu_gart_size == -1) 286 adev->gmc.gart_size = 287 default_size + adev->pm.smu_prv_buffer_size; 288 else 289 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 290 } 291 292 /** 293 * amdgpu_gmc_gart_location - try to find GART location 294 * 295 * @adev: amdgpu device structure holding all necessary information 296 * @mc: memory controller structure holding memory information 297 * @gart_placement: GART placement policy with respect to VRAM 298 * 299 * Function will try to place GART before or after VRAM. 300 * If GART size is bigger than space left then we ajust GART size. 301 * Thus function will never fails. 302 */ 303 void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc, 304 enum amdgpu_gart_placement gart_placement) 305 { 306 u64 size_af, size_bf; 307 /*To avoid the hole, limit the max mc address to AMDGPU_GMC_HOLE_START*/ 308 u64 max_mc_address = min(adev->gmc.mc_mask, AMDGPU_GMC_HOLE_START - 1); 309 310 /* VCE doesn't like it when BOs cross a 4GB segment, so align 311 * the GART base on a 4GB boundary as well. 312 */ 313 size_bf = mc->fb_start; 314 size_af = max_mc_address + 1 - ALIGN(mc->fb_end + 1, four_gb); 315 316 if (mc->gart_size > max(size_bf, size_af)) { 317 dev_warn(adev->dev, "limiting GART\n"); 318 mc->gart_size = max(size_bf, size_af); 319 } 320 321 switch (gart_placement) { 322 case AMDGPU_GART_PLACEMENT_HIGH: 323 mc->gart_start = max_mc_address - mc->gart_size + 1; 324 break; 325 case AMDGPU_GART_PLACEMENT_LOW: 326 if (size_bf >= mc->gart_size) 327 mc->gart_start = 0; 328 else 329 mc->gart_start = ALIGN(mc->fb_end, four_gb); 330 break; 331 case AMDGPU_GART_PLACEMENT_BEST_FIT: 332 default: 333 if ((size_bf >= mc->gart_size && size_bf < size_af) || 334 (size_af < mc->gart_size)) 335 mc->gart_start = 0; 336 else 337 mc->gart_start = max_mc_address - mc->gart_size + 1; 338 break; 339 } 340 341 mc->gart_start &= ~(four_gb - 1); 342 mc->gart_end = mc->gart_start + mc->gart_size - 1; 343 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 344 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 345 } 346 347 /** 348 * amdgpu_gmc_agp_location - try to find AGP location 349 * @adev: amdgpu device structure holding all necessary information 350 * @mc: memory controller structure holding memory information 351 * 352 * Function will place try to find a place for the AGP BAR in the MC address 353 * space. 354 * 355 * AGP BAR will be assigned the largest available hole in the address space. 356 * Should be called after VRAM and GART locations are setup. 357 */ 358 void amdgpu_gmc_agp_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc) 359 { 360 const uint64_t sixteen_gb = 1ULL << 34; 361 const uint64_t sixteen_gb_mask = ~(sixteen_gb - 1); 362 u64 size_af, size_bf; 363 364 if (mc->fb_start > mc->gart_start) { 365 size_bf = (mc->fb_start & sixteen_gb_mask) - 366 ALIGN(mc->gart_end + 1, sixteen_gb); 367 size_af = mc->mc_mask + 1 - ALIGN(mc->fb_end + 1, sixteen_gb); 368 } else { 369 size_bf = mc->fb_start & sixteen_gb_mask; 370 size_af = (mc->gart_start & sixteen_gb_mask) - 371 ALIGN(mc->fb_end + 1, sixteen_gb); 372 } 373 374 if (size_bf > size_af) { 375 mc->agp_start = (mc->fb_start - size_bf) & sixteen_gb_mask; 376 mc->agp_size = size_bf; 377 } else { 378 mc->agp_start = ALIGN(mc->fb_end + 1, sixteen_gb); 379 mc->agp_size = size_af; 380 } 381 382 mc->agp_end = mc->agp_start + mc->agp_size - 1; 383 dev_info(adev->dev, "AGP: %lluM 0x%016llX - 0x%016llX\n", 384 mc->agp_size >> 20, mc->agp_start, mc->agp_end); 385 } 386 387 /** 388 * amdgpu_gmc_set_agp_default - Set the default AGP aperture value. 389 * @adev: amdgpu device structure holding all necessary information 390 * @mc: memory controller structure holding memory information 391 * 392 * To disable the AGP aperture, you need to set the start to a larger 393 * value than the end. This function sets the default value which 394 * can then be overridden using amdgpu_gmc_agp_location() if you want 395 * to enable the AGP aperture on a specific chip. 396 * 397 */ 398 void amdgpu_gmc_set_agp_default(struct amdgpu_device *adev, 399 struct amdgpu_gmc *mc) 400 { 401 mc->agp_start = 0xffffffffffff; 402 mc->agp_end = 0; 403 mc->agp_size = 0; 404 } 405 406 /** 407 * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid 408 * 409 * @addr: 48 bit physical address, page aligned (36 significant bits) 410 * @pasid: 16 bit process address space identifier 411 */ 412 static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid) 413 { 414 return addr << 4 | pasid; 415 } 416 417 /** 418 * amdgpu_gmc_filter_faults - filter VM faults 419 * 420 * @adev: amdgpu device structure 421 * @ih: interrupt ring that the fault received from 422 * @addr: address of the VM fault 423 * @pasid: PASID of the process causing the fault 424 * @timestamp: timestamp of the fault 425 * 426 * Returns: 427 * True if the fault was filtered and should not be processed further. 428 * False if the fault is a new one and needs to be handled. 429 */ 430 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, 431 struct amdgpu_ih_ring *ih, uint64_t addr, 432 uint16_t pasid, uint64_t timestamp) 433 { 434 struct amdgpu_gmc *gmc = &adev->gmc; 435 uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid); 436 struct amdgpu_gmc_fault *fault; 437 uint32_t hash; 438 439 /* Stale retry fault if timestamp goes backward */ 440 if (amdgpu_ih_ts_after(timestamp, ih->processed_timestamp)) 441 return true; 442 443 /* If we don't have space left in the ring buffer return immediately */ 444 stamp = max(timestamp, AMDGPU_GMC_FAULT_TIMEOUT + 1) - 445 AMDGPU_GMC_FAULT_TIMEOUT; 446 if (gmc->fault_ring[gmc->last_fault].timestamp >= stamp) 447 return true; 448 449 /* Try to find the fault in the hash */ 450 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 451 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 452 while (fault->timestamp >= stamp) { 453 uint64_t tmp; 454 455 if (atomic64_read(&fault->key) == key) { 456 /* 457 * if we get a fault which is already present in 458 * the fault_ring and the timestamp of 459 * the fault is after the expired timestamp, 460 * then this is a new fault that needs to be added 461 * into the fault ring. 462 */ 463 if (fault->timestamp_expiry != 0 && 464 amdgpu_ih_ts_after(fault->timestamp_expiry, 465 timestamp)) 466 break; 467 else 468 return true; 469 } 470 471 tmp = fault->timestamp; 472 fault = &gmc->fault_ring[fault->next]; 473 474 /* Check if the entry was reused */ 475 if (fault->timestamp >= tmp) 476 break; 477 } 478 479 /* Add the fault to the ring */ 480 fault = &gmc->fault_ring[gmc->last_fault]; 481 atomic64_set(&fault->key, key); 482 fault->timestamp = timestamp; 483 484 /* And update the hash */ 485 fault->next = gmc->fault_hash[hash].idx; 486 gmc->fault_hash[hash].idx = gmc->last_fault++; 487 return false; 488 } 489 490 /** 491 * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter 492 * 493 * @adev: amdgpu device structure 494 * @addr: address of the VM fault 495 * @pasid: PASID of the process causing the fault 496 * 497 * Remove the address from fault filter, then future vm fault on this address 498 * will pass to retry fault handler to recover. 499 */ 500 void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr, 501 uint16_t pasid) 502 { 503 struct amdgpu_gmc *gmc = &adev->gmc; 504 uint64_t key = amdgpu_gmc_fault_key(addr, pasid); 505 struct amdgpu_ih_ring *ih; 506 struct amdgpu_gmc_fault *fault; 507 uint32_t last_wptr; 508 uint64_t last_ts; 509 uint32_t hash; 510 uint64_t tmp; 511 512 if (adev->irq.retry_cam_enabled) 513 return; 514 else if (adev->irq.ih1.ring_size) 515 ih = &adev->irq.ih1; 516 else if (adev->irq.ih_soft.enabled) 517 ih = &adev->irq.ih_soft; 518 else 519 return; 520 521 /* Get the WPTR of the last entry in IH ring */ 522 last_wptr = amdgpu_ih_get_wptr(adev, ih); 523 /* Order wptr with ring data. */ 524 rmb(); 525 /* Get the timetamp of the last entry in IH ring */ 526 last_ts = amdgpu_ih_decode_iv_ts(adev, ih, last_wptr, -1); 527 528 hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER); 529 fault = &gmc->fault_ring[gmc->fault_hash[hash].idx]; 530 do { 531 if (atomic64_read(&fault->key) == key) { 532 /* 533 * Update the timestamp when this fault 534 * expired. 535 */ 536 fault->timestamp_expiry = last_ts; 537 break; 538 } 539 540 tmp = fault->timestamp; 541 fault = &gmc->fault_ring[fault->next]; 542 } while (fault->timestamp < tmp); 543 } 544 545 int amdgpu_gmc_handle_retry_fault(struct amdgpu_device *adev, 546 struct amdgpu_iv_entry *entry, 547 u64 addr, 548 u32 cam_index, 549 u32 node_id, 550 bool write_fault) 551 { 552 int ret; 553 554 if (adev->irq.retry_cam_enabled) { 555 /* Delegate it to a different ring if the hardware hasn't 556 * already done it. 557 */ 558 if (entry->ih == &adev->irq.ih) { 559 amdgpu_irq_delegate(adev, entry, 8); 560 return 1; 561 } 562 563 ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 564 addr, entry->timestamp, write_fault); 565 WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); 566 if (ret) 567 return 1; 568 } else { 569 /* Process it only if it's the first fault for this address */ 570 if (entry->ih != &adev->irq.ih_soft && 571 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 572 entry->timestamp)) 573 return 1; 574 575 /* Delegate it to a different ring if the hardware hasn't 576 * already done it. 577 */ 578 if (entry->ih == &adev->irq.ih) { 579 amdgpu_irq_delegate(adev, entry, 8); 580 return 1; 581 } 582 583 /* Try to handle the recoverable page faults by filling page 584 * tables 585 */ 586 if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 587 addr, entry->timestamp, write_fault)) 588 return 1; 589 } 590 return 0; 591 } 592 593 int amdgpu_gmc_ras_sw_init(struct amdgpu_device *adev) 594 { 595 int r; 596 597 /* umc ras block */ 598 r = amdgpu_umc_ras_sw_init(adev); 599 if (r) 600 return r; 601 602 /* mmhub ras block */ 603 r = amdgpu_mmhub_ras_sw_init(adev); 604 if (r) 605 return r; 606 607 /* hdp ras block */ 608 r = amdgpu_hdp_ras_sw_init(adev); 609 if (r) 610 return r; 611 612 /* mca.x ras block */ 613 r = amdgpu_mca_mp0_ras_sw_init(adev); 614 if (r) 615 return r; 616 617 r = amdgpu_mca_mp1_ras_sw_init(adev); 618 if (r) 619 return r; 620 621 r = amdgpu_mca_mpio_ras_sw_init(adev); 622 if (r) 623 return r; 624 625 /* xgmi ras block */ 626 r = amdgpu_xgmi_ras_sw_init(adev); 627 if (r) 628 return r; 629 630 return 0; 631 } 632 633 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev) 634 { 635 return 0; 636 } 637 638 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev) 639 { 640 641 } 642 643 /* 644 * The latest engine allocation on gfx9/10 is: 645 * Engine 2, 3: firmware 646 * Engine 0, 1, 4~16: amdgpu ring, 647 * subject to change when ring number changes 648 * Engine 17: Gart flushes 649 */ 650 #define AMDGPU_VMHUB_INV_ENG_BITMAP 0x1FFF3 651 652 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) 653 { 654 struct amdgpu_ring *ring; 655 unsigned vm_inv_engs[AMDGPU_MAX_VMHUBS] = {0}; 656 unsigned i; 657 unsigned vmhub, inv_eng; 658 struct amdgpu_ring *shared_ring; 659 660 /* init the vm inv eng for all vmhubs */ 661 for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 662 vm_inv_engs[i] = AMDGPU_VMHUB_INV_ENG_BITMAP; 663 /* reserve engine 5 for firmware */ 664 if (adev->enable_mes) 665 vm_inv_engs[i] &= ~(1 << 5); 666 /* reserve engine 6 for uni mes */ 667 if (adev->enable_uni_mes) 668 vm_inv_engs[i] &= ~(1 << 6); 669 /* reserve mmhub engine 3 for firmware */ 670 if (adev->enable_umsch_mm) 671 vm_inv_engs[i] &= ~(1 << 3); 672 } 673 674 for (i = 0; i < adev->num_rings; ++i) { 675 ring = adev->rings[i]; 676 vmhub = ring->vm_hub; 677 678 if (ring == &adev->mes.ring[0] || 679 ring == &adev->mes.ring[1] || 680 ring == &adev->umsch_mm.ring || 681 ring == &adev->cper.ring_buf) 682 continue; 683 684 /* Skip if the ring is a shared ring */ 685 if (amdgpu_sdma_is_shared_inv_eng(adev, ring)) 686 continue; 687 688 inv_eng = ffs(vm_inv_engs[vmhub]); 689 if (!inv_eng) { 690 dev_err(adev->dev, "no VM inv eng for ring %s\n", 691 ring->name); 692 return -EINVAL; 693 } 694 695 ring->vm_inv_eng = inv_eng - 1; 696 vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng); 697 698 dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n", 699 ring->name, ring->vm_inv_eng, ring->vm_hub); 700 /* SDMA has a special packet which allows it to use the same 701 * invalidation engine for all the rings in one instance. 702 * Therefore, we do not allocate a separate VM invalidation engine 703 * for SDMA page rings. Instead, they share the VM invalidation 704 * engine with the SDMA gfx ring. This change ensures efficient 705 * resource management and avoids the issue of insufficient VM 706 * invalidation engines. 707 */ 708 shared_ring = amdgpu_sdma_get_shared_ring(adev, ring); 709 if (shared_ring) { 710 shared_ring->vm_inv_eng = ring->vm_inv_eng; 711 dev_info(adev->dev, "ring %s shares VM invalidation engine %u with ring %s on hub %u\n", 712 ring->name, ring->vm_inv_eng, shared_ring->name, ring->vm_hub); 713 continue; 714 } 715 } 716 717 return 0; 718 } 719 720 void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 721 uint32_t vmhub, uint32_t flush_type) 722 { 723 struct amdgpu_ring *ring; 724 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 725 struct dma_fence *fence; 726 struct amdgpu_job *job; 727 int r; 728 729 ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]); 730 731 if (!hub->sdma_invalidation_workaround || vmid || 732 !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready || 733 !ring->sched.ready) { 734 /* 735 * A GPU reset should flush all TLBs anyway, so no need to do 736 * this while one is ongoing. 737 */ 738 if (!down_read_trylock(&adev->reset_domain->sem)) 739 return; 740 741 if (adev->gmc.flush_tlb_needs_extra_type_2) 742 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 743 vmhub, 2); 744 745 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 746 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, 747 vmhub, 0); 748 749 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub, 750 flush_type); 751 up_read(&adev->reset_domain->sem); 752 return; 753 } 754 755 /* The SDMA on Navi 1x has a bug which can theoretically result in memory 756 * corruption if an invalidation happens at the same time as an VA 757 * translation. Avoid this by doing the invalidation from the SDMA 758 * itself at least for GART. 759 */ 760 mutex_lock(&adev->mman.default_entity.lock); 761 r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.default_entity.base, 762 AMDGPU_FENCE_OWNER_UNDEFINED, 763 16 * 4, AMDGPU_IB_POOL_IMMEDIATE, 764 &job, AMDGPU_KERNEL_JOB_ID_FLUSH_GPU_TLB); 765 if (r) 766 goto error_alloc; 767 768 job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo); 769 job->vm_needs_flush = true; 770 job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop; 771 amdgpu_ring_pad_ib(ring, &job->ibs[0]); 772 fence = amdgpu_job_submit(job); 773 mutex_unlock(&adev->mman.default_entity.lock); 774 775 dma_fence_wait(fence, false); 776 dma_fence_put(fence); 777 778 return; 779 780 error_alloc: 781 mutex_unlock(&adev->mman.default_entity.lock); 782 dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r); 783 } 784 785 int amdgpu_gmc_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, 786 uint32_t flush_type, bool all_hub, 787 uint32_t inst) 788 { 789 struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; 790 struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; 791 unsigned int ndw; 792 int r, cnt = 0; 793 uint32_t seq; 794 795 /* 796 * A GPU reset should flush all TLBs anyway, so no need to do 797 * this while one is ongoing. 798 */ 799 if (!down_read_trylock(&adev->reset_domain->sem)) 800 return 0; 801 802 if (!adev->gmc.flush_pasid_uses_kiq || !ring->sched.ready) { 803 804 if (!adev->gmc.gmc_funcs->flush_gpu_tlb_pasid) { 805 r = 0; 806 goto error_unlock_reset; 807 } 808 809 if (adev->gmc.flush_tlb_needs_extra_type_2) 810 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 811 2, all_hub, 812 inst); 813 814 if (adev->gmc.flush_tlb_needs_extra_type_0 && flush_type == 2) 815 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 816 0, all_hub, 817 inst); 818 819 adev->gmc.gmc_funcs->flush_gpu_tlb_pasid(adev, pasid, 820 flush_type, all_hub, 821 inst); 822 r = 0; 823 } else { 824 /* 2 dwords flush + 8 dwords fence */ 825 ndw = kiq->pmf->invalidate_tlbs_size + 8; 826 827 if (adev->gmc.flush_tlb_needs_extra_type_2) 828 ndw += kiq->pmf->invalidate_tlbs_size; 829 830 if (adev->gmc.flush_tlb_needs_extra_type_0) 831 ndw += kiq->pmf->invalidate_tlbs_size; 832 833 spin_lock(&adev->gfx.kiq[inst].ring_lock); 834 r = amdgpu_ring_alloc(ring, ndw); 835 if (r) { 836 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 837 goto error_unlock_reset; 838 } 839 if (adev->gmc.flush_tlb_needs_extra_type_2) 840 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 2, all_hub); 841 842 if (flush_type == 2 && adev->gmc.flush_tlb_needs_extra_type_0) 843 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, 0, all_hub); 844 845 kiq->pmf->kiq_invalidate_tlbs(ring, pasid, flush_type, all_hub); 846 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 847 if (r) { 848 amdgpu_ring_undo(ring); 849 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 850 goto error_unlock_reset; 851 } 852 853 amdgpu_ring_commit(ring); 854 spin_unlock(&adev->gfx.kiq[inst].ring_lock); 855 856 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 857 858 might_sleep(); 859 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 860 !amdgpu_reset_pending(adev->reset_domain)) { 861 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 862 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 863 } 864 865 if (cnt > MAX_KIQ_REG_TRY) { 866 dev_err(adev->dev, "timeout waiting for kiq fence\n"); 867 r = -ETIME; 868 } else 869 r = 0; 870 } 871 872 error_unlock_reset: 873 up_read(&adev->reset_domain->sem); 874 return r; 875 } 876 877 void amdgpu_gmc_fw_reg_write_reg_wait(struct amdgpu_device *adev, 878 uint32_t reg0, uint32_t reg1, 879 uint32_t ref, uint32_t mask, 880 uint32_t xcc_inst) 881 { 882 struct amdgpu_kiq *kiq = &adev->gfx.kiq[xcc_inst]; 883 struct amdgpu_ring *ring = &kiq->ring; 884 signed long r, cnt = 0; 885 unsigned long flags; 886 uint32_t seq; 887 888 if (adev->mes.ring[MES_PIPE_INST(xcc_inst, 0)].sched.ready) { 889 amdgpu_mes_reg_write_reg_wait(adev, reg0, reg1, 890 ref, mask, xcc_inst); 891 return; 892 } 893 894 spin_lock_irqsave(&kiq->ring_lock, flags); 895 amdgpu_ring_alloc(ring, 32); 896 amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 897 ref, mask); 898 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 899 if (r) 900 goto failed_undo; 901 902 amdgpu_ring_commit(ring); 903 spin_unlock_irqrestore(&kiq->ring_lock, flags); 904 905 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 906 907 /* don't wait anymore for IRQ context */ 908 if (r < 1 && in_interrupt()) 909 goto failed_kiq; 910 911 might_sleep(); 912 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY && 913 !amdgpu_reset_pending(adev->reset_domain)) { 914 915 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 916 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 917 } 918 919 if (cnt > MAX_KIQ_REG_TRY) 920 goto failed_kiq; 921 922 return; 923 924 failed_undo: 925 amdgpu_ring_undo(ring); 926 spin_unlock_irqrestore(&kiq->ring_lock, flags); 927 failed_kiq: 928 dev_err(adev->dev, "failed to write reg %x wait reg %x\n", reg0, reg1); 929 } 930 931 /** 932 * amdgpu_gmc_tmz_set -- check and set if a device supports TMZ 933 * @adev: amdgpu_device pointer 934 * 935 * Check and set if an the device @adev supports Trusted Memory 936 * Zones (TMZ). 937 */ 938 void amdgpu_gmc_tmz_set(struct amdgpu_device *adev) 939 { 940 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 941 /* RAVEN */ 942 case IP_VERSION(9, 2, 2): 943 case IP_VERSION(9, 1, 0): 944 /* RENOIR looks like RAVEN */ 945 case IP_VERSION(9, 3, 0): 946 /* GC 10.3.7 */ 947 case IP_VERSION(10, 3, 7): 948 /* GC 11.0.1 */ 949 case IP_VERSION(11, 0, 1): 950 if (amdgpu_tmz == 0) { 951 adev->gmc.tmz_enabled = false; 952 dev_info(adev->dev, 953 "Trusted Memory Zone (TMZ) feature disabled (cmd line)\n"); 954 } else { 955 adev->gmc.tmz_enabled = true; 956 dev_info(adev->dev, 957 "Trusted Memory Zone (TMZ) feature enabled\n"); 958 } 959 break; 960 case IP_VERSION(10, 1, 10): 961 case IP_VERSION(10, 1, 1): 962 case IP_VERSION(10, 1, 2): 963 case IP_VERSION(10, 1, 3): 964 case IP_VERSION(10, 3, 0): 965 case IP_VERSION(10, 3, 2): 966 case IP_VERSION(10, 3, 4): 967 case IP_VERSION(10, 3, 5): 968 case IP_VERSION(10, 3, 6): 969 /* VANGOGH */ 970 case IP_VERSION(10, 3, 1): 971 /* YELLOW_CARP*/ 972 case IP_VERSION(10, 3, 3): 973 case IP_VERSION(11, 0, 4): 974 case IP_VERSION(11, 5, 0): 975 case IP_VERSION(11, 5, 1): 976 case IP_VERSION(11, 5, 2): 977 case IP_VERSION(11, 5, 3): 978 case IP_VERSION(11, 5, 4): 979 /* Don't enable it by default yet. 980 */ 981 if (amdgpu_tmz < 1) { 982 adev->gmc.tmz_enabled = false; 983 dev_info(adev->dev, 984 "Trusted Memory Zone (TMZ) feature disabled as experimental (default)\n"); 985 } else { 986 adev->gmc.tmz_enabled = true; 987 dev_info(adev->dev, 988 "Trusted Memory Zone (TMZ) feature enabled as experimental (cmd line)\n"); 989 } 990 break; 991 default: 992 adev->gmc.tmz_enabled = false; 993 dev_info(adev->dev, 994 "Trusted Memory Zone (TMZ) feature not supported\n"); 995 break; 996 } 997 } 998 999 /** 1000 * amdgpu_gmc_noretry_set -- set per asic noretry defaults 1001 * @adev: amdgpu_device pointer 1002 * 1003 * Set a per asic default for the no-retry parameter. 1004 * 1005 */ 1006 void amdgpu_gmc_noretry_set(struct amdgpu_device *adev) 1007 { 1008 struct amdgpu_gmc *gmc = &adev->gmc; 1009 uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); 1010 bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) || 1011 gc_ver == IP_VERSION(9, 4, 0) || 1012 gc_ver == IP_VERSION(9, 4, 1) || 1013 gc_ver == IP_VERSION(9, 4, 2) || 1014 gc_ver == IP_VERSION(9, 4, 3) || 1015 gc_ver == IP_VERSION(9, 4, 4) || 1016 gc_ver == IP_VERSION(9, 5, 0) || 1017 gc_ver >= IP_VERSION(10, 3, 0)); 1018 1019 /* For GFX12.1 B0, set xnack (retry) on as default */ 1020 if (gc_ver == IP_VERSION(12, 1, 0) && (adev->rev_id & 0xf) == 0x1) 1021 noretry_default = false; 1022 if (!amdgpu_sriov_xnack_support(adev)) 1023 gmc->noretry = 1; 1024 else 1025 gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry; 1026 } 1027 1028 void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type, 1029 bool enable) 1030 { 1031 struct amdgpu_vmhub *hub; 1032 u32 tmp, reg, i; 1033 1034 hub = &adev->vmhub[hub_type]; 1035 for (i = 0; i < 16; i++) { 1036 reg = hub->vm_context0_cntl + hub->ctx_distance * i; 1037 1038 tmp = (hub_type == AMDGPU_GFXHUB(0)) ? 1039 RREG32_SOC15_IP(GC, reg) : 1040 RREG32_SOC15_IP(MMHUB, reg); 1041 1042 if (enable) 1043 tmp |= hub->vm_cntx_cntl_vm_fault; 1044 else 1045 tmp &= ~hub->vm_cntx_cntl_vm_fault; 1046 1047 (hub_type == AMDGPU_GFXHUB(0)) ? 1048 WREG32_SOC15_IP(GC, reg, tmp) : 1049 WREG32_SOC15_IP(MMHUB, reg, tmp); 1050 } 1051 } 1052 1053 void amdgpu_gmc_init_vga_resv_regions(struct amdgpu_device *adev) 1054 { 1055 unsigned size; 1056 1057 if (adev->gmc.is_app_apu) 1058 return; 1059 1060 /* 1061 * Some ASICs need to reserve a region of video memory to avoid access 1062 * from driver 1063 */ 1064 /* 1065 * TODO: 1066 * Currently there is a bug where some memory client outside 1067 * of the driver writes to first 8M of VRAM on S3 resume, 1068 * this overrides GART which by default gets placed in first 8M and 1069 * causes VM_FAULTS once GTT is accessed. 1070 * Keep the stolen memory reservation until the while this is not solved. 1071 */ 1072 switch (adev->asic_type) { 1073 case CHIP_VEGA10: 1074 adev->mman.keep_stolen_vga_memory = true; 1075 /* 1076 * VEGA10 SRIOV VF with MS_HYPERV host needs some firmware reserved area. 1077 */ 1078 #ifdef CONFIG_X86 1079 if (amdgpu_sriov_vf(adev) && hypervisor_is_type(X86_HYPER_MS_HYPERV)) { 1080 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_RESERVED, 1081 0x500000, 0x200000, false); 1082 } 1083 #endif 1084 break; 1085 case CHIP_RAVEN: 1086 case CHIP_RENOIR: 1087 adev->mman.keep_stolen_vga_memory = true; 1088 break; 1089 case CHIP_POLARIS10: 1090 case CHIP_POLARIS11: 1091 case CHIP_POLARIS12: 1092 /* MacBookPros with switchable graphics put VRAM at 0 when 1093 * the iGPU is enabled which results in cursor issues if 1094 * the cursor ends up at 0. Reserve vram at 0 in that case. 1095 */ 1096 if (adev->gmc.vram_start == 0) 1097 adev->mman.keep_stolen_vga_memory = true; 1098 break; 1099 default: 1100 adev->mman.keep_stolen_vga_memory = false; 1101 break; 1102 } 1103 1104 if (amdgpu_sriov_vf(adev) || 1105 !amdgpu_device_has_display_hardware(adev)) { 1106 size = 0; 1107 } else { 1108 size = amdgpu_gmc_get_vbios_fb_size(adev); 1109 1110 if (adev->mman.keep_stolen_vga_memory) 1111 size = max(size, (unsigned)AMDGPU_VBIOS_VGA_ALLOCATION); 1112 } 1113 1114 /* set to 0 if the pre-OS buffer uses up most of vram */ 1115 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 1116 size = 0; 1117 1118 if (size > AMDGPU_VBIOS_VGA_ALLOCATION) { 1119 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1120 0, AMDGPU_VBIOS_VGA_ALLOCATION, false); 1121 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_EXTENDED, 1122 AMDGPU_VBIOS_VGA_ALLOCATION, 1123 size - AMDGPU_VBIOS_VGA_ALLOCATION, false); 1124 } else { 1125 amdgpu_ttm_init_vram_resv(adev, AMDGPU_RESV_STOLEN_VGA, 1126 0, size, false); 1127 } 1128 } 1129 1130 /** 1131 * amdgpu_gmc_init_pdb0 - initialize PDB0 1132 * 1133 * @adev: amdgpu_device pointer 1134 * 1135 * This function is only used when GART page table is used 1136 * for FB address translatioin. In such a case, we construct 1137 * a 2-level system VM page table: PDB0->PTB, to cover both 1138 * VRAM of the hive and system memory. 1139 * 1140 * PDB0 is static, initialized once on driver initialization. 1141 * The first n entries of PDB0 are used as PTE by setting 1142 * P bit to 1, pointing to VRAM. The n+1'th entry points 1143 * to a big PTB covering system memory. 1144 * 1145 */ 1146 void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev) 1147 { 1148 int i; 1149 uint64_t flags = adev->gart.gart_pte_flags; //TODO it is UC. explore NC/RW? 1150 /* Each PDE0 (used as PTE) covers (2^vmid0_page_table_block_size)*2M 1151 */ 1152 u64 vram_size = adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes; 1153 u64 pde0_page_size = (1ULL<<adev->gmc.vmid0_page_table_block_size)<<21; 1154 u64 vram_addr, vram_end; 1155 u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo); 1156 int idx; 1157 1158 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 1159 return; 1160 1161 flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE; 1162 flags |= AMDGPU_PTE_WRITEABLE; 1163 flags |= AMDGPU_PTE_SNOOPED; 1164 flags |= AMDGPU_PTE_FRAG((adev->gmc.vmid0_page_table_block_size + 9*1)); 1165 flags |= AMDGPU_PDE_PTE_FLAG(adev); 1166 1167 vram_addr = adev->vm_manager.vram_base_offset; 1168 if (!amdgpu_virt_xgmi_migrate_enabled(adev)) 1169 vram_addr -= adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1170 vram_end = vram_addr + vram_size; 1171 1172 /* The first n PDE0 entries are used as PTE, 1173 * pointing to vram 1174 */ 1175 for (i = 0; vram_addr < vram_end; i++, vram_addr += pde0_page_size) 1176 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, vram_addr, flags); 1177 1178 /* The n+1'th PDE0 entry points to a huge 1179 * PTB who has more than 512 entries each 1180 * pointing to a 4K system page 1181 */ 1182 flags = AMDGPU_PTE_VALID; 1183 flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0); 1184 /* Requires gart_ptb_gpu_pa to be 4K aligned */ 1185 amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags); 1186 drm_dev_exit(idx); 1187 } 1188 1189 /** 1190 * amdgpu_gmc_vram_mc2pa - calculate vram buffer's physical address from MC 1191 * address 1192 * 1193 * @adev: amdgpu_device pointer 1194 * @mc_addr: MC address of buffer 1195 */ 1196 uint64_t amdgpu_gmc_vram_mc2pa(struct amdgpu_device *adev, uint64_t mc_addr) 1197 { 1198 return mc_addr - adev->gmc.vram_start + adev->vm_manager.vram_base_offset; 1199 } 1200 1201 /** 1202 * amdgpu_gmc_vram_pa - calculate vram buffer object's physical address from 1203 * GPU's view 1204 * 1205 * @adev: amdgpu_device pointer 1206 * @bo: amdgpu buffer object 1207 */ 1208 uint64_t amdgpu_gmc_vram_pa(struct amdgpu_device *adev, struct amdgpu_bo *bo) 1209 { 1210 return amdgpu_gmc_vram_mc2pa(adev, amdgpu_bo_gpu_offset(bo)); 1211 } 1212 1213 int amdgpu_gmc_vram_checking(struct amdgpu_device *adev) 1214 { 1215 struct amdgpu_bo *vram_bo = NULL; 1216 uint64_t vram_gpu = 0; 1217 void *vram_ptr = NULL; 1218 1219 int ret, size = 0x100000; 1220 uint8_t cptr[10]; 1221 1222 ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, 1223 AMDGPU_GEM_DOMAIN_VRAM, 1224 &vram_bo, 1225 &vram_gpu, 1226 &vram_ptr); 1227 if (ret) 1228 return ret; 1229 1230 memset(vram_ptr, 0x86, size); 1231 memset(cptr, 0x86, 10); 1232 1233 /** 1234 * Check the start, the mid, and the end of the memory if the content of 1235 * each byte is the pattern "0x86". If yes, we suppose the vram bo is 1236 * workable. 1237 * 1238 * Note: If check the each byte of whole 1M bo, it will cost too many 1239 * seconds, so here, we just pick up three parts for emulation. 1240 */ 1241 ret = memcmp(vram_ptr, cptr, 10); 1242 if (ret) { 1243 ret = -EIO; 1244 goto release_buffer; 1245 } 1246 1247 ret = memcmp(vram_ptr + (size / 2), cptr, 10); 1248 if (ret) { 1249 ret = -EIO; 1250 goto release_buffer; 1251 } 1252 1253 ret = memcmp(vram_ptr + size - 10, cptr, 10); 1254 if (ret) { 1255 ret = -EIO; 1256 goto release_buffer; 1257 } 1258 1259 release_buffer: 1260 amdgpu_bo_free_kernel(&vram_bo, &vram_gpu, 1261 &vram_ptr); 1262 1263 return ret; 1264 } 1265 1266 static const char *nps_desc[] = { 1267 [AMDGPU_NPS1_PARTITION_MODE] = "NPS1", 1268 [AMDGPU_NPS2_PARTITION_MODE] = "NPS2", 1269 [AMDGPU_NPS3_PARTITION_MODE] = "NPS3", 1270 [AMDGPU_NPS4_PARTITION_MODE] = "NPS4", 1271 [AMDGPU_NPS6_PARTITION_MODE] = "NPS6", 1272 [AMDGPU_NPS8_PARTITION_MODE] = "NPS8", 1273 }; 1274 1275 static ssize_t available_memory_partition_show(struct device *dev, 1276 struct device_attribute *addr, 1277 char *buf) 1278 { 1279 struct drm_device *ddev = dev_get_drvdata(dev); 1280 struct amdgpu_device *adev = drm_to_adev(ddev); 1281 int size = 0, mode; 1282 char *sep = ""; 1283 1284 for_each_inst(mode, adev->gmc.supported_nps_modes) { 1285 size += sysfs_emit_at(buf, size, "%s%s", sep, nps_desc[mode]); 1286 sep = ", "; 1287 } 1288 size += sysfs_emit_at(buf, size, "\n"); 1289 1290 return size; 1291 } 1292 1293 static ssize_t current_memory_partition_store(struct device *dev, 1294 struct device_attribute *attr, 1295 const char *buf, size_t count) 1296 { 1297 struct drm_device *ddev = dev_get_drvdata(dev); 1298 struct amdgpu_device *adev = drm_to_adev(ddev); 1299 enum amdgpu_memory_partition mode; 1300 struct amdgpu_hive_info *hive; 1301 int i; 1302 1303 mode = UNKNOWN_MEMORY_PARTITION_MODE; 1304 for_each_inst(i, adev->gmc.supported_nps_modes) { 1305 if (!strncasecmp(nps_desc[i], buf, strlen(nps_desc[i]))) { 1306 mode = i; 1307 break; 1308 } 1309 } 1310 1311 if (mode == UNKNOWN_MEMORY_PARTITION_MODE) 1312 return -EINVAL; 1313 1314 if (mode == adev->gmc.gmc_funcs->query_mem_partition_mode(adev)) { 1315 dev_info( 1316 adev->dev, 1317 "requested NPS mode is same as current NPS mode, skipping\n"); 1318 return count; 1319 } 1320 1321 /* If device is part of hive, all devices in the hive should request the 1322 * same mode. Hence store the requested mode in hive. 1323 */ 1324 hive = amdgpu_get_xgmi_hive(adev); 1325 if (hive) { 1326 atomic_set(&hive->requested_nps_mode, mode); 1327 amdgpu_put_xgmi_hive(hive); 1328 } else { 1329 adev->gmc.requested_nps_mode = mode; 1330 } 1331 1332 dev_info( 1333 adev->dev, 1334 "NPS mode change requested, please remove and reload the driver\n"); 1335 1336 return count; 1337 } 1338 1339 static ssize_t current_memory_partition_show( 1340 struct device *dev, struct device_attribute *addr, char *buf) 1341 { 1342 struct drm_device *ddev = dev_get_drvdata(dev); 1343 struct amdgpu_device *adev = drm_to_adev(ddev); 1344 enum amdgpu_memory_partition mode; 1345 1346 /* Only minimal precaution taken to reject requests while in reset */ 1347 if (amdgpu_in_reset(adev)) 1348 return -EPERM; 1349 1350 mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1351 if ((mode >= ARRAY_SIZE(nps_desc)) || 1352 (BIT(mode) & AMDGPU_ALL_NPS_MASK) != BIT(mode)) 1353 return sysfs_emit(buf, "UNKNOWN\n"); 1354 1355 return sysfs_emit(buf, "%s\n", nps_desc[mode]); 1356 } 1357 1358 static DEVICE_ATTR_RW(current_memory_partition); 1359 static DEVICE_ATTR_RO(available_memory_partition); 1360 1361 int amdgpu_gmc_sysfs_init(struct amdgpu_device *adev) 1362 { 1363 bool nps_switch_support; 1364 int r = 0; 1365 1366 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1367 return 0; 1368 1369 nps_switch_support = (hweight32(adev->gmc.supported_nps_modes & 1370 AMDGPU_ALL_NPS_MASK) > 1); 1371 if (!nps_switch_support) 1372 dev_attr_current_memory_partition.attr.mode &= 1373 ~(S_IWUSR | S_IWGRP | S_IWOTH); 1374 else 1375 r = device_create_file(adev->dev, 1376 &dev_attr_available_memory_partition); 1377 1378 if (r) 1379 return r; 1380 1381 return device_create_file(adev->dev, 1382 &dev_attr_current_memory_partition); 1383 } 1384 1385 void amdgpu_gmc_sysfs_fini(struct amdgpu_device *adev) 1386 { 1387 if (!adev->gmc.gmc_funcs->query_mem_partition_mode) 1388 return; 1389 1390 device_remove_file(adev->dev, &dev_attr_current_memory_partition); 1391 device_remove_file(adev->dev, &dev_attr_available_memory_partition); 1392 } 1393 1394 int amdgpu_gmc_get_nps_memranges(struct amdgpu_device *adev, 1395 struct amdgpu_mem_partition_info *mem_ranges, 1396 uint8_t *exp_ranges) 1397 { 1398 struct amdgpu_gmc_memrange ranges[AMDGPU_MAX_MEM_RANGES]; 1399 int range_cnt, ret, i, j; 1400 uint32_t nps_type; 1401 bool refresh; 1402 1403 if (!mem_ranges || !exp_ranges) 1404 return -EINVAL; 1405 range_cnt = AMDGPU_MAX_MEM_RANGES; 1406 refresh = (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) && 1407 (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS); 1408 ret = amdgpu_discovery_get_nps_info(adev, &nps_type, ranges, &range_cnt, 1409 refresh); 1410 1411 if (ret) 1412 return ret; 1413 1414 /* TODO: For now, expect ranges and partition count to be the same. 1415 * Adjust if there are holes expected in any NPS domain. 1416 */ 1417 if (*exp_ranges && (range_cnt != *exp_ranges)) { 1418 dev_warn( 1419 adev->dev, 1420 "NPS config mismatch - expected ranges: %d discovery - nps mode: %d, nps ranges: %d", 1421 *exp_ranges, nps_type, range_cnt); 1422 ret = -EINVAL; 1423 goto err; 1424 } 1425 1426 for (i = 0; i < range_cnt; ++i) { 1427 if (ranges[i].base_address >= ranges[i].limit_address) { 1428 dev_warn( 1429 adev->dev, 1430 "Invalid NPS range - nps mode: %d, range[%d]: base: %llx limit: %llx", 1431 nps_type, i, ranges[i].base_address, 1432 ranges[i].limit_address); 1433 ret = -EINVAL; 1434 goto err; 1435 } 1436 1437 /* Check for overlaps, not expecting any now */ 1438 for (j = i - 1; j >= 0; j--) { 1439 if (max(ranges[j].base_address, 1440 ranges[i].base_address) <= 1441 min(ranges[j].limit_address, 1442 ranges[i].limit_address)) { 1443 dev_warn( 1444 adev->dev, 1445 "overlapping ranges detected [ %llx - %llx ] | [%llx - %llx]", 1446 ranges[j].base_address, 1447 ranges[j].limit_address, 1448 ranges[i].base_address, 1449 ranges[i].limit_address); 1450 ret = -EINVAL; 1451 goto err; 1452 } 1453 } 1454 1455 mem_ranges[i].range.fpfn = 1456 (ranges[i].base_address - 1457 adev->vm_manager.vram_base_offset) >> 1458 AMDGPU_GPU_PAGE_SHIFT; 1459 mem_ranges[i].range.lpfn = 1460 (ranges[i].limit_address - 1461 adev->vm_manager.vram_base_offset) >> 1462 AMDGPU_GPU_PAGE_SHIFT; 1463 mem_ranges[i].size = 1464 ranges[i].limit_address - ranges[i].base_address + 1; 1465 } 1466 1467 if (!*exp_ranges) 1468 *exp_ranges = range_cnt; 1469 err: 1470 return ret; 1471 } 1472 1473 int amdgpu_gmc_request_memory_partition(struct amdgpu_device *adev, 1474 int nps_mode) 1475 { 1476 /* Not supported on VF devices and APUs */ 1477 if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU)) 1478 return -EOPNOTSUPP; 1479 1480 if (!adev->psp.funcs) { 1481 dev_err(adev->dev, 1482 "PSP interface not available for nps mode change request"); 1483 return -EINVAL; 1484 } 1485 1486 return psp_memory_partition(&adev->psp, nps_mode); 1487 } 1488 1489 static inline bool amdgpu_gmc_need_nps_switch_req(struct amdgpu_device *adev, 1490 int req_nps_mode, 1491 int cur_nps_mode) 1492 { 1493 return (((BIT(req_nps_mode) & adev->gmc.supported_nps_modes) == 1494 BIT(req_nps_mode)) && 1495 req_nps_mode != cur_nps_mode); 1496 } 1497 1498 void amdgpu_gmc_prepare_nps_mode_change(struct amdgpu_device *adev) 1499 { 1500 int req_nps_mode, cur_nps_mode, r; 1501 struct amdgpu_hive_info *hive; 1502 1503 if (amdgpu_sriov_vf(adev) || !adev->gmc.supported_nps_modes || 1504 !adev->gmc.gmc_funcs->request_mem_partition_mode) 1505 return; 1506 1507 cur_nps_mode = adev->gmc.gmc_funcs->query_mem_partition_mode(adev); 1508 hive = amdgpu_get_xgmi_hive(adev); 1509 if (hive) { 1510 req_nps_mode = atomic_read(&hive->requested_nps_mode); 1511 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, 1512 cur_nps_mode)) { 1513 amdgpu_put_xgmi_hive(hive); 1514 return; 1515 } 1516 r = amdgpu_xgmi_request_nps_change(adev, hive, req_nps_mode); 1517 amdgpu_put_xgmi_hive(hive); 1518 goto out; 1519 } 1520 1521 req_nps_mode = adev->gmc.requested_nps_mode; 1522 if (!amdgpu_gmc_need_nps_switch_req(adev, req_nps_mode, cur_nps_mode)) 1523 return; 1524 1525 /* even if this fails, we should let driver unload w/o blocking */ 1526 r = adev->gmc.gmc_funcs->request_mem_partition_mode(adev, req_nps_mode); 1527 out: 1528 if (r) 1529 dev_err(adev->dev, "NPS mode change request failed\n"); 1530 else 1531 dev_info( 1532 adev->dev, 1533 "NPS mode change request done, reload driver to complete the change\n"); 1534 } 1535 1536 bool amdgpu_gmc_need_reset_on_init(struct amdgpu_device *adev) 1537 { 1538 if (adev->gmc.gmc_funcs->need_reset_on_init) 1539 return adev->gmc.gmc_funcs->need_reset_on_init(adev); 1540 1541 return false; 1542 } 1543 1544 enum amdgpu_memory_partition 1545 amdgpu_gmc_get_vf_memory_partition(struct amdgpu_device *adev) 1546 { 1547 switch (adev->gmc.num_mem_partitions) { 1548 case 0: 1549 return UNKNOWN_MEMORY_PARTITION_MODE; 1550 case 1: 1551 return AMDGPU_NPS1_PARTITION_MODE; 1552 case 2: 1553 return AMDGPU_NPS2_PARTITION_MODE; 1554 case 4: 1555 return AMDGPU_NPS4_PARTITION_MODE; 1556 case 8: 1557 return AMDGPU_NPS8_PARTITION_MODE; 1558 default: 1559 return AMDGPU_NPS1_PARTITION_MODE; 1560 } 1561 } 1562 1563 enum amdgpu_memory_partition 1564 amdgpu_gmc_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes) 1565 { 1566 enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE; 1567 1568 if (adev->nbio.funcs && 1569 adev->nbio.funcs->get_memory_partition_mode) 1570 mode = adev->nbio.funcs->get_memory_partition_mode(adev, 1571 supp_modes); 1572 else 1573 dev_warn(adev->dev, "memory partition mode query is not supported\n"); 1574 1575 return mode; 1576 } 1577 1578 enum amdgpu_memory_partition 1579 amdgpu_gmc_query_memory_partition(struct amdgpu_device *adev) 1580 { 1581 if (amdgpu_sriov_vf(adev)) 1582 return amdgpu_gmc_get_vf_memory_partition(adev); 1583 else 1584 return amdgpu_gmc_get_memory_partition(adev, NULL); 1585 } 1586 1587 static bool amdgpu_gmc_validate_partition_info(struct amdgpu_device *adev) 1588 { 1589 enum amdgpu_memory_partition mode; 1590 u32 supp_modes; 1591 bool valid; 1592 1593 mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes); 1594 1595 /* Mode detected by hardware not present in supported modes */ 1596 if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && 1597 !(BIT(mode - 1) & supp_modes)) 1598 return false; 1599 1600 switch (mode) { 1601 case UNKNOWN_MEMORY_PARTITION_MODE: 1602 case AMDGPU_NPS1_PARTITION_MODE: 1603 valid = (adev->gmc.num_mem_partitions == 1); 1604 break; 1605 case AMDGPU_NPS2_PARTITION_MODE: 1606 valid = (adev->gmc.num_mem_partitions == 2); 1607 break; 1608 case AMDGPU_NPS4_PARTITION_MODE: 1609 valid = (adev->gmc.num_mem_partitions == 3 || 1610 adev->gmc.num_mem_partitions == 4); 1611 break; 1612 case AMDGPU_NPS8_PARTITION_MODE: 1613 valid = (adev->gmc.num_mem_partitions == 8); 1614 break; 1615 default: 1616 valid = false; 1617 } 1618 1619 return valid; 1620 } 1621 1622 static bool amdgpu_gmc_is_node_present(int *node_ids, int num_ids, int nid) 1623 { 1624 int i; 1625 1626 /* Check if node with id 'nid' is present in 'node_ids' array */ 1627 for (i = 0; i < num_ids; ++i) 1628 if (node_ids[i] == nid) 1629 return true; 1630 1631 return false; 1632 } 1633 1634 static void 1635 amdgpu_gmc_init_acpi_mem_ranges(struct amdgpu_device *adev, 1636 struct amdgpu_mem_partition_info *mem_ranges) 1637 { 1638 struct amdgpu_numa_info numa_info; 1639 int node_ids[AMDGPU_MAX_MEM_RANGES]; 1640 int num_ranges = 0, ret; 1641 int num_xcc, xcc_id; 1642 uint32_t xcc_mask; 1643 1644 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1645 xcc_mask = (1U << num_xcc) - 1; 1646 1647 for_each_inst(xcc_id, xcc_mask) { 1648 ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info); 1649 if (ret) 1650 continue; 1651 1652 if (numa_info.nid == NUMA_NO_NODE) { 1653 mem_ranges[0].size = numa_info.size; 1654 mem_ranges[0].numa.node = numa_info.nid; 1655 num_ranges = 1; 1656 break; 1657 } 1658 1659 if (amdgpu_gmc_is_node_present(node_ids, num_ranges, 1660 numa_info.nid)) 1661 continue; 1662 1663 node_ids[num_ranges] = numa_info.nid; 1664 mem_ranges[num_ranges].numa.node = numa_info.nid; 1665 mem_ranges[num_ranges].size = numa_info.size; 1666 ++num_ranges; 1667 } 1668 1669 adev->gmc.num_mem_partitions = num_ranges; 1670 } 1671 1672 void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev, 1673 struct amdgpu_mem_partition_info *mem_ranges) 1674 { 1675 enum amdgpu_memory_partition mode; 1676 u32 start_addr = 0, size; 1677 int i, r, l; 1678 1679 mode = amdgpu_gmc_query_memory_partition(adev); 1680 1681 switch (mode) { 1682 case UNKNOWN_MEMORY_PARTITION_MODE: 1683 adev->gmc.num_mem_partitions = 0; 1684 break; 1685 case AMDGPU_NPS1_PARTITION_MODE: 1686 adev->gmc.num_mem_partitions = 1; 1687 break; 1688 case AMDGPU_NPS2_PARTITION_MODE: 1689 adev->gmc.num_mem_partitions = 2; 1690 break; 1691 case AMDGPU_NPS4_PARTITION_MODE: 1692 if (adev->flags & AMD_IS_APU) 1693 adev->gmc.num_mem_partitions = 3; 1694 else 1695 adev->gmc.num_mem_partitions = 4; 1696 break; 1697 case AMDGPU_NPS8_PARTITION_MODE: 1698 adev->gmc.num_mem_partitions = 8; 1699 break; 1700 default: 1701 adev->gmc.num_mem_partitions = 1; 1702 break; 1703 } 1704 1705 /* Use NPS range info, if populated */ 1706 r = amdgpu_gmc_get_nps_memranges(adev, mem_ranges, 1707 &adev->gmc.num_mem_partitions); 1708 if (!r) { 1709 l = 0; 1710 for (i = 1; i < adev->gmc.num_mem_partitions; ++i) { 1711 if (mem_ranges[i].range.lpfn > 1712 mem_ranges[i - 1].range.lpfn) 1713 l = i; 1714 } 1715 1716 } else { 1717 if (!adev->gmc.num_mem_partitions) { 1718 dev_warn(adev->dev, 1719 "Not able to detect NPS mode, fall back to NPS1\n"); 1720 adev->gmc.num_mem_partitions = 1; 1721 } 1722 /* Fallback to sw based calculation */ 1723 size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT; 1724 size /= adev->gmc.num_mem_partitions; 1725 1726 for (i = 0; i < adev->gmc.num_mem_partitions; ++i) { 1727 mem_ranges[i].range.fpfn = start_addr; 1728 mem_ranges[i].size = 1729 ((u64)size << AMDGPU_GPU_PAGE_SHIFT); 1730 mem_ranges[i].range.lpfn = start_addr + size - 1; 1731 start_addr += size; 1732 } 1733 1734 l = adev->gmc.num_mem_partitions - 1; 1735 } 1736 1737 /* Adjust the last one */ 1738 mem_ranges[l].range.lpfn = 1739 (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1; 1740 mem_ranges[l].size = 1741 adev->gmc.real_vram_size - 1742 ((u64)mem_ranges[l].range.fpfn << AMDGPU_GPU_PAGE_SHIFT); 1743 } 1744 1745 int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev) 1746 { 1747 bool valid; 1748 1749 adev->gmc.mem_partitions = kzalloc_objs(struct amdgpu_mem_partition_info, 1750 AMDGPU_MAX_MEM_RANGES); 1751 if (!adev->gmc.mem_partitions) 1752 return -ENOMEM; 1753 1754 if (adev->gmc.is_app_apu) 1755 amdgpu_gmc_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions); 1756 else 1757 amdgpu_gmc_init_sw_mem_ranges(adev, adev->gmc.mem_partitions); 1758 1759 if (amdgpu_sriov_vf(adev)) 1760 valid = true; 1761 else 1762 valid = amdgpu_gmc_validate_partition_info(adev); 1763 if (!valid) { 1764 /* TODO: handle invalid case */ 1765 dev_warn(adev->dev, 1766 "Mem ranges not matching with hardware config\n"); 1767 } 1768 1769 return 0; 1770 } 1771 1772 int amdgpu_gmc_get_vram_info(struct amdgpu_device *adev, 1773 int *vram_width, int *vram_type, int *vram_vendor) 1774 { 1775 int ret = 0; 1776 1777 if (adev->flags & AMD_IS_APU) 1778 return amdgpu_atomfirmware_get_integrated_system_info(adev, 1779 vram_width, vram_type, vram_vendor); 1780 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1781 case IP_VERSION(12, 0, 0): 1782 case IP_VERSION(12, 0, 1): 1783 return amdgpu_atomfirmware_get_umc_info(adev, 1784 vram_width, vram_type, vram_vendor); 1785 case IP_VERSION(9, 5, 0): 1786 case IP_VERSION(9, 4, 4): 1787 case IP_VERSION(9, 4, 3): 1788 ret = amdgpu_atomfirmware_get_umc_info(adev, 1789 vram_width, vram_type, vram_vendor); 1790 if (vram_width && !ret) 1791 *vram_width *= hweight32(adev->aid_mask); 1792 return ret; 1793 default: 1794 return amdgpu_atomfirmware_get_vram_info(adev, 1795 vram_width, vram_type, vram_vendor); 1796 } 1797 return 0; 1798 } 1799