1 /* 2 * Copyright 2019 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/firmware.h> 24 #include <linux/pci.h> 25 26 #include <drm/drm_cache.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_ip.h" 30 #include "amdgpu_atomfirmware.h" 31 #include "gmc_v10_0.h" 32 #include "umc_v8_7.h" 33 34 #include "athub/athub_2_0_0_sh_mask.h" 35 #include "athub/athub_2_0_0_offset.h" 36 #include "dcn/dcn_2_0_0_offset.h" 37 #include "dcn/dcn_2_0_0_sh_mask.h" 38 #include "oss/osssys_5_0_0_offset.h" 39 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 40 #include "navi10_enum.h" 41 42 #include "soc15.h" 43 #include "soc15d.h" 44 #include "soc15_common.h" 45 46 #include "nbio_v2_3.h" 47 48 #include "gfxhub_v2_0.h" 49 #include "gfxhub_v2_1.h" 50 #include "mmhub_v2_0.h" 51 #include "mmhub_v2_3.h" 52 #include "athub_v2_0.h" 53 #include "athub_v2_1.h" 54 55 static int gmc_v10_0_ecc_interrupt_state(struct amdgpu_device *adev, 56 struct amdgpu_irq_src *src, 57 unsigned int type, 58 enum amdgpu_interrupt_state state) 59 { 60 return 0; 61 } 62 63 static int 64 gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 65 struct amdgpu_irq_src *src, unsigned int type, 66 enum amdgpu_interrupt_state state) 67 { 68 switch (state) { 69 case AMDGPU_IRQ_STATE_DISABLE: 70 /* MM HUB */ 71 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), false); 72 /* GFX HUB */ 73 /* This works because this interrupt is only 74 * enabled at init/resume and disabled in 75 * fini/suspend, so the overall state doesn't 76 * change over the course of suspend/resume. 77 */ 78 if (!adev->in_s0ix) 79 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), false); 80 break; 81 case AMDGPU_IRQ_STATE_ENABLE: 82 /* MM HUB */ 83 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), true); 84 /* GFX HUB */ 85 /* This works because this interrupt is only 86 * enabled at init/resume and disabled in 87 * fini/suspend, so the overall state doesn't 88 * change over the course of suspend/resume. 89 */ 90 if (!adev->in_s0ix) 91 amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), true); 92 break; 93 default: 94 break; 95 } 96 97 return 0; 98 } 99 100 static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev, 101 struct amdgpu_irq_src *source, 102 struct amdgpu_iv_entry *entry) 103 { 104 uint32_t vmhub_index = entry->client_id == SOC15_IH_CLIENTID_VMC ? 105 AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0); 106 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub_index]; 107 bool retry_fault = !!(entry->src_data[1] & 108 AMDGPU_GMC9_FAULT_SOURCE_DATA_RETRY); 109 bool write_fault = !!(entry->src_data[1] & 110 AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE); 111 struct amdgpu_task_info *task_info; 112 uint32_t status = 0; 113 u64 addr; 114 115 addr = (u64)entry->src_data[0] << 12; 116 addr |= ((u64)entry->src_data[1] & 0xf) << 44; 117 118 if (retry_fault) { 119 int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, 0, 0, 120 write_fault); 121 /* Returning 1 here also prevents sending the IV to the KFD */ 122 if (ret == 1) 123 return 1; 124 } 125 126 if (!amdgpu_sriov_vf(adev)) { 127 /* 128 * Issue a dummy read to wait for the status register to 129 * be updated to avoid reading an incorrect value due to 130 * the new fast GRBM interface. 131 */ 132 if ((entry->vmid_src == AMDGPU_GFXHUB(0)) && 133 (amdgpu_ip_version(adev, GC_HWIP, 0) < 134 IP_VERSION(10, 3, 0))) 135 RREG32(hub->vm_l2_pro_fault_status); 136 137 status = RREG32(hub->vm_l2_pro_fault_status); 138 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 139 140 amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, 141 entry->vmid_src ? AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0)); 142 } 143 144 if (!printk_ratelimit()) 145 return 0; 146 147 dev_err(adev->dev, 148 "[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", 149 entry->vmid_src ? "mmhub" : "gfxhub", 150 entry->src_id, entry->ring_id, entry->vmid, entry->pasid); 151 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid); 152 if (task_info) { 153 amdgpu_vm_print_task_info(adev, task_info); 154 amdgpu_vm_put_task_info(task_info); 155 } 156 157 dev_err(adev->dev, " in page starting at address 0x%016llx from client 0x%x (%s)\n", 158 addr, entry->client_id, 159 soc15_ih_clientid_name[entry->client_id]); 160 161 /* Only print L2 fault status if the status register could be read and 162 * contains useful information 163 */ 164 if (status != 0) 165 hub->vmhub_funcs->print_l2_protection_fault_status(adev, 166 status); 167 168 return 0; 169 } 170 171 static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = { 172 .set = gmc_v10_0_vm_fault_interrupt_state, 173 .process = gmc_v10_0_process_interrupt, 174 }; 175 176 static const struct amdgpu_irq_src_funcs gmc_v10_0_ecc_funcs = { 177 .set = gmc_v10_0_ecc_interrupt_state, 178 .process = amdgpu_umc_process_ecc_irq, 179 }; 180 181 static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev) 182 { 183 adev->gmc.vm_fault.num_types = 1; 184 adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs; 185 186 if (!amdgpu_sriov_vf(adev)) { 187 adev->gmc.ecc_irq.num_types = 1; 188 adev->gmc.ecc_irq.funcs = &gmc_v10_0_ecc_funcs; 189 } 190 } 191 192 /** 193 * gmc_v10_0_use_invalidate_semaphore - judge whether to use semaphore 194 * 195 * @adev: amdgpu_device pointer 196 * @vmhub: vmhub type 197 * 198 */ 199 static bool gmc_v10_0_use_invalidate_semaphore(struct amdgpu_device *adev, 200 uint32_t vmhub) 201 { 202 return ((vmhub == AMDGPU_MMHUB0(0)) && 203 (!amdgpu_sriov_vf(adev))); 204 } 205 206 static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info( 207 struct amdgpu_device *adev, 208 uint8_t vmid, uint16_t *p_pasid) 209 { 210 uint32_t value; 211 212 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 213 + vmid); 214 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 215 216 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 217 } 218 219 /* 220 * GART 221 * VMID 0 is the physical GPU addresses as used by the kernel. 222 * VMIDs 1-15 are used for userspace clients and are handled 223 * by the amdgpu vm/hsa code. 224 */ 225 226 /** 227 * gmc_v10_0_flush_gpu_tlb - gart tlb flush callback 228 * 229 * @adev: amdgpu_device pointer 230 * @vmid: vm instance to flush 231 * @vmhub: vmhub type 232 * @flush_type: the flush type 233 * 234 * Flush the TLB for the requested page table. 235 */ 236 static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 237 uint32_t vmhub, uint32_t flush_type) 238 { 239 bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(adev, vmhub); 240 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 241 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); 242 /* Use register 17 for GART */ 243 const unsigned int eng = 17; 244 unsigned char hub_ip = 0; 245 u32 sem, req, ack; 246 unsigned int i; 247 u32 tmp; 248 249 sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng; 250 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 251 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 252 253 /* flush hdp cache */ 254 amdgpu_device_flush_hdp(adev, NULL); 255 256 /* This is necessary for SRIOV as well as for GFXOFF to function 257 * properly under bare metal 258 */ 259 if (adev->gfx.kiq[0].ring.sched.ready && !adev->enable_mes && 260 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) { 261 amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, 262 1 << vmid, GET_INST(GC, 0)); 263 return; 264 } 265 266 /* This path is needed before KIQ/MES/GFXOFF are set up */ 267 hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ? GC_HWIP : MMHUB_HWIP; 268 269 spin_lock(&adev->gmc.invalidate_lock); 270 /* 271 * It may lose gpuvm invalidate acknowldege state across power-gating 272 * off cycle, add semaphore acquire before invalidation and semaphore 273 * release after invalidation to avoid entering power gated state 274 * to WA the Issue 275 */ 276 277 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 278 if (use_semaphore) { 279 for (i = 0; i < adev->usec_timeout; i++) { 280 /* a read return value of 1 means semaphore acuqire */ 281 tmp = RREG32_RLC_NO_KIQ(sem, hub_ip); 282 if (tmp & 0x1) 283 break; 284 udelay(1); 285 } 286 287 if (i >= adev->usec_timeout) 288 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 289 } 290 291 WREG32_RLC_NO_KIQ(req, inv_req, hub_ip); 292 293 /* 294 * Issue a dummy read to wait for the ACK register to be cleared 295 * to avoid a false ACK due to the new fast GRBM interface. 296 */ 297 if ((vmhub == AMDGPU_GFXHUB(0)) && 298 (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(10, 3, 0))) 299 RREG32_RLC_NO_KIQ(req, hub_ip); 300 301 /* Wait for ACK with a delay.*/ 302 for (i = 0; i < adev->usec_timeout; i++) { 303 tmp = RREG32_RLC_NO_KIQ(ack, hub_ip); 304 tmp &= 1 << vmid; 305 if (tmp) 306 break; 307 308 udelay(1); 309 } 310 311 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 312 if (use_semaphore) 313 WREG32_RLC_NO_KIQ(sem, 0, hub_ip); 314 315 spin_unlock(&adev->gmc.invalidate_lock); 316 317 if (i >= adev->usec_timeout) 318 dev_err(adev->dev, "Timeout waiting for VM flush hub: %d!\n", 319 vmhub); 320 } 321 322 /** 323 * gmc_v10_0_flush_gpu_tlb_pasid - tlb flush via pasid 324 * 325 * @adev: amdgpu_device pointer 326 * @pasid: pasid to be flush 327 * @flush_type: the flush type 328 * @all_hub: Used with PACKET3_INVALIDATE_TLBS_ALL_HUB() 329 * @inst: is used to select which instance of KIQ to use for the invalidation 330 * 331 * Flush the TLB for the requested pasid. 332 */ 333 static void gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 334 uint16_t pasid, uint32_t flush_type, 335 bool all_hub, uint32_t inst) 336 { 337 uint16_t queried; 338 int vmid, i; 339 340 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 341 bool valid; 342 343 valid = gmc_v10_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 344 &queried); 345 if (!valid || queried != pasid) 346 continue; 347 348 if (all_hub) { 349 for_each_set_bit(i, adev->vmhubs_mask, 350 AMDGPU_MAX_VMHUBS) 351 gmc_v10_0_flush_gpu_tlb(adev, vmid, i, 352 flush_type); 353 } else { 354 gmc_v10_0_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0), 355 flush_type); 356 } 357 } 358 } 359 360 static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 361 unsigned int vmid, uint64_t pd_addr) 362 { 363 bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->vm_hub); 364 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; 365 uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); 366 unsigned int eng = ring->vm_inv_eng; 367 368 /* 369 * It may lose gpuvm invalidate acknowldege state across power-gating 370 * off cycle, add semaphore acquire before invalidation and semaphore 371 * release after invalidation to avoid entering power gated state 372 * to WA the Issue 373 */ 374 375 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 376 if (use_semaphore) 377 /* a read return value of 1 means semaphore acuqire */ 378 amdgpu_ring_emit_reg_wait(ring, 379 hub->vm_inv_eng0_sem + 380 hub->eng_distance * eng, 0x1, 0x1); 381 382 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 383 (hub->ctx_addr_distance * vmid), 384 lower_32_bits(pd_addr)); 385 386 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 387 (hub->ctx_addr_distance * vmid), 388 upper_32_bits(pd_addr)); 389 390 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 391 hub->eng_distance * eng, 392 hub->vm_inv_eng0_ack + 393 hub->eng_distance * eng, 394 req, 1 << vmid); 395 396 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 397 if (use_semaphore) 398 /* 399 * add semaphore release after invalidation, 400 * write with 0 means semaphore release 401 */ 402 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 403 hub->eng_distance * eng, 0); 404 405 return pd_addr; 406 } 407 408 static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid, 409 unsigned int pasid) 410 { 411 struct amdgpu_device *adev = ring->adev; 412 uint32_t reg; 413 414 if (ring->vm_hub == AMDGPU_GFXHUB(0)) 415 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 416 else 417 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 418 419 amdgpu_ring_emit_wreg(ring, reg, pasid); 420 } 421 422 /* 423 * PTE format on NAVI 10: 424 * 63:59 reserved 425 * 58 reserved and for sienna_cichlid is used for MALL noalloc 426 * 57 reserved 427 * 56 F 428 * 55 L 429 * 54 reserved 430 * 53:52 SW 431 * 51 T 432 * 50:48 mtype 433 * 47:12 4k physical page base address 434 * 11:7 fragment 435 * 6 write 436 * 5 read 437 * 4 exe 438 * 3 Z 439 * 2 snooped 440 * 1 system 441 * 0 valid 442 * 443 * PDE format on NAVI 10: 444 * 63:59 block fragment size 445 * 58:55 reserved 446 * 54 P 447 * 53:48 reserved 448 * 47:6 physical base address of PD or PTE 449 * 5:3 reserved 450 * 2 C 451 * 1 system 452 * 0 valid 453 */ 454 455 static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level, 456 uint64_t *addr, uint64_t *flags) 457 { 458 if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 459 *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); 460 BUG_ON(*addr & 0xFFFF00000000003FULL); 461 462 if (!adev->gmc.translate_further) 463 return; 464 465 if (level == AMDGPU_VM_PDB1) { 466 /* Set the block fragment size */ 467 if (!(*flags & AMDGPU_PDE_PTE)) 468 *flags |= AMDGPU_PDE_BFS(0x9); 469 470 } else if (level == AMDGPU_VM_PDB0) { 471 if (*flags & AMDGPU_PDE_PTE) 472 *flags &= ~AMDGPU_PDE_PTE; 473 else 474 *flags |= AMDGPU_PTE_TF; 475 } 476 } 477 478 static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev, 479 struct amdgpu_vm *vm, 480 struct amdgpu_bo *bo, 481 uint32_t vm_flags, 482 uint64_t *flags) 483 { 484 if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE) 485 *flags |= AMDGPU_PTE_EXECUTABLE; 486 else 487 *flags &= ~AMDGPU_PTE_EXECUTABLE; 488 489 switch (vm_flags & AMDGPU_VM_MTYPE_MASK) { 490 case AMDGPU_VM_MTYPE_DEFAULT: 491 case AMDGPU_VM_MTYPE_NC: 492 default: 493 *flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_NC); 494 break; 495 case AMDGPU_VM_MTYPE_WC: 496 *flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_WC); 497 break; 498 case AMDGPU_VM_MTYPE_CC: 499 *flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_CC); 500 break; 501 case AMDGPU_VM_MTYPE_UC: 502 *flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC); 503 break; 504 } 505 506 if (vm_flags & AMDGPU_VM_PAGE_NOALLOC) 507 *flags |= AMDGPU_PTE_NOALLOC; 508 else 509 *flags &= ~AMDGPU_PTE_NOALLOC; 510 511 if (vm_flags & AMDGPU_VM_PAGE_PRT) { 512 *flags |= AMDGPU_PTE_PRT; 513 *flags |= AMDGPU_PTE_SNOOPED; 514 *flags |= AMDGPU_PTE_LOG; 515 *flags |= AMDGPU_PTE_SYSTEM; 516 *flags &= ~AMDGPU_PTE_VALID; 517 } 518 519 if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT | 520 AMDGPU_GEM_CREATE_EXT_COHERENT | 521 AMDGPU_GEM_CREATE_UNCACHED)) 522 *flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC); 523 } 524 525 static unsigned int gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev) 526 { 527 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 528 unsigned int size; 529 530 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 531 size = AMDGPU_VBIOS_VGA_ALLOCATION; 532 } else { 533 u32 viewport; 534 u32 pitch; 535 536 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 537 pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH); 538 size = (REG_GET_FIELD(viewport, 539 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 540 REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) * 541 4); 542 } 543 544 return size; 545 } 546 547 static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = { 548 .flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb, 549 .flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid, 550 .emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb, 551 .emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping, 552 .get_vm_pde = gmc_v10_0_get_vm_pde, 553 .get_vm_pte = gmc_v10_0_get_vm_pte, 554 .get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size, 555 }; 556 557 static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev) 558 { 559 if (adev->gmc.gmc_funcs == NULL) 560 adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs; 561 } 562 563 static void gmc_v10_0_set_umc_funcs(struct amdgpu_device *adev) 564 { 565 switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) { 566 case IP_VERSION(8, 7, 0): 567 adev->umc.max_ras_err_cnt_per_query = UMC_V8_7_TOTAL_CHANNEL_NUM; 568 adev->umc.channel_inst_num = UMC_V8_7_CHANNEL_INSTANCE_NUM; 569 adev->umc.umc_inst_num = UMC_V8_7_UMC_INSTANCE_NUM; 570 adev->umc.channel_offs = UMC_V8_7_PER_CHANNEL_OFFSET_SIENNA; 571 adev->umc.retire_unit = 1; 572 adev->umc.channel_idx_tbl = &umc_v8_7_channel_idx_tbl[0][0]; 573 adev->umc.ras = &umc_v8_7_ras; 574 break; 575 default: 576 break; 577 } 578 } 579 580 static void gmc_v10_0_set_mmhub_funcs(struct amdgpu_device *adev) 581 { 582 switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { 583 case IP_VERSION(2, 3, 0): 584 case IP_VERSION(2, 4, 0): 585 case IP_VERSION(2, 4, 1): 586 adev->mmhub.funcs = &mmhub_v2_3_funcs; 587 break; 588 default: 589 adev->mmhub.funcs = &mmhub_v2_0_funcs; 590 break; 591 } 592 } 593 594 static void gmc_v10_0_set_gfxhub_funcs(struct amdgpu_device *adev) 595 { 596 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 597 case IP_VERSION(10, 3, 0): 598 case IP_VERSION(10, 3, 2): 599 case IP_VERSION(10, 3, 1): 600 case IP_VERSION(10, 3, 4): 601 case IP_VERSION(10, 3, 5): 602 case IP_VERSION(10, 3, 6): 603 case IP_VERSION(10, 3, 3): 604 case IP_VERSION(10, 3, 7): 605 adev->gfxhub.funcs = &gfxhub_v2_1_funcs; 606 break; 607 default: 608 adev->gfxhub.funcs = &gfxhub_v2_0_funcs; 609 break; 610 } 611 } 612 613 614 static int gmc_v10_0_early_init(struct amdgpu_ip_block *ip_block) 615 { 616 struct amdgpu_device *adev = ip_block->adev; 617 618 gmc_v10_0_set_mmhub_funcs(adev); 619 gmc_v10_0_set_gfxhub_funcs(adev); 620 gmc_v10_0_set_gmc_funcs(adev); 621 gmc_v10_0_set_irq_funcs(adev); 622 gmc_v10_0_set_umc_funcs(adev); 623 624 adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 625 adev->gmc.shared_aperture_end = 626 adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 627 adev->gmc.private_aperture_start = 0x1000000000000000ULL; 628 adev->gmc.private_aperture_end = 629 adev->gmc.private_aperture_start + (4ULL << 30) - 1; 630 adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF; 631 632 return 0; 633 } 634 635 static int gmc_v10_0_late_init(struct amdgpu_ip_block *ip_block) 636 { 637 struct amdgpu_device *adev = ip_block->adev; 638 int r; 639 640 r = amdgpu_gmc_allocate_vm_inv_eng(adev); 641 if (r) 642 return r; 643 644 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 645 } 646 647 static void gmc_v10_0_vram_gtt_location(struct amdgpu_device *adev, 648 struct amdgpu_gmc *mc) 649 { 650 u64 base = 0; 651 652 base = adev->gfxhub.funcs->get_fb_location(adev); 653 654 /* add the xgmi offset of the physical node */ 655 base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 656 657 amdgpu_gmc_set_agp_default(adev, mc); 658 amdgpu_gmc_vram_location(adev, &adev->gmc, base); 659 amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_BEST_FIT); 660 if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1)) 661 amdgpu_gmc_agp_location(adev, mc); 662 663 /* base offset of vram pages */ 664 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 665 666 /* add the xgmi offset of the physical node */ 667 adev->vm_manager.vram_base_offset += 668 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 669 } 670 671 /** 672 * gmc_v10_0_mc_init - initialize the memory controller driver params 673 * 674 * @adev: amdgpu_device pointer 675 * 676 * Look up the amount of vram, vram width, and decide how to place 677 * vram and gart within the GPU's physical address space. 678 * Returns 0 for success. 679 */ 680 static int gmc_v10_0_mc_init(struct amdgpu_device *adev) 681 { 682 int r; 683 684 /* size in MB on si */ 685 adev->gmc.mc_vram_size = 686 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 687 adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 688 689 if (!(adev->flags & AMD_IS_APU)) { 690 r = amdgpu_device_resize_fb_bar(adev); 691 if (r) 692 return r; 693 } 694 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 695 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 696 697 #ifdef CONFIG_X86_64 698 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) { 699 adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev); 700 adev->gmc.aper_size = adev->gmc.real_vram_size; 701 } 702 #endif 703 704 adev->gmc.visible_vram_size = adev->gmc.aper_size; 705 706 /* set the gart size */ 707 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 708 case IP_VERSION(10, 3, 1): /* DCE SG support */ 709 case IP_VERSION(10, 3, 3): /* DCE SG support */ 710 case IP_VERSION(10, 3, 6): /* DCE SG support */ 711 case IP_VERSION(10, 3, 7): /* DCE SG support */ 712 amdgpu_gmc_set_gart_size(adev, SZ_1G); 713 break; 714 default: 715 amdgpu_gmc_set_gart_size(adev, SZ_512M); 716 break; 717 } 718 719 gmc_v10_0_vram_gtt_location(adev, &adev->gmc); 720 721 return 0; 722 } 723 724 static int gmc_v10_0_gart_init(struct amdgpu_device *adev) 725 { 726 int r; 727 728 if (adev->gart.bo) { 729 WARN(1, "NAVI10 PCIE GART already initialized\n"); 730 return 0; 731 } 732 733 /* Initialize common gart structure */ 734 r = amdgpu_gart_init(adev); 735 if (r) 736 return r; 737 738 adev->gart.table_size = adev->gart.num_gpu_pages * 8; 739 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC) | 740 AMDGPU_PTE_EXECUTABLE; 741 742 return amdgpu_gart_table_vram_alloc(adev); 743 } 744 745 static int gmc_v10_0_sw_init(struct amdgpu_ip_block *ip_block) 746 { 747 int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 748 struct amdgpu_device *adev = ip_block->adev; 749 750 adev->gfxhub.funcs->init(adev); 751 752 adev->mmhub.funcs->init(adev); 753 754 spin_lock_init(&adev->gmc.invalidate_lock); 755 756 if ((adev->flags & AMD_IS_APU) && amdgpu_emu_mode == 1) { 757 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4; 758 adev->gmc.vram_width = 64; 759 } else if (amdgpu_emu_mode == 1) { 760 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_GDDR6; 761 adev->gmc.vram_width = 1 * 128; /* numchan * chansize */ 762 } else { 763 r = amdgpu_gmc_get_vram_info(adev, 764 &vram_width, &vram_type, &vram_vendor); 765 adev->gmc.vram_width = vram_width; 766 767 adev->gmc.vram_type = vram_type; 768 adev->gmc.vram_vendor = vram_vendor; 769 } 770 771 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 772 case IP_VERSION(10, 3, 0): 773 adev->gmc.mall_size = 128 * 1024 * 1024; 774 break; 775 case IP_VERSION(10, 3, 2): 776 adev->gmc.mall_size = 96 * 1024 * 1024; 777 break; 778 case IP_VERSION(10, 3, 4): 779 adev->gmc.mall_size = 32 * 1024 * 1024; 780 break; 781 case IP_VERSION(10, 3, 5): 782 adev->gmc.mall_size = 16 * 1024 * 1024; 783 break; 784 default: 785 adev->gmc.mall_size = 0; 786 break; 787 } 788 789 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 790 case IP_VERSION(10, 1, 10): 791 case IP_VERSION(10, 1, 1): 792 case IP_VERSION(10, 1, 2): 793 case IP_VERSION(10, 1, 3): 794 case IP_VERSION(10, 1, 4): 795 case IP_VERSION(10, 3, 0): 796 case IP_VERSION(10, 3, 2): 797 case IP_VERSION(10, 3, 1): 798 case IP_VERSION(10, 3, 4): 799 case IP_VERSION(10, 3, 5): 800 case IP_VERSION(10, 3, 6): 801 case IP_VERSION(10, 3, 3): 802 case IP_VERSION(10, 3, 7): 803 set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); 804 set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); 805 /* 806 * To fulfill 4-level page support, 807 * vm size is 256TB (48bit), maximum size of Navi10/Navi14/Navi12, 808 * block size 512 (9bit) 809 */ 810 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 811 break; 812 default: 813 break; 814 } 815 816 /* This interrupt is VMC page fault.*/ 817 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, 818 VMC_1_0__SRCID__VM_FAULT, 819 &adev->gmc.vm_fault); 820 821 if (r) 822 return r; 823 824 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, 825 UTCL2_1_0__SRCID__FAULT, 826 &adev->gmc.vm_fault); 827 if (r) 828 return r; 829 830 if (!amdgpu_sriov_vf(adev)) { 831 /* interrupt sent to DF. */ 832 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 833 &adev->gmc.ecc_irq); 834 if (r) 835 return r; 836 } 837 838 /* 839 * Set the internal MC address mask This is the max address of the GPU's 840 * internal address space. 841 */ 842 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 843 adev->gmc.pte_addr_mask = 0x0000FFFFFFFFF000ULL; /* 48 bit PA */ 844 845 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 846 if (r) { 847 drm_warn(adev_to_drm(adev), "No suitable DMA available.\n"); 848 return r; 849 } 850 851 adev->need_swiotlb = drm_need_swiotlb(44); 852 853 r = gmc_v10_0_mc_init(adev); 854 if (r) 855 return r; 856 857 /* Memory manager */ 858 r = amdgpu_bo_init(adev); 859 if (r) 860 return r; 861 862 r = gmc_v10_0_gart_init(adev); 863 if (r) 864 return r; 865 866 /* 867 * number of VMs 868 * VMID 0 is reserved for System 869 * amdgpu graphics/compute will use VMIDs 1-7 870 * amdkfd will use VMIDs 8-15 871 */ 872 adev->vm_manager.first_kfd_vmid = 8; 873 amdgpu_vmid_mgr_set_vmid_mask(adev, 874 GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1), 875 false); 876 amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true); 877 878 amdgpu_vm_manager_init(adev); 879 880 r = amdgpu_gmc_ras_sw_init(adev); 881 if (r) 882 return r; 883 884 return 0; 885 } 886 887 /** 888 * gmc_v10_0_gart_fini - vm fini callback 889 * 890 * @adev: amdgpu_device pointer 891 * 892 * Tears down the driver GART/VM setup (CIK). 893 */ 894 static void gmc_v10_0_gart_fini(struct amdgpu_device *adev) 895 { 896 amdgpu_gart_table_vram_free(adev); 897 } 898 899 static int gmc_v10_0_sw_fini(struct amdgpu_ip_block *ip_block) 900 { 901 struct amdgpu_device *adev = ip_block->adev; 902 903 amdgpu_vm_manager_fini(adev); 904 gmc_v10_0_gart_fini(adev); 905 amdgpu_gem_force_release(adev); 906 amdgpu_bo_fini(adev); 907 908 return 0; 909 } 910 911 static void gmc_v10_0_init_golden_registers(struct amdgpu_device *adev) 912 { 913 } 914 915 /** 916 * gmc_v10_0_gart_enable - gart enable 917 * 918 * @adev: amdgpu_device pointer 919 */ 920 static int gmc_v10_0_gart_enable(struct amdgpu_device *adev) 921 { 922 int r; 923 bool value; 924 925 if (adev->gart.bo == NULL) { 926 dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 927 return -EINVAL; 928 } 929 930 amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr); 931 932 if (!adev->in_s0ix) { 933 r = adev->gfxhub.funcs->gart_enable(adev); 934 if (r) 935 return r; 936 } 937 938 r = adev->mmhub.funcs->gart_enable(adev); 939 if (r) 940 return r; 941 942 adev->hdp.funcs->init_registers(adev); 943 944 /* Flush HDP after it is initialized */ 945 amdgpu_device_flush_hdp(adev, NULL); 946 947 value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS; 948 949 if (!adev->in_s0ix) 950 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 951 adev->mmhub.funcs->set_fault_enable_default(adev, value); 952 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0); 953 if (!adev->in_s0ix) 954 gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0); 955 956 drm_info(adev_to_drm(adev), "PCIE GART of %uM enabled (table at 0x%016llX).\n", 957 (unsigned int)(adev->gmc.gart_size >> 20), 958 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 959 960 return 0; 961 } 962 963 static int gmc_v10_0_hw_init(struct amdgpu_ip_block *ip_block) 964 { 965 struct amdgpu_device *adev = ip_block->adev; 966 int r; 967 968 adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode; 969 970 /* The sequence of these two function calls matters.*/ 971 gmc_v10_0_init_golden_registers(adev); 972 973 /* 974 * harvestable groups in gc_utcl2 need to be programmed before any GFX block 975 * register setup within GMC, or else system hang when harvesting SA. 976 */ 977 if (!adev->in_s0ix && adev->gfxhub.funcs && adev->gfxhub.funcs->utcl2_harvest) 978 adev->gfxhub.funcs->utcl2_harvest(adev); 979 980 r = gmc_v10_0_gart_enable(adev); 981 if (r) 982 return r; 983 984 if (amdgpu_emu_mode == 1) { 985 r = amdgpu_gmc_vram_checking(adev); 986 if (r) 987 return r; 988 } 989 990 if (adev->umc.funcs && adev->umc.funcs->init_registers) 991 adev->umc.funcs->init_registers(adev); 992 993 return 0; 994 } 995 996 /** 997 * gmc_v10_0_gart_disable - gart disable 998 * 999 * @adev: amdgpu_device pointer 1000 * 1001 * This disables all VM page table. 1002 */ 1003 static void gmc_v10_0_gart_disable(struct amdgpu_device *adev) 1004 { 1005 if (!adev->in_s0ix) 1006 adev->gfxhub.funcs->gart_disable(adev); 1007 adev->mmhub.funcs->gart_disable(adev); 1008 } 1009 1010 static int gmc_v10_0_hw_fini(struct amdgpu_ip_block *ip_block) 1011 { 1012 struct amdgpu_device *adev = ip_block->adev; 1013 1014 gmc_v10_0_gart_disable(adev); 1015 1016 if (amdgpu_sriov_vf(adev)) { 1017 /* full access mode, so don't touch any GMC register */ 1018 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 1019 return 0; 1020 } 1021 1022 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1023 1024 if (adev->gmc.ecc_irq.funcs && 1025 amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC)) 1026 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1027 1028 return 0; 1029 } 1030 1031 static int gmc_v10_0_suspend(struct amdgpu_ip_block *ip_block) 1032 { 1033 gmc_v10_0_hw_fini(ip_block); 1034 1035 return 0; 1036 } 1037 1038 static int gmc_v10_0_resume(struct amdgpu_ip_block *ip_block) 1039 { 1040 int r; 1041 1042 r = gmc_v10_0_hw_init(ip_block); 1043 if (r) 1044 return r; 1045 1046 amdgpu_vmid_reset_all(ip_block->adev); 1047 1048 return 0; 1049 } 1050 1051 static int gmc_v10_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 1052 { 1053 /* There is no need to wait for MC idle in GMC v10.*/ 1054 return 0; 1055 } 1056 1057 static int gmc_v10_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 1058 enum amd_clockgating_state state) 1059 { 1060 int r; 1061 struct amdgpu_device *adev = ip_block->adev; 1062 1063 /* 1064 * The issue mmhub can't disconnect from DF with MMHUB clock gating being disabled 1065 * is a new problem observed at DF 3.0.3, however with the same suspend sequence not 1066 * seen any issue on the DF 3.0.2 series platform. 1067 */ 1068 if (adev->in_s0ix && 1069 amdgpu_ip_version(adev, DF_HWIP, 0) > IP_VERSION(3, 0, 2)) { 1070 dev_dbg(adev->dev, "keep mmhub clock gating being enabled for s0ix\n"); 1071 return 0; 1072 } 1073 1074 r = adev->mmhub.funcs->set_clockgating(adev, state); 1075 if (r) 1076 return r; 1077 1078 if (amdgpu_ip_version(adev, ATHUB_HWIP, 0) >= IP_VERSION(2, 1, 0)) 1079 return athub_v2_1_set_clockgating(adev, state); 1080 else 1081 return athub_v2_0_set_clockgating(adev, state); 1082 } 1083 1084 static void gmc_v10_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 1085 { 1086 struct amdgpu_device *adev = ip_block->adev; 1087 1088 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 1, 3) || 1089 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(10, 1, 4)) 1090 return; 1091 1092 adev->mmhub.funcs->get_clockgating(adev, flags); 1093 1094 if (amdgpu_ip_version(adev, ATHUB_HWIP, 0) >= IP_VERSION(2, 1, 0)) 1095 athub_v2_1_get_clockgating(adev, flags); 1096 else 1097 athub_v2_0_get_clockgating(adev, flags); 1098 } 1099 1100 static int gmc_v10_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 1101 enum amd_powergating_state state) 1102 { 1103 return 0; 1104 } 1105 1106 const struct amd_ip_funcs gmc_v10_0_ip_funcs = { 1107 .name = "gmc_v10_0", 1108 .early_init = gmc_v10_0_early_init, 1109 .late_init = gmc_v10_0_late_init, 1110 .sw_init = gmc_v10_0_sw_init, 1111 .sw_fini = gmc_v10_0_sw_fini, 1112 .hw_init = gmc_v10_0_hw_init, 1113 .hw_fini = gmc_v10_0_hw_fini, 1114 .suspend = gmc_v10_0_suspend, 1115 .resume = gmc_v10_0_resume, 1116 .wait_for_idle = gmc_v10_0_wait_for_idle, 1117 .set_clockgating_state = gmc_v10_0_set_clockgating_state, 1118 .set_powergating_state = gmc_v10_0_set_powergating_state, 1119 .get_clockgating_state = gmc_v10_0_get_clockgating_state, 1120 }; 1121 1122 const struct amdgpu_ip_block_version gmc_v10_0_ip_block = { 1123 .type = AMD_IP_BLOCK_TYPE_GMC, 1124 .major = 10, 1125 .minor = 0, 1126 .rev = 0, 1127 .funcs = &gmc_v10_0_ip_funcs, 1128 }; 1129