1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/firmware.h> 24 #include <linux/slab.h> 25 #include <linux/module.h> 26 #include <linux/pci.h> 27 28 #include "amdgpu.h" 29 #include "amdgpu_atombios.h" 30 #include "amdgpu_ih.h" 31 #include "amdgpu_uvd.h" 32 #include "amdgpu_vce.h" 33 #include "amdgpu_ucode.h" 34 #include "amdgpu_psp.h" 35 #include "atom.h" 36 #include "amd_pcie.h" 37 38 #include "uvd/uvd_7_0_offset.h" 39 #include "gc/gc_9_0_offset.h" 40 #include "gc/gc_9_0_sh_mask.h" 41 #include "sdma0/sdma0_4_0_offset.h" 42 #include "sdma1/sdma1_4_0_offset.h" 43 #include "hdp/hdp_4_0_offset.h" 44 #include "hdp/hdp_4_0_sh_mask.h" 45 #include "smuio/smuio_9_0_offset.h" 46 #include "smuio/smuio_9_0_sh_mask.h" 47 #include "nbio/nbio_7_0_default.h" 48 #include "nbio/nbio_7_0_offset.h" 49 #include "nbio/nbio_7_0_sh_mask.h" 50 #include "nbio/nbio_7_0_smn.h" 51 #include "mp/mp_9_0_offset.h" 52 53 #include "soc15.h" 54 #include "soc15_common.h" 55 #include "gfx_v9_0.h" 56 #include "gmc_v9_0.h" 57 #include "gfxhub_v1_0.h" 58 #include "mmhub_v1_0.h" 59 #include "df_v1_7.h" 60 #include "df_v3_6.h" 61 #include "vega10_ih.h" 62 #include "sdma_v4_0.h" 63 #include "uvd_v7_0.h" 64 #include "vce_v4_0.h" 65 #include "vcn_v1_0.h" 66 #include "vcn_v2_5.h" 67 #include "dce_virtual.h" 68 #include "mxgpu_ai.h" 69 #include "amdgpu_smu.h" 70 #include "amdgpu_ras.h" 71 #include "amdgpu_xgmi.h" 72 #include <uapi/linux/kfd_ioctl.h> 73 74 #define mmMP0_MISC_CGTT_CTRL0 0x01b9 75 #define mmMP0_MISC_CGTT_CTRL0_BASE_IDX 0 76 #define mmMP0_MISC_LIGHT_SLEEP_CTRL 0x01ba 77 #define mmMP0_MISC_LIGHT_SLEEP_CTRL_BASE_IDX 0 78 79 /* for Vega20 register name change */ 80 #define mmHDP_MEM_POWER_CTRL 0x00d4 81 #define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK 0x00000001L 82 #define HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK 0x00000002L 83 #define HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK 0x00010000L 84 #define HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK 0x00020000L 85 #define mmHDP_MEM_POWER_CTRL_BASE_IDX 0 86 /* 87 * Indirect registers accessor 88 */ 89 static u32 soc15_pcie_rreg(struct amdgpu_device *adev, u32 reg) 90 { 91 unsigned long flags, address, data; 92 u32 r; 93 address = adev->nbio_funcs->get_pcie_index_offset(adev); 94 data = adev->nbio_funcs->get_pcie_data_offset(adev); 95 96 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 97 WREG32(address, reg); 98 (void)RREG32(address); 99 r = RREG32(data); 100 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 101 return r; 102 } 103 104 static void soc15_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 105 { 106 unsigned long flags, address, data; 107 108 address = adev->nbio_funcs->get_pcie_index_offset(adev); 109 data = adev->nbio_funcs->get_pcie_data_offset(adev); 110 111 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 112 WREG32(address, reg); 113 (void)RREG32(address); 114 WREG32(data, v); 115 (void)RREG32(data); 116 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 117 } 118 119 static u64 soc15_pcie_rreg64(struct amdgpu_device *adev, u32 reg) 120 { 121 unsigned long flags, address, data; 122 u64 r; 123 address = adev->nbio_funcs->get_pcie_index_offset(adev); 124 data = adev->nbio_funcs->get_pcie_data_offset(adev); 125 126 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 127 /* read low 32 bit */ 128 WREG32(address, reg); 129 (void)RREG32(address); 130 r = RREG32(data); 131 132 /* read high 32 bit*/ 133 WREG32(address, reg + 4); 134 (void)RREG32(address); 135 r |= ((u64)RREG32(data) << 32); 136 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 137 return r; 138 } 139 140 static void soc15_pcie_wreg64(struct amdgpu_device *adev, u32 reg, u64 v) 141 { 142 unsigned long flags, address, data; 143 144 address = adev->nbio_funcs->get_pcie_index_offset(adev); 145 data = adev->nbio_funcs->get_pcie_data_offset(adev); 146 147 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 148 /* write low 32 bit */ 149 WREG32(address, reg); 150 (void)RREG32(address); 151 WREG32(data, (u32)(v & 0xffffffffULL)); 152 (void)RREG32(data); 153 154 /* write high 32 bit */ 155 WREG32(address, reg + 4); 156 (void)RREG32(address); 157 WREG32(data, (u32)(v >> 32)); 158 (void)RREG32(data); 159 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 160 } 161 162 static u32 soc15_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg) 163 { 164 unsigned long flags, address, data; 165 u32 r; 166 167 address = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_INDEX); 168 data = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_DATA); 169 170 spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags); 171 WREG32(address, ((reg) & 0x1ff)); 172 r = RREG32(data); 173 spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags); 174 return r; 175 } 176 177 static void soc15_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 178 { 179 unsigned long flags, address, data; 180 181 address = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_INDEX); 182 data = SOC15_REG_OFFSET(UVD, 0, mmUVD_CTX_DATA); 183 184 spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags); 185 WREG32(address, ((reg) & 0x1ff)); 186 WREG32(data, (v)); 187 spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags); 188 } 189 190 static u32 soc15_didt_rreg(struct amdgpu_device *adev, u32 reg) 191 { 192 unsigned long flags, address, data; 193 u32 r; 194 195 address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX); 196 data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA); 197 198 spin_lock_irqsave(&adev->didt_idx_lock, flags); 199 WREG32(address, (reg)); 200 r = RREG32(data); 201 spin_unlock_irqrestore(&adev->didt_idx_lock, flags); 202 return r; 203 } 204 205 static void soc15_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 206 { 207 unsigned long flags, address, data; 208 209 address = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_INDEX); 210 data = SOC15_REG_OFFSET(GC, 0, mmDIDT_IND_DATA); 211 212 spin_lock_irqsave(&adev->didt_idx_lock, flags); 213 WREG32(address, (reg)); 214 WREG32(data, (v)); 215 spin_unlock_irqrestore(&adev->didt_idx_lock, flags); 216 } 217 218 static u32 soc15_gc_cac_rreg(struct amdgpu_device *adev, u32 reg) 219 { 220 unsigned long flags; 221 u32 r; 222 223 spin_lock_irqsave(&adev->gc_cac_idx_lock, flags); 224 WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg)); 225 r = RREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA); 226 spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags); 227 return r; 228 } 229 230 static void soc15_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 231 { 232 unsigned long flags; 233 234 spin_lock_irqsave(&adev->gc_cac_idx_lock, flags); 235 WREG32_SOC15(GC, 0, mmGC_CAC_IND_INDEX, (reg)); 236 WREG32_SOC15(GC, 0, mmGC_CAC_IND_DATA, (v)); 237 spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags); 238 } 239 240 static u32 soc15_se_cac_rreg(struct amdgpu_device *adev, u32 reg) 241 { 242 unsigned long flags; 243 u32 r; 244 245 spin_lock_irqsave(&adev->se_cac_idx_lock, flags); 246 WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg)); 247 r = RREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA); 248 spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags); 249 return r; 250 } 251 252 static void soc15_se_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 253 { 254 unsigned long flags; 255 256 spin_lock_irqsave(&adev->se_cac_idx_lock, flags); 257 WREG32_SOC15(GC, 0, mmSE_CAC_IND_INDEX, (reg)); 258 WREG32_SOC15(GC, 0, mmSE_CAC_IND_DATA, (v)); 259 spin_unlock_irqrestore(&adev->se_cac_idx_lock, flags); 260 } 261 262 static u32 soc15_get_config_memsize(struct amdgpu_device *adev) 263 { 264 return adev->nbio_funcs->get_memsize(adev); 265 } 266 267 static u32 soc15_get_xclk(struct amdgpu_device *adev) 268 { 269 return adev->clock.spll.reference_freq; 270 } 271 272 273 void soc15_grbm_select(struct amdgpu_device *adev, 274 u32 me, u32 pipe, u32 queue, u32 vmid) 275 { 276 u32 grbm_gfx_cntl = 0; 277 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, PIPEID, pipe); 278 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, MEID, me); 279 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, VMID, vmid); 280 grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, QUEUEID, queue); 281 282 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_CNTL, grbm_gfx_cntl); 283 } 284 285 static void soc15_vga_set_state(struct amdgpu_device *adev, bool state) 286 { 287 /* todo */ 288 } 289 290 static bool soc15_read_disabled_bios(struct amdgpu_device *adev) 291 { 292 /* todo */ 293 return false; 294 } 295 296 static bool soc15_read_bios_from_rom(struct amdgpu_device *adev, 297 u8 *bios, u32 length_bytes) 298 { 299 u32 *dw_ptr; 300 u32 i, length_dw; 301 302 if (bios == NULL) 303 return false; 304 if (length_bytes == 0) 305 return false; 306 /* APU vbios image is part of sbios image */ 307 if (adev->flags & AMD_IS_APU) 308 return false; 309 310 dw_ptr = (u32 *)bios; 311 length_dw = ALIGN(length_bytes, 4) / 4; 312 313 /* set rom index to 0 */ 314 WREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_INDEX), 0); 315 /* read out the rom data */ 316 for (i = 0; i < length_dw; i++) 317 dw_ptr[i] = RREG32(SOC15_REG_OFFSET(SMUIO, 0, mmROM_DATA)); 318 319 return true; 320 } 321 322 static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = { 323 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS)}, 324 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS2)}, 325 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE0)}, 326 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE1)}, 327 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE2)}, 328 { SOC15_REG_ENTRY(GC, 0, mmGRBM_STATUS_SE3)}, 329 { SOC15_REG_ENTRY(SDMA0, 0, mmSDMA0_STATUS_REG)}, 330 { SOC15_REG_ENTRY(SDMA1, 0, mmSDMA1_STATUS_REG)}, 331 { SOC15_REG_ENTRY(GC, 0, mmCP_STAT)}, 332 { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT1)}, 333 { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT2)}, 334 { SOC15_REG_ENTRY(GC, 0, mmCP_STALLED_STAT3)}, 335 { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_BUSY_STAT)}, 336 { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STALLED_STAT1)}, 337 { SOC15_REG_ENTRY(GC, 0, mmCP_CPF_STATUS)}, 338 { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STALLED_STAT1)}, 339 { SOC15_REG_ENTRY(GC, 0, mmCP_CPC_STATUS)}, 340 { SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)}, 341 { SOC15_REG_ENTRY(GC, 0, mmDB_DEBUG2)}, 342 }; 343 344 static uint32_t soc15_read_indexed_register(struct amdgpu_device *adev, u32 se_num, 345 u32 sh_num, u32 reg_offset) 346 { 347 uint32_t val; 348 349 mutex_lock(&adev->grbm_idx_mutex); 350 if (se_num != 0xffffffff || sh_num != 0xffffffff) 351 amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff); 352 353 val = RREG32(reg_offset); 354 355 if (se_num != 0xffffffff || sh_num != 0xffffffff) 356 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 357 mutex_unlock(&adev->grbm_idx_mutex); 358 return val; 359 } 360 361 static uint32_t soc15_get_register_value(struct amdgpu_device *adev, 362 bool indexed, u32 se_num, 363 u32 sh_num, u32 reg_offset) 364 { 365 if (indexed) { 366 return soc15_read_indexed_register(adev, se_num, sh_num, reg_offset); 367 } else { 368 if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG)) 369 return adev->gfx.config.gb_addr_config; 370 else if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmDB_DEBUG2)) 371 return adev->gfx.config.db_debug2; 372 return RREG32(reg_offset); 373 } 374 } 375 376 static int soc15_read_register(struct amdgpu_device *adev, u32 se_num, 377 u32 sh_num, u32 reg_offset, u32 *value) 378 { 379 uint32_t i; 380 struct soc15_allowed_register_entry *en; 381 382 *value = 0; 383 for (i = 0; i < ARRAY_SIZE(soc15_allowed_read_registers); i++) { 384 en = &soc15_allowed_read_registers[i]; 385 if (reg_offset != (adev->reg_offset[en->hwip][en->inst][en->seg] 386 + en->reg_offset)) 387 continue; 388 389 *value = soc15_get_register_value(adev, 390 soc15_allowed_read_registers[i].grbm_indexed, 391 se_num, sh_num, reg_offset); 392 return 0; 393 } 394 return -EINVAL; 395 } 396 397 398 /** 399 * soc15_program_register_sequence - program an array of registers. 400 * 401 * @adev: amdgpu_device pointer 402 * @regs: pointer to the register array 403 * @array_size: size of the register array 404 * 405 * Programs an array or registers with and and or masks. 406 * This is a helper for setting golden registers. 407 */ 408 409 void soc15_program_register_sequence(struct amdgpu_device *adev, 410 const struct soc15_reg_golden *regs, 411 const u32 array_size) 412 { 413 const struct soc15_reg_golden *entry; 414 u32 tmp, reg; 415 int i; 416 417 for (i = 0; i < array_size; ++i) { 418 entry = ®s[i]; 419 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 420 421 if (entry->and_mask == 0xffffffff) { 422 tmp = entry->or_mask; 423 } else { 424 tmp = RREG32(reg); 425 tmp &= ~(entry->and_mask); 426 tmp |= (entry->or_mask & entry->and_mask); 427 } 428 429 if (reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3) || 430 reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE) || 431 reg == SOC15_REG_OFFSET(GC, 0, mmPA_SC_ENHANCE_1) || 432 reg == SOC15_REG_OFFSET(GC, 0, mmSH_MEM_CONFIG)) 433 WREG32_RLC(reg, tmp); 434 else 435 WREG32(reg, tmp); 436 437 } 438 439 } 440 441 static int soc15_asic_mode1_reset(struct amdgpu_device *adev) 442 { 443 u32 i; 444 int ret = 0; 445 446 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 447 448 dev_info(adev->dev, "GPU mode1 reset\n"); 449 450 /* disable BM */ 451 pci_clear_master(adev->pdev); 452 453 pci_save_state(adev->pdev); 454 455 ret = psp_gpu_reset(adev); 456 if (ret) 457 dev_err(adev->dev, "GPU mode1 reset failed\n"); 458 459 pci_restore_state(adev->pdev); 460 461 /* wait for asic to come out of reset */ 462 for (i = 0; i < adev->usec_timeout; i++) { 463 u32 memsize = adev->nbio_funcs->get_memsize(adev); 464 465 if (memsize != 0xffffffff) 466 break; 467 udelay(1); 468 } 469 470 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 471 472 return ret; 473 } 474 475 static int soc15_asic_get_baco_capability(struct amdgpu_device *adev, bool *cap) 476 { 477 void *pp_handle = adev->powerplay.pp_handle; 478 const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; 479 480 if (!pp_funcs || !pp_funcs->get_asic_baco_capability) { 481 *cap = false; 482 return -ENOENT; 483 } 484 485 return pp_funcs->get_asic_baco_capability(pp_handle, cap); 486 } 487 488 static int soc15_asic_baco_reset(struct amdgpu_device *adev) 489 { 490 void *pp_handle = adev->powerplay.pp_handle; 491 const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs; 492 493 if (!pp_funcs ||!pp_funcs->get_asic_baco_state ||!pp_funcs->set_asic_baco_state) 494 return -ENOENT; 495 496 /* enter BACO state */ 497 if (pp_funcs->set_asic_baco_state(pp_handle, 1)) 498 return -EIO; 499 500 /* exit BACO state */ 501 if (pp_funcs->set_asic_baco_state(pp_handle, 0)) 502 return -EIO; 503 504 dev_info(adev->dev, "GPU BACO reset\n"); 505 506 adev->in_baco_reset = 1; 507 508 return 0; 509 } 510 511 static enum amd_reset_method 512 soc15_asic_reset_method(struct amdgpu_device *adev) 513 { 514 bool baco_reset; 515 516 switch (adev->asic_type) { 517 case CHIP_RAVEN: 518 return AMD_RESET_METHOD_MODE2; 519 case CHIP_VEGA10: 520 case CHIP_VEGA12: 521 soc15_asic_get_baco_capability(adev, &baco_reset); 522 break; 523 case CHIP_VEGA20: 524 if (adev->psp.sos_fw_version >= 0x80067) 525 soc15_asic_get_baco_capability(adev, &baco_reset); 526 else 527 baco_reset = false; 528 if (baco_reset) { 529 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0); 530 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 531 532 if (hive || (ras && ras->supported)) 533 baco_reset = false; 534 } 535 break; 536 default: 537 baco_reset = false; 538 break; 539 } 540 541 if (baco_reset) 542 return AMD_RESET_METHOD_BACO; 543 else 544 return AMD_RESET_METHOD_MODE1; 545 } 546 547 static int soc15_asic_reset(struct amdgpu_device *adev) 548 { 549 int ret; 550 551 if (soc15_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) 552 ret = soc15_asic_baco_reset(adev); 553 else 554 ret = soc15_asic_mode1_reset(adev); 555 556 return ret; 557 } 558 559 /*static int soc15_set_uvd_clock(struct amdgpu_device *adev, u32 clock, 560 u32 cntl_reg, u32 status_reg) 561 { 562 return 0; 563 }*/ 564 565 static int soc15_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk) 566 { 567 /*int r; 568 569 r = soc15_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS); 570 if (r) 571 return r; 572 573 r = soc15_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS); 574 */ 575 return 0; 576 } 577 578 static int soc15_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk) 579 { 580 /* todo */ 581 582 return 0; 583 } 584 585 static void soc15_pcie_gen3_enable(struct amdgpu_device *adev) 586 { 587 if (pci_is_root_bus(adev->pdev->bus)) 588 return; 589 590 if (amdgpu_pcie_gen2 == 0) 591 return; 592 593 if (adev->flags & AMD_IS_APU) 594 return; 595 596 if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 597 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3))) 598 return; 599 600 /* todo */ 601 } 602 603 static void soc15_program_aspm(struct amdgpu_device *adev) 604 { 605 606 if (amdgpu_aspm == 0) 607 return; 608 609 /* todo */ 610 } 611 612 static void soc15_enable_doorbell_aperture(struct amdgpu_device *adev, 613 bool enable) 614 { 615 adev->nbio_funcs->enable_doorbell_aperture(adev, enable); 616 adev->nbio_funcs->enable_doorbell_selfring_aperture(adev, enable); 617 } 618 619 static const struct amdgpu_ip_block_version vega10_common_ip_block = 620 { 621 .type = AMD_IP_BLOCK_TYPE_COMMON, 622 .major = 2, 623 .minor = 0, 624 .rev = 0, 625 .funcs = &soc15_common_ip_funcs, 626 }; 627 628 static uint32_t soc15_get_rev_id(struct amdgpu_device *adev) 629 { 630 return adev->nbio_funcs->get_rev_id(adev); 631 } 632 633 int soc15_set_ip_blocks(struct amdgpu_device *adev) 634 { 635 /* Set IP register base before any HW register access */ 636 switch (adev->asic_type) { 637 case CHIP_VEGA10: 638 case CHIP_VEGA12: 639 case CHIP_RAVEN: 640 vega10_reg_base_init(adev); 641 break; 642 case CHIP_VEGA20: 643 vega20_reg_base_init(adev); 644 break; 645 case CHIP_ARCTURUS: 646 arct_reg_base_init(adev); 647 break; 648 default: 649 return -EINVAL; 650 } 651 652 if (adev->asic_type == CHIP_VEGA20 || adev->asic_type == CHIP_ARCTURUS) 653 adev->gmc.xgmi.supported = true; 654 655 if (adev->flags & AMD_IS_APU) 656 adev->nbio_funcs = &nbio_v7_0_funcs; 657 else if (adev->asic_type == CHIP_VEGA20 || 658 adev->asic_type == CHIP_ARCTURUS) 659 adev->nbio_funcs = &nbio_v7_4_funcs; 660 else 661 adev->nbio_funcs = &nbio_v6_1_funcs; 662 663 if (adev->asic_type == CHIP_VEGA20 || adev->asic_type == CHIP_ARCTURUS) 664 adev->df_funcs = &df_v3_6_funcs; 665 else 666 adev->df_funcs = &df_v1_7_funcs; 667 668 adev->rev_id = soc15_get_rev_id(adev); 669 adev->nbio_funcs->detect_hw_virt(adev); 670 671 if (amdgpu_sriov_vf(adev)) 672 adev->virt.ops = &xgpu_ai_virt_ops; 673 674 switch (adev->asic_type) { 675 case CHIP_VEGA10: 676 case CHIP_VEGA12: 677 case CHIP_VEGA20: 678 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block); 679 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block); 680 681 /* For Vega10 SR-IOV, PSP need to be initialized before IH */ 682 if (amdgpu_sriov_vf(adev)) { 683 if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 684 if (adev->asic_type == CHIP_VEGA20) 685 amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); 686 else 687 amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block); 688 } 689 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); 690 } else { 691 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); 692 if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 693 if (adev->asic_type == CHIP_VEGA20) 694 amdgpu_device_ip_block_add(adev, &psp_v11_0_ip_block); 695 else 696 amdgpu_device_ip_block_add(adev, &psp_v3_1_ip_block); 697 } 698 } 699 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block); 700 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); 701 if (!amdgpu_sriov_vf(adev)) { 702 if (is_support_sw_smu(adev)) 703 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); 704 else 705 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); 706 } 707 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) 708 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); 709 #if defined(CONFIG_DRM_AMD_DC) 710 else if (amdgpu_device_has_dc_support(adev)) 711 amdgpu_device_ip_block_add(adev, &dm_ip_block); 712 #endif 713 if (!(adev->asic_type == CHIP_VEGA20 && amdgpu_sriov_vf(adev))) { 714 amdgpu_device_ip_block_add(adev, &uvd_v7_0_ip_block); 715 amdgpu_device_ip_block_add(adev, &vce_v4_0_ip_block); 716 } 717 break; 718 case CHIP_RAVEN: 719 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block); 720 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block); 721 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); 722 if (likely(adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) 723 amdgpu_device_ip_block_add(adev, &psp_v10_0_ip_block); 724 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block); 725 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); 726 amdgpu_device_ip_block_add(adev, &pp_smu_ip_block); 727 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) 728 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); 729 #if defined(CONFIG_DRM_AMD_DC) 730 else if (amdgpu_device_has_dc_support(adev)) 731 amdgpu_device_ip_block_add(adev, &dm_ip_block); 732 #endif 733 amdgpu_device_ip_block_add(adev, &vcn_v1_0_ip_block); 734 break; 735 case CHIP_ARCTURUS: 736 amdgpu_device_ip_block_add(adev, &vega10_common_ip_block); 737 amdgpu_device_ip_block_add(adev, &gmc_v9_0_ip_block); 738 amdgpu_device_ip_block_add(adev, &vega10_ih_ip_block); 739 if (adev->enable_virtual_display || amdgpu_sriov_vf(adev)) 740 amdgpu_device_ip_block_add(adev, &dce_virtual_ip_block); 741 amdgpu_device_ip_block_add(adev, &gfx_v9_0_ip_block); 742 amdgpu_device_ip_block_add(adev, &sdma_v4_0_ip_block); 743 amdgpu_device_ip_block_add(adev, &smu_v11_0_ip_block); 744 amdgpu_device_ip_block_add(adev, &vcn_v2_5_ip_block); 745 break; 746 default: 747 return -EINVAL; 748 } 749 750 return 0; 751 } 752 753 static void soc15_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring) 754 { 755 adev->nbio_funcs->hdp_flush(adev, ring); 756 } 757 758 static void soc15_invalidate_hdp(struct amdgpu_device *adev, 759 struct amdgpu_ring *ring) 760 { 761 if (!ring || !ring->funcs->emit_wreg) 762 WREG32_SOC15_NO_KIQ(HDP, 0, mmHDP_READ_CACHE_INVALIDATE, 1); 763 else 764 amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( 765 HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1); 766 } 767 768 static bool soc15_need_full_reset(struct amdgpu_device *adev) 769 { 770 /* change this when we implement soft reset */ 771 return true; 772 } 773 static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, 774 uint64_t *count1) 775 { 776 uint32_t perfctr = 0; 777 uint64_t cnt0_of, cnt1_of; 778 int tmp; 779 780 /* This reports 0 on APUs, so return to avoid writing/reading registers 781 * that may or may not be different from their GPU counterparts 782 */ 783 if (adev->flags & AMD_IS_APU) 784 return; 785 786 /* Set the 2 events that we wish to watch, defined above */ 787 /* Reg 40 is # received msgs */ 788 /* Reg 104 is # of posted requests sent */ 789 perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40); 790 perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104); 791 792 /* Write to enable desired perf counters */ 793 WREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK, perfctr); 794 /* Zero out and enable the perf counters 795 * Write 0x5: 796 * Bit 0 = Start all counters(1) 797 * Bit 2 = Global counter reset enable(1) 798 */ 799 WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000005); 800 801 msleep(1000); 802 803 /* Load the shadow and disable the perf counters 804 * Write 0x2: 805 * Bit 0 = Stop counters(0) 806 * Bit 1 = Load the shadow counters(1) 807 */ 808 WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000002); 809 810 /* Read register values to get any >32bit overflow */ 811 tmp = RREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK); 812 cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER); 813 cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER); 814 815 /* Get the values and add the overflow */ 816 *count0 = RREG32_PCIE(smnPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32); 817 *count1 = RREG32_PCIE(smnPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32); 818 } 819 820 static void vega20_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, 821 uint64_t *count1) 822 { 823 uint32_t perfctr = 0; 824 uint64_t cnt0_of, cnt1_of; 825 int tmp; 826 827 /* This reports 0 on APUs, so return to avoid writing/reading registers 828 * that may or may not be different from their GPU counterparts 829 */ 830 if (adev->flags & AMD_IS_APU) 831 return; 832 833 /* Set the 2 events that we wish to watch, defined above */ 834 /* Reg 40 is # received msgs */ 835 /* Reg 108 is # of posted requests sent on VG20 */ 836 perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK3, 837 EVENT0_SEL, 40); 838 perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK3, 839 EVENT1_SEL, 108); 840 841 /* Write to enable desired perf counters */ 842 WREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK3, perfctr); 843 /* Zero out and enable the perf counters 844 * Write 0x5: 845 * Bit 0 = Start all counters(1) 846 * Bit 2 = Global counter reset enable(1) 847 */ 848 WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000005); 849 850 msleep(1000); 851 852 /* Load the shadow and disable the perf counters 853 * Write 0x2: 854 * Bit 0 = Stop counters(0) 855 * Bit 1 = Load the shadow counters(1) 856 */ 857 WREG32_PCIE(smnPCIE_PERF_COUNT_CNTL, 0x00000002); 858 859 /* Read register values to get any >32bit overflow */ 860 tmp = RREG32_PCIE(smnPCIE_PERF_CNTL_TXCLK3); 861 cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK3, COUNTER0_UPPER); 862 cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK3, COUNTER1_UPPER); 863 864 /* Get the values and add the overflow */ 865 *count0 = RREG32_PCIE(smnPCIE_PERF_COUNT0_TXCLK3) | (cnt0_of << 32); 866 *count1 = RREG32_PCIE(smnPCIE_PERF_COUNT1_TXCLK3) | (cnt1_of << 32); 867 } 868 869 static bool soc15_need_reset_on_init(struct amdgpu_device *adev) 870 { 871 u32 sol_reg; 872 873 /* Just return false for soc15 GPUs. Reset does not seem to 874 * be necessary. 875 */ 876 if (!amdgpu_passthrough(adev)) 877 return false; 878 879 if (adev->flags & AMD_IS_APU) 880 return false; 881 882 /* Check sOS sign of life register to confirm sys driver and sOS 883 * are already been loaded. 884 */ 885 sol_reg = RREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_81); 886 if (sol_reg) 887 return true; 888 889 return false; 890 } 891 892 static uint64_t soc15_get_pcie_replay_count(struct amdgpu_device *adev) 893 { 894 uint64_t nak_r, nak_g; 895 896 /* Get the number of NAKs received and generated */ 897 nak_r = RREG32_PCIE(smnPCIE_RX_NUM_NAK); 898 nak_g = RREG32_PCIE(smnPCIE_RX_NUM_NAK_GENERATED); 899 900 /* Add the total number of NAKs, i.e the number of replays */ 901 return (nak_r + nak_g); 902 } 903 904 static const struct amdgpu_asic_funcs soc15_asic_funcs = 905 { 906 .read_disabled_bios = &soc15_read_disabled_bios, 907 .read_bios_from_rom = &soc15_read_bios_from_rom, 908 .read_register = &soc15_read_register, 909 .reset = &soc15_asic_reset, 910 .reset_method = &soc15_asic_reset_method, 911 .set_vga_state = &soc15_vga_set_state, 912 .get_xclk = &soc15_get_xclk, 913 .set_uvd_clocks = &soc15_set_uvd_clocks, 914 .set_vce_clocks = &soc15_set_vce_clocks, 915 .get_config_memsize = &soc15_get_config_memsize, 916 .flush_hdp = &soc15_flush_hdp, 917 .invalidate_hdp = &soc15_invalidate_hdp, 918 .need_full_reset = &soc15_need_full_reset, 919 .init_doorbell_index = &vega10_doorbell_index_init, 920 .get_pcie_usage = &soc15_get_pcie_usage, 921 .need_reset_on_init = &soc15_need_reset_on_init, 922 .get_pcie_replay_count = &soc15_get_pcie_replay_count, 923 }; 924 925 static const struct amdgpu_asic_funcs vega20_asic_funcs = 926 { 927 .read_disabled_bios = &soc15_read_disabled_bios, 928 .read_bios_from_rom = &soc15_read_bios_from_rom, 929 .read_register = &soc15_read_register, 930 .reset = &soc15_asic_reset, 931 .set_vga_state = &soc15_vga_set_state, 932 .get_xclk = &soc15_get_xclk, 933 .set_uvd_clocks = &soc15_set_uvd_clocks, 934 .set_vce_clocks = &soc15_set_vce_clocks, 935 .get_config_memsize = &soc15_get_config_memsize, 936 .flush_hdp = &soc15_flush_hdp, 937 .invalidate_hdp = &soc15_invalidate_hdp, 938 .need_full_reset = &soc15_need_full_reset, 939 .init_doorbell_index = &vega20_doorbell_index_init, 940 .get_pcie_usage = &vega20_get_pcie_usage, 941 .need_reset_on_init = &soc15_need_reset_on_init, 942 .get_pcie_replay_count = &soc15_get_pcie_replay_count, 943 .reset_method = &soc15_asic_reset_method 944 }; 945 946 static int soc15_common_early_init(void *handle) 947 { 948 #define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE) 949 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 950 951 adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET; 952 adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET; 953 adev->smc_rreg = NULL; 954 adev->smc_wreg = NULL; 955 adev->pcie_rreg = &soc15_pcie_rreg; 956 adev->pcie_wreg = &soc15_pcie_wreg; 957 adev->pcie_rreg64 = &soc15_pcie_rreg64; 958 adev->pcie_wreg64 = &soc15_pcie_wreg64; 959 adev->uvd_ctx_rreg = &soc15_uvd_ctx_rreg; 960 adev->uvd_ctx_wreg = &soc15_uvd_ctx_wreg; 961 adev->didt_rreg = &soc15_didt_rreg; 962 adev->didt_wreg = &soc15_didt_wreg; 963 adev->gc_cac_rreg = &soc15_gc_cac_rreg; 964 adev->gc_cac_wreg = &soc15_gc_cac_wreg; 965 adev->se_cac_rreg = &soc15_se_cac_rreg; 966 adev->se_cac_wreg = &soc15_se_cac_wreg; 967 968 969 adev->external_rev_id = 0xFF; 970 switch (adev->asic_type) { 971 case CHIP_VEGA10: 972 adev->asic_funcs = &soc15_asic_funcs; 973 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | 974 AMD_CG_SUPPORT_GFX_MGLS | 975 AMD_CG_SUPPORT_GFX_RLC_LS | 976 AMD_CG_SUPPORT_GFX_CP_LS | 977 AMD_CG_SUPPORT_GFX_3D_CGCG | 978 AMD_CG_SUPPORT_GFX_3D_CGLS | 979 AMD_CG_SUPPORT_GFX_CGCG | 980 AMD_CG_SUPPORT_GFX_CGLS | 981 AMD_CG_SUPPORT_BIF_MGCG | 982 AMD_CG_SUPPORT_BIF_LS | 983 AMD_CG_SUPPORT_HDP_LS | 984 AMD_CG_SUPPORT_DRM_MGCG | 985 AMD_CG_SUPPORT_DRM_LS | 986 AMD_CG_SUPPORT_ROM_MGCG | 987 AMD_CG_SUPPORT_DF_MGCG | 988 AMD_CG_SUPPORT_SDMA_MGCG | 989 AMD_CG_SUPPORT_SDMA_LS | 990 AMD_CG_SUPPORT_MC_MGCG | 991 AMD_CG_SUPPORT_MC_LS; 992 adev->pg_flags = 0; 993 adev->external_rev_id = 0x1; 994 break; 995 case CHIP_VEGA12: 996 adev->asic_funcs = &soc15_asic_funcs; 997 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | 998 AMD_CG_SUPPORT_GFX_MGLS | 999 AMD_CG_SUPPORT_GFX_CGCG | 1000 AMD_CG_SUPPORT_GFX_CGLS | 1001 AMD_CG_SUPPORT_GFX_3D_CGCG | 1002 AMD_CG_SUPPORT_GFX_3D_CGLS | 1003 AMD_CG_SUPPORT_GFX_CP_LS | 1004 AMD_CG_SUPPORT_MC_LS | 1005 AMD_CG_SUPPORT_MC_MGCG | 1006 AMD_CG_SUPPORT_SDMA_MGCG | 1007 AMD_CG_SUPPORT_SDMA_LS | 1008 AMD_CG_SUPPORT_BIF_MGCG | 1009 AMD_CG_SUPPORT_BIF_LS | 1010 AMD_CG_SUPPORT_HDP_MGCG | 1011 AMD_CG_SUPPORT_HDP_LS | 1012 AMD_CG_SUPPORT_ROM_MGCG | 1013 AMD_CG_SUPPORT_VCE_MGCG | 1014 AMD_CG_SUPPORT_UVD_MGCG; 1015 adev->pg_flags = 0; 1016 adev->external_rev_id = adev->rev_id + 0x14; 1017 break; 1018 case CHIP_VEGA20: 1019 adev->asic_funcs = &vega20_asic_funcs; 1020 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | 1021 AMD_CG_SUPPORT_GFX_MGLS | 1022 AMD_CG_SUPPORT_GFX_CGCG | 1023 AMD_CG_SUPPORT_GFX_CGLS | 1024 AMD_CG_SUPPORT_GFX_3D_CGCG | 1025 AMD_CG_SUPPORT_GFX_3D_CGLS | 1026 AMD_CG_SUPPORT_GFX_CP_LS | 1027 AMD_CG_SUPPORT_MC_LS | 1028 AMD_CG_SUPPORT_MC_MGCG | 1029 AMD_CG_SUPPORT_SDMA_MGCG | 1030 AMD_CG_SUPPORT_SDMA_LS | 1031 AMD_CG_SUPPORT_BIF_MGCG | 1032 AMD_CG_SUPPORT_BIF_LS | 1033 AMD_CG_SUPPORT_HDP_MGCG | 1034 AMD_CG_SUPPORT_HDP_LS | 1035 AMD_CG_SUPPORT_ROM_MGCG | 1036 AMD_CG_SUPPORT_VCE_MGCG | 1037 AMD_CG_SUPPORT_UVD_MGCG; 1038 adev->pg_flags = 0; 1039 adev->external_rev_id = adev->rev_id + 0x28; 1040 break; 1041 case CHIP_RAVEN: 1042 adev->asic_funcs = &soc15_asic_funcs; 1043 if (adev->rev_id >= 0x8) 1044 adev->external_rev_id = adev->rev_id + 0x79; 1045 else if (adev->pdev->device == 0x15d8) 1046 adev->external_rev_id = adev->rev_id + 0x41; 1047 else if (adev->rev_id == 1) 1048 adev->external_rev_id = adev->rev_id + 0x20; 1049 else 1050 adev->external_rev_id = adev->rev_id + 0x01; 1051 1052 if (adev->rev_id >= 0x8) { 1053 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | 1054 AMD_CG_SUPPORT_GFX_MGLS | 1055 AMD_CG_SUPPORT_GFX_CP_LS | 1056 AMD_CG_SUPPORT_GFX_3D_CGCG | 1057 AMD_CG_SUPPORT_GFX_3D_CGLS | 1058 AMD_CG_SUPPORT_GFX_CGCG | 1059 AMD_CG_SUPPORT_GFX_CGLS | 1060 AMD_CG_SUPPORT_BIF_LS | 1061 AMD_CG_SUPPORT_HDP_LS | 1062 AMD_CG_SUPPORT_ROM_MGCG | 1063 AMD_CG_SUPPORT_MC_MGCG | 1064 AMD_CG_SUPPORT_MC_LS | 1065 AMD_CG_SUPPORT_SDMA_MGCG | 1066 AMD_CG_SUPPORT_SDMA_LS | 1067 AMD_CG_SUPPORT_VCN_MGCG; 1068 1069 adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN; 1070 } else if (adev->pdev->device == 0x15d8) { 1071 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | 1072 AMD_CG_SUPPORT_GFX_MGLS | 1073 AMD_CG_SUPPORT_GFX_CP_LS | 1074 AMD_CG_SUPPORT_GFX_3D_CGCG | 1075 AMD_CG_SUPPORT_GFX_3D_CGLS | 1076 AMD_CG_SUPPORT_GFX_CGCG | 1077 AMD_CG_SUPPORT_GFX_CGLS | 1078 AMD_CG_SUPPORT_BIF_LS | 1079 AMD_CG_SUPPORT_HDP_LS | 1080 AMD_CG_SUPPORT_ROM_MGCG | 1081 AMD_CG_SUPPORT_MC_MGCG | 1082 AMD_CG_SUPPORT_MC_LS | 1083 AMD_CG_SUPPORT_SDMA_MGCG | 1084 AMD_CG_SUPPORT_SDMA_LS; 1085 1086 adev->pg_flags = AMD_PG_SUPPORT_SDMA | 1087 AMD_PG_SUPPORT_MMHUB | 1088 AMD_PG_SUPPORT_VCN | 1089 AMD_PG_SUPPORT_VCN_DPG; 1090 } else { 1091 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | 1092 AMD_CG_SUPPORT_GFX_MGLS | 1093 AMD_CG_SUPPORT_GFX_RLC_LS | 1094 AMD_CG_SUPPORT_GFX_CP_LS | 1095 AMD_CG_SUPPORT_GFX_3D_CGCG | 1096 AMD_CG_SUPPORT_GFX_3D_CGLS | 1097 AMD_CG_SUPPORT_GFX_CGCG | 1098 AMD_CG_SUPPORT_GFX_CGLS | 1099 AMD_CG_SUPPORT_BIF_MGCG | 1100 AMD_CG_SUPPORT_BIF_LS | 1101 AMD_CG_SUPPORT_HDP_MGCG | 1102 AMD_CG_SUPPORT_HDP_LS | 1103 AMD_CG_SUPPORT_DRM_MGCG | 1104 AMD_CG_SUPPORT_DRM_LS | 1105 AMD_CG_SUPPORT_ROM_MGCG | 1106 AMD_CG_SUPPORT_MC_MGCG | 1107 AMD_CG_SUPPORT_MC_LS | 1108 AMD_CG_SUPPORT_SDMA_MGCG | 1109 AMD_CG_SUPPORT_SDMA_LS | 1110 AMD_CG_SUPPORT_VCN_MGCG; 1111 1112 adev->pg_flags = AMD_PG_SUPPORT_SDMA | AMD_PG_SUPPORT_VCN; 1113 } 1114 1115 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1116 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1117 AMD_PG_SUPPORT_CP | 1118 AMD_PG_SUPPORT_RLC_SMU_HS; 1119 break; 1120 case CHIP_ARCTURUS: 1121 adev->asic_funcs = &vega20_asic_funcs; 1122 adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG | 1123 AMD_CG_SUPPORT_GFX_MGLS | 1124 AMD_CG_SUPPORT_GFX_CGCG | 1125 AMD_CG_SUPPORT_GFX_CGLS | 1126 AMD_CG_SUPPORT_GFX_CP_LS | 1127 AMD_CG_SUPPORT_HDP_MGCG | 1128 AMD_CG_SUPPORT_HDP_LS | 1129 AMD_CG_SUPPORT_SDMA_MGCG | 1130 AMD_CG_SUPPORT_SDMA_LS | 1131 AMD_CG_SUPPORT_MC_MGCG | 1132 AMD_CG_SUPPORT_MC_LS; 1133 adev->pg_flags = 0; 1134 adev->external_rev_id = adev->rev_id + 0x32; 1135 break; 1136 default: 1137 /* FIXME: not supported yet */ 1138 return -EINVAL; 1139 } 1140 1141 if (amdgpu_sriov_vf(adev)) { 1142 amdgpu_virt_init_setting(adev); 1143 xgpu_ai_mailbox_set_irq_funcs(adev); 1144 } 1145 1146 return 0; 1147 } 1148 1149 static int soc15_common_late_init(void *handle) 1150 { 1151 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1152 1153 if (amdgpu_sriov_vf(adev)) 1154 xgpu_ai_mailbox_get_irq(adev); 1155 1156 return 0; 1157 } 1158 1159 static int soc15_common_sw_init(void *handle) 1160 { 1161 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1162 1163 if (amdgpu_sriov_vf(adev)) 1164 xgpu_ai_mailbox_add_irq_id(adev); 1165 1166 adev->df_funcs->sw_init(adev); 1167 1168 return 0; 1169 } 1170 1171 static int soc15_common_sw_fini(void *handle) 1172 { 1173 return 0; 1174 } 1175 1176 static void soc15_doorbell_range_init(struct amdgpu_device *adev) 1177 { 1178 int i; 1179 struct amdgpu_ring *ring; 1180 1181 /* sdma/ih doorbell range are programed by hypervisor */ 1182 if (!amdgpu_sriov_vf(adev)) { 1183 for (i = 0; i < adev->sdma.num_instances; i++) { 1184 ring = &adev->sdma.instance[i].ring; 1185 adev->nbio_funcs->sdma_doorbell_range(adev, i, 1186 ring->use_doorbell, ring->doorbell_index, 1187 adev->doorbell_index.sdma_doorbell_range); 1188 } 1189 1190 adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell, 1191 adev->irq.ih.doorbell_index); 1192 } 1193 } 1194 1195 static int soc15_common_hw_init(void *handle) 1196 { 1197 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1198 1199 /* enable pcie gen2/3 link */ 1200 soc15_pcie_gen3_enable(adev); 1201 /* enable aspm */ 1202 soc15_program_aspm(adev); 1203 /* setup nbio registers */ 1204 adev->nbio_funcs->init_registers(adev); 1205 /* remap HDP registers to a hole in mmio space, 1206 * for the purpose of expose those registers 1207 * to process space 1208 */ 1209 if (adev->nbio_funcs->remap_hdp_registers) 1210 adev->nbio_funcs->remap_hdp_registers(adev); 1211 1212 /* enable the doorbell aperture */ 1213 soc15_enable_doorbell_aperture(adev, true); 1214 /* HW doorbell routing policy: doorbell writing not 1215 * in SDMA/IH/MM/ACV range will be routed to CP. So 1216 * we need to init SDMA/IH/MM/ACV doorbell range prior 1217 * to CP ip block init and ring test. 1218 */ 1219 soc15_doorbell_range_init(adev); 1220 1221 return 0; 1222 } 1223 1224 static int soc15_common_hw_fini(void *handle) 1225 { 1226 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1227 1228 /* disable the doorbell aperture */ 1229 soc15_enable_doorbell_aperture(adev, false); 1230 if (amdgpu_sriov_vf(adev)) 1231 xgpu_ai_mailbox_put_irq(adev); 1232 1233 return 0; 1234 } 1235 1236 static int soc15_common_suspend(void *handle) 1237 { 1238 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1239 1240 return soc15_common_hw_fini(adev); 1241 } 1242 1243 static int soc15_common_resume(void *handle) 1244 { 1245 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1246 1247 return soc15_common_hw_init(adev); 1248 } 1249 1250 static bool soc15_common_is_idle(void *handle) 1251 { 1252 return true; 1253 } 1254 1255 static int soc15_common_wait_for_idle(void *handle) 1256 { 1257 return 0; 1258 } 1259 1260 static int soc15_common_soft_reset(void *handle) 1261 { 1262 return 0; 1263 } 1264 1265 static void soc15_update_hdp_light_sleep(struct amdgpu_device *adev, bool enable) 1266 { 1267 uint32_t def, data; 1268 1269 if (adev->asic_type == CHIP_VEGA20 || 1270 adev->asic_type == CHIP_ARCTURUS) { 1271 def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL)); 1272 1273 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) 1274 data |= HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | 1275 HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | 1276 HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | 1277 HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK; 1278 else 1279 data &= ~(HDP_MEM_POWER_CTRL__IPH_MEM_POWER_CTRL_EN_MASK | 1280 HDP_MEM_POWER_CTRL__IPH_MEM_POWER_LS_EN_MASK | 1281 HDP_MEM_POWER_CTRL__RC_MEM_POWER_CTRL_EN_MASK | 1282 HDP_MEM_POWER_CTRL__RC_MEM_POWER_LS_EN_MASK); 1283 1284 if (def != data) 1285 WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_CTRL), data); 1286 } else { 1287 def = data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); 1288 1289 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS)) 1290 data |= HDP_MEM_POWER_LS__LS_ENABLE_MASK; 1291 else 1292 data &= ~HDP_MEM_POWER_LS__LS_ENABLE_MASK; 1293 1294 if (def != data) 1295 WREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS), data); 1296 } 1297 } 1298 1299 static void soc15_update_drm_clock_gating(struct amdgpu_device *adev, bool enable) 1300 { 1301 uint32_t def, data; 1302 1303 def = data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0)); 1304 1305 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DRM_MGCG)) 1306 data &= ~(0x01000000 | 1307 0x02000000 | 1308 0x04000000 | 1309 0x08000000 | 1310 0x10000000 | 1311 0x20000000 | 1312 0x40000000 | 1313 0x80000000); 1314 else 1315 data |= (0x01000000 | 1316 0x02000000 | 1317 0x04000000 | 1318 0x08000000 | 1319 0x10000000 | 1320 0x20000000 | 1321 0x40000000 | 1322 0x80000000); 1323 1324 if (def != data) 1325 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0), data); 1326 } 1327 1328 static void soc15_update_drm_light_sleep(struct amdgpu_device *adev, bool enable) 1329 { 1330 uint32_t def, data; 1331 1332 def = data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_LIGHT_SLEEP_CTRL)); 1333 1334 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_DRM_LS)) 1335 data |= 1; 1336 else 1337 data &= ~1; 1338 1339 if (def != data) 1340 WREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_LIGHT_SLEEP_CTRL), data); 1341 } 1342 1343 static void soc15_update_rom_medium_grain_clock_gating(struct amdgpu_device *adev, 1344 bool enable) 1345 { 1346 uint32_t def, data; 1347 1348 def = data = RREG32(SOC15_REG_OFFSET(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0)); 1349 1350 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_ROM_MGCG)) 1351 data &= ~(CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK | 1352 CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK); 1353 else 1354 data |= CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK | 1355 CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE1_MASK; 1356 1357 if (def != data) 1358 WREG32(SOC15_REG_OFFSET(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0), data); 1359 } 1360 1361 static int soc15_common_set_clockgating_state(void *handle, 1362 enum amd_clockgating_state state) 1363 { 1364 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1365 1366 if (amdgpu_sriov_vf(adev)) 1367 return 0; 1368 1369 switch (adev->asic_type) { 1370 case CHIP_VEGA10: 1371 case CHIP_VEGA12: 1372 case CHIP_VEGA20: 1373 adev->nbio_funcs->update_medium_grain_clock_gating(adev, 1374 state == AMD_CG_STATE_GATE ? true : false); 1375 adev->nbio_funcs->update_medium_grain_light_sleep(adev, 1376 state == AMD_CG_STATE_GATE ? true : false); 1377 soc15_update_hdp_light_sleep(adev, 1378 state == AMD_CG_STATE_GATE ? true : false); 1379 soc15_update_drm_clock_gating(adev, 1380 state == AMD_CG_STATE_GATE ? true : false); 1381 soc15_update_drm_light_sleep(adev, 1382 state == AMD_CG_STATE_GATE ? true : false); 1383 soc15_update_rom_medium_grain_clock_gating(adev, 1384 state == AMD_CG_STATE_GATE ? true : false); 1385 adev->df_funcs->update_medium_grain_clock_gating(adev, 1386 state == AMD_CG_STATE_GATE ? true : false); 1387 break; 1388 case CHIP_RAVEN: 1389 adev->nbio_funcs->update_medium_grain_clock_gating(adev, 1390 state == AMD_CG_STATE_GATE ? true : false); 1391 adev->nbio_funcs->update_medium_grain_light_sleep(adev, 1392 state == AMD_CG_STATE_GATE ? true : false); 1393 soc15_update_hdp_light_sleep(adev, 1394 state == AMD_CG_STATE_GATE ? true : false); 1395 soc15_update_drm_clock_gating(adev, 1396 state == AMD_CG_STATE_GATE ? true : false); 1397 soc15_update_drm_light_sleep(adev, 1398 state == AMD_CG_STATE_GATE ? true : false); 1399 soc15_update_rom_medium_grain_clock_gating(adev, 1400 state == AMD_CG_STATE_GATE ? true : false); 1401 break; 1402 case CHIP_ARCTURUS: 1403 soc15_update_hdp_light_sleep(adev, 1404 state == AMD_CG_STATE_GATE ? true : false); 1405 break; 1406 default: 1407 break; 1408 } 1409 return 0; 1410 } 1411 1412 static void soc15_common_get_clockgating_state(void *handle, u32 *flags) 1413 { 1414 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1415 int data; 1416 1417 if (amdgpu_sriov_vf(adev)) 1418 *flags = 0; 1419 1420 adev->nbio_funcs->get_clockgating_state(adev, flags); 1421 1422 /* AMD_CG_SUPPORT_HDP_LS */ 1423 data = RREG32(SOC15_REG_OFFSET(HDP, 0, mmHDP_MEM_POWER_LS)); 1424 if (data & HDP_MEM_POWER_LS__LS_ENABLE_MASK) 1425 *flags |= AMD_CG_SUPPORT_HDP_LS; 1426 1427 /* AMD_CG_SUPPORT_DRM_MGCG */ 1428 data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_CGTT_CTRL0)); 1429 if (!(data & 0x01000000)) 1430 *flags |= AMD_CG_SUPPORT_DRM_MGCG; 1431 1432 /* AMD_CG_SUPPORT_DRM_LS */ 1433 data = RREG32(SOC15_REG_OFFSET(MP0, 0, mmMP0_MISC_LIGHT_SLEEP_CTRL)); 1434 if (data & 0x1) 1435 *flags |= AMD_CG_SUPPORT_DRM_LS; 1436 1437 /* AMD_CG_SUPPORT_ROM_MGCG */ 1438 data = RREG32(SOC15_REG_OFFSET(SMUIO, 0, mmCGTT_ROM_CLK_CTRL0)); 1439 if (!(data & CGTT_ROM_CLK_CTRL0__SOFT_OVERRIDE0_MASK)) 1440 *flags |= AMD_CG_SUPPORT_ROM_MGCG; 1441 1442 adev->df_funcs->get_clockgating_state(adev, flags); 1443 } 1444 1445 static int soc15_common_set_powergating_state(void *handle, 1446 enum amd_powergating_state state) 1447 { 1448 /* todo */ 1449 return 0; 1450 } 1451 1452 const struct amd_ip_funcs soc15_common_ip_funcs = { 1453 .name = "soc15_common", 1454 .early_init = soc15_common_early_init, 1455 .late_init = soc15_common_late_init, 1456 .sw_init = soc15_common_sw_init, 1457 .sw_fini = soc15_common_sw_fini, 1458 .hw_init = soc15_common_hw_init, 1459 .hw_fini = soc15_common_hw_fini, 1460 .suspend = soc15_common_suspend, 1461 .resume = soc15_common_resume, 1462 .is_idle = soc15_common_is_idle, 1463 .wait_for_idle = soc15_common_wait_for_idle, 1464 .soft_reset = soc15_common_soft_reset, 1465 .set_clockgating_state = soc15_common_set_clockgating_state, 1466 .set_powergating_state = soc15_common_set_powergating_state, 1467 .get_clockgating_state= soc15_common_get_clockgating_state, 1468 }; 1469