1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "mmhub_v1_8.h" 25 26 #include "mmhub/mmhub_1_8_0_offset.h" 27 #include "mmhub/mmhub_1_8_0_sh_mask.h" 28 #include "vega10_enum.h" 29 30 #include "soc15_common.h" 31 #include "soc15.h" 32 #include "amdgpu_ras.h" 33 #include "amdgpu_psp.h" 34 35 #define regVM_L2_CNTL3_DEFAULT 0x80100007 36 #define regVM_L2_CNTL4_DEFAULT 0x000000c1 37 38 static u64 mmhub_v1_8_get_fb_location(struct amdgpu_device *adev) 39 { 40 u64 base = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE); 41 u64 top = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP); 42 43 base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 44 base <<= 24; 45 46 top &= MC_VM_FB_LOCATION_TOP__FB_TOP_MASK; 47 top <<= 24; 48 49 adev->gmc.fb_start = base; 50 adev->gmc.fb_end = top; 51 52 return base; 53 } 54 55 static void mmhub_v1_8_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 56 uint64_t page_table_base) 57 { 58 struct amdgpu_vmhub *hub; 59 u32 inst_mask; 60 int i; 61 62 inst_mask = adev->aid_mask; 63 for_each_inst(i, inst_mask) { 64 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 65 WREG32_SOC15_OFFSET(MMHUB, i, 66 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 67 hub->ctx_addr_distance * vmid, 68 lower_32_bits(page_table_base)); 69 70 WREG32_SOC15_OFFSET(MMHUB, i, 71 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 72 hub->ctx_addr_distance * vmid, 73 upper_32_bits(page_table_base)); 74 } 75 } 76 77 static void mmhub_v1_8_init_gart_aperture_regs(struct amdgpu_device *adev) 78 { 79 uint64_t pt_base; 80 u32 inst_mask; 81 int i; 82 83 if (adev->gmc.pdb0_bo) 84 pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo); 85 else 86 pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 87 88 mmhub_v1_8_setup_vm_pt_regs(adev, 0, pt_base); 89 90 /* If use GART for FB translation, vmid0 page table covers both 91 * vram and system memory (gart) 92 */ 93 inst_mask = adev->aid_mask; 94 for_each_inst(i, inst_mask) { 95 if (adev->gmc.pdb0_bo) { 96 WREG32_SOC15(MMHUB, i, 97 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 98 (u32)(adev->gmc.fb_start >> 12)); 99 WREG32_SOC15(MMHUB, i, 100 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 101 (u32)(adev->gmc.fb_start >> 44)); 102 103 WREG32_SOC15(MMHUB, i, 104 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 105 (u32)(adev->gmc.gart_end >> 12)); 106 WREG32_SOC15(MMHUB, i, 107 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 108 (u32)(adev->gmc.gart_end >> 44)); 109 110 } else { 111 WREG32_SOC15(MMHUB, i, 112 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 113 (u32)(adev->gmc.gart_start >> 12)); 114 WREG32_SOC15(MMHUB, i, 115 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 116 (u32)(adev->gmc.gart_start >> 44)); 117 118 WREG32_SOC15(MMHUB, i, 119 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 120 (u32)(adev->gmc.gart_end >> 12)); 121 WREG32_SOC15(MMHUB, i, 122 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 123 (u32)(adev->gmc.gart_end >> 44)); 124 } 125 } 126 } 127 128 static void mmhub_v1_8_init_system_aperture_regs(struct amdgpu_device *adev) 129 { 130 uint32_t tmp, inst_mask; 131 uint64_t value; 132 int i; 133 134 if (amdgpu_sriov_vf(adev)) 135 return; 136 137 inst_mask = adev->aid_mask; 138 for_each_inst(i, inst_mask) { 139 /* Program the AGP BAR */ 140 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BASE, 0); 141 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 142 adev->gmc.agp_start >> 24); 143 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 144 adev->gmc.agp_end >> 24); 145 146 /* Program the system aperture low logical page number. */ 147 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 148 min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); 149 150 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 151 max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18); 152 153 /* In the case squeezing vram into GART aperture, we don't use 154 * FB aperture and AGP aperture. Disable them. 155 */ 156 if (adev->gmc.pdb0_bo) { 157 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 0xFFFFFF); 158 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 0); 159 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_TOP, 0); 160 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_BASE, 161 0x00FFFFFF); 162 WREG32_SOC15(MMHUB, i, 163 regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 164 0x3FFFFFFF); 165 WREG32_SOC15(MMHUB, i, 166 regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0); 167 } 168 169 /* Set default page address. */ 170 value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr); 171 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 172 (u32)(value >> 12)); 173 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 174 (u32)(value >> 44)); 175 176 /* Program "protection fault". */ 177 WREG32_SOC15(MMHUB, i, 178 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 179 (u32)(adev->dummy_page_addr >> 12)); 180 WREG32_SOC15(MMHUB, i, 181 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 182 (u32)((u64)adev->dummy_page_addr >> 44)); 183 184 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2); 185 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2, 186 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 187 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2, tmp); 188 } 189 } 190 191 static void mmhub_v1_8_init_tlb_regs(struct amdgpu_device *adev) 192 { 193 uint32_t tmp, inst_mask; 194 int i; 195 196 if (amdgpu_sriov_reg_indirect_l1_tlb_cntl(adev)) { 197 tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL); 198 199 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 200 1); 201 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 202 SYSTEM_ACCESS_MODE, 3); 203 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 204 ENABLE_ADVANCED_DRIVER_MODEL, 1); 205 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 206 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 207 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 208 MTYPE, MTYPE_UC);/* XXX for emulation. */ 209 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); 210 211 psp_reg_program_no_ring(&adev->psp, tmp, PSP_REG_MMHUB_L1_TLB_CNTL); 212 } else { 213 inst_mask = adev->aid_mask; 214 for_each_inst(i, inst_mask) { 215 tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); 216 217 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 218 1); 219 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 220 SYSTEM_ACCESS_MODE, 3); 221 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 222 ENABLE_ADVANCED_DRIVER_MODEL, 1); 223 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 224 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 225 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 226 MTYPE, MTYPE_UC);/* XXX for emulation. */ 227 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); 228 229 WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); 230 } 231 } 232 } 233 234 /* Set snoop bit for SDMA so that SDMA writes probe-invalidates RW lines */ 235 static void mmhub_v1_8_init_snoop_override_regs(struct amdgpu_device *adev) 236 { 237 uint32_t tmp, inst_mask; 238 int i, j; 239 uint32_t distance = regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE - 240 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE; 241 242 inst_mask = adev->aid_mask; 243 for_each_inst(i, inst_mask) { 244 for (j = 0; j < 5; j++) { /* DAGB instances */ 245 tmp = RREG32_SOC15_OFFSET(MMHUB, i, 246 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance); 247 tmp |= (1 << 15); /* SDMA client is BIT15 */ 248 WREG32_SOC15_OFFSET(MMHUB, i, 249 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance, tmp); 250 251 tmp = RREG32_SOC15_OFFSET(MMHUB, i, 252 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance); 253 tmp |= (1 << 15); 254 WREG32_SOC15_OFFSET(MMHUB, i, 255 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance, tmp); 256 } 257 } 258 } 259 260 static void mmhub_v1_8_init_cache_regs(struct amdgpu_device *adev) 261 { 262 uint32_t tmp, inst_mask; 263 int i; 264 265 if (amdgpu_sriov_vf(adev)) 266 return; 267 268 /* Setup L2 cache */ 269 inst_mask = adev->aid_mask; 270 for_each_inst(i, inst_mask) { 271 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL); 272 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); 273 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 274 ENABLE_L2_FRAGMENT_PROCESSING, 1); 275 /* XXX for emulation, Refer to closed source code.*/ 276 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 277 L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 278 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 279 0); 280 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 281 CONTEXT1_IDENTITY_ACCESS_MODE, 1); 282 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 283 IDENTITY_MODE_FRAGMENT_SIZE, 0); 284 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL, tmp); 285 286 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL2); 287 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 288 1); 289 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 290 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL2, tmp); 291 292 tmp = regVM_L2_CNTL3_DEFAULT; 293 if (adev->gmc.translate_further) { 294 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12); 295 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 296 L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 297 } else { 298 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9); 299 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 300 L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 301 } 302 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL3, tmp); 303 304 tmp = regVM_L2_CNTL4_DEFAULT; 305 /* For AMD APP APUs setup WC memory */ 306 if (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu) { 307 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 308 VMC_TAP_PDE_REQUEST_PHYSICAL, 1); 309 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 310 VMC_TAP_PTE_REQUEST_PHYSICAL, 1); 311 } else { 312 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 313 VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 314 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 315 VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 316 } 317 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL4, tmp); 318 } 319 } 320 321 static void mmhub_v1_8_enable_system_domain(struct amdgpu_device *adev) 322 { 323 uint32_t tmp, inst_mask; 324 int i; 325 326 inst_mask = adev->aid_mask; 327 for_each_inst(i, inst_mask) { 328 tmp = RREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL); 329 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 330 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 331 adev->gmc.vmid0_page_table_depth); 332 tmp = REG_SET_FIELD(tmp, 333 VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE, 334 adev->gmc.vmid0_page_table_block_size); 335 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, 336 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 337 WREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL, tmp); 338 } 339 } 340 341 static void mmhub_v1_8_disable_identity_aperture(struct amdgpu_device *adev) 342 { 343 u32 inst_mask; 344 int i; 345 346 if (amdgpu_sriov_vf(adev)) 347 return; 348 349 inst_mask = adev->aid_mask; 350 for_each_inst(i, inst_mask) { 351 WREG32_SOC15(MMHUB, i, 352 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 353 0XFFFFFFFF); 354 WREG32_SOC15(MMHUB, i, 355 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 356 0x0000000F); 357 358 WREG32_SOC15(MMHUB, i, 359 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 360 0); 361 WREG32_SOC15(MMHUB, i, 362 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 363 0); 364 365 WREG32_SOC15(MMHUB, i, 366 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); 367 WREG32_SOC15(MMHUB, i, 368 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); 369 } 370 } 371 372 static void mmhub_v1_8_setup_vmid_config(struct amdgpu_device *adev) 373 { 374 struct amdgpu_vmhub *hub; 375 unsigned int num_level, block_size; 376 uint32_t tmp, inst_mask; 377 int i, j; 378 379 num_level = adev->vm_manager.num_level; 380 block_size = adev->vm_manager.block_size; 381 if (adev->gmc.translate_further) 382 num_level -= 1; 383 else 384 block_size -= 9; 385 386 inst_mask = adev->aid_mask; 387 for_each_inst(j, inst_mask) { 388 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 389 for (i = 0; i <= 14; i++) { 390 tmp = RREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 391 i * hub->ctx_distance); 392 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 393 ENABLE_CONTEXT, 1); 394 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 395 PAGE_TABLE_DEPTH, num_level); 396 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 397 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 398 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 399 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 400 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 401 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 402 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 403 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 404 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 405 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 406 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 407 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 408 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 409 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 410 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 411 PAGE_TABLE_BLOCK_SIZE, 412 block_size); 413 /* On 9.4.3, XNACK can be enabled in the SQ 414 * per-process. Retry faults need to be enabled for 415 * that to work. 416 */ 417 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 418 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1); 419 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 420 i * hub->ctx_distance, tmp); 421 WREG32_SOC15_OFFSET(MMHUB, j, 422 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 423 i * hub->ctx_addr_distance, 0); 424 WREG32_SOC15_OFFSET(MMHUB, j, 425 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 426 i * hub->ctx_addr_distance, 0); 427 WREG32_SOC15_OFFSET(MMHUB, j, 428 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 429 i * hub->ctx_addr_distance, 430 lower_32_bits(adev->vm_manager.max_pfn - 1)); 431 WREG32_SOC15_OFFSET(MMHUB, j, 432 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 433 i * hub->ctx_addr_distance, 434 upper_32_bits(adev->vm_manager.max_pfn - 1)); 435 } 436 } 437 } 438 439 static void mmhub_v1_8_program_invalidation(struct amdgpu_device *adev) 440 { 441 struct amdgpu_vmhub *hub; 442 u32 i, j, inst_mask; 443 444 inst_mask = adev->aid_mask; 445 for_each_inst(j, inst_mask) { 446 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 447 for (i = 0; i < 18; ++i) { 448 WREG32_SOC15_OFFSET(MMHUB, j, 449 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 450 i * hub->eng_addr_distance, 0xffffffff); 451 WREG32_SOC15_OFFSET(MMHUB, j, 452 regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 453 i * hub->eng_addr_distance, 0x1f); 454 } 455 } 456 } 457 458 static int mmhub_v1_8_gart_enable(struct amdgpu_device *adev) 459 { 460 /* GART Enable. */ 461 mmhub_v1_8_init_gart_aperture_regs(adev); 462 mmhub_v1_8_init_system_aperture_regs(adev); 463 mmhub_v1_8_init_tlb_regs(adev); 464 mmhub_v1_8_init_cache_regs(adev); 465 mmhub_v1_8_init_snoop_override_regs(adev); 466 467 mmhub_v1_8_enable_system_domain(adev); 468 mmhub_v1_8_disable_identity_aperture(adev); 469 mmhub_v1_8_setup_vmid_config(adev); 470 mmhub_v1_8_program_invalidation(adev); 471 472 return 0; 473 } 474 475 static void mmhub_v1_8_disable_l1_tlb(struct amdgpu_device *adev) 476 { 477 u32 tmp; 478 u32 i, inst_mask; 479 480 if (amdgpu_sriov_reg_indirect_l1_tlb_cntl(adev)) { 481 tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL); 482 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 483 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 484 ENABLE_ADVANCED_DRIVER_MODEL, 0); 485 psp_reg_program_no_ring(&adev->psp, tmp, PSP_REG_MMHUB_L1_TLB_CNTL); 486 } else { 487 inst_mask = adev->aid_mask; 488 for_each_inst(i, inst_mask) { 489 tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); 490 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 491 0); 492 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 493 ENABLE_ADVANCED_DRIVER_MODEL, 0); 494 WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); 495 } 496 } 497 } 498 499 static void mmhub_v1_8_gart_disable(struct amdgpu_device *adev) 500 { 501 struct amdgpu_vmhub *hub; 502 u32 tmp; 503 u32 i, j, inst_mask; 504 505 /* Disable all tables */ 506 inst_mask = adev->aid_mask; 507 for_each_inst(j, inst_mask) { 508 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 509 for (i = 0; i < 16; i++) 510 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT0_CNTL, 511 i * hub->ctx_distance, 0); 512 if (!amdgpu_sriov_vf(adev)) { 513 /* Setup L2 cache */ 514 tmp = RREG32_SOC15(MMHUB, j, regVM_L2_CNTL); 515 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 516 0); 517 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL, tmp); 518 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL3, 0); 519 } 520 } 521 522 mmhub_v1_8_disable_l1_tlb(adev); 523 } 524 525 /** 526 * mmhub_v1_8_set_fault_enable_default - update GART/VM fault handling 527 * 528 * @adev: amdgpu_device pointer 529 * @value: true redirects VM faults to the default page 530 */ 531 static void mmhub_v1_8_set_fault_enable_default(struct amdgpu_device *adev, bool value) 532 { 533 u32 tmp, inst_mask; 534 int i; 535 536 if (amdgpu_sriov_vf(adev)) 537 return; 538 539 inst_mask = adev->aid_mask; 540 for_each_inst(i, inst_mask) { 541 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL); 542 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 543 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 544 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 545 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 546 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 547 PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 548 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 549 PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 550 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 551 TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 552 value); 553 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 554 NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 555 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 556 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 557 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 558 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 559 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 560 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 561 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 562 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 563 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 564 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 565 if (!value) { 566 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 567 CRASH_ON_NO_RETRY_FAULT, 1); 568 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 569 CRASH_ON_RETRY_FAULT, 1); 570 } 571 572 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL, tmp); 573 } 574 } 575 576 static void mmhub_v1_8_init(struct amdgpu_device *adev) 577 { 578 struct amdgpu_vmhub *hub; 579 u32 inst_mask; 580 int i; 581 582 inst_mask = adev->aid_mask; 583 for_each_inst(i, inst_mask) { 584 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 585 586 hub->ctx0_ptb_addr_lo32 = SOC15_REG_OFFSET(MMHUB, i, 587 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 588 hub->ctx0_ptb_addr_hi32 = SOC15_REG_OFFSET(MMHUB, i, 589 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 590 hub->vm_inv_eng0_req = 591 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_REQ); 592 hub->vm_inv_eng0_ack = 593 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_ACK); 594 hub->vm_context0_cntl = 595 SOC15_REG_OFFSET(MMHUB, i, regVM_CONTEXT0_CNTL); 596 hub->vm_l2_pro_fault_status = SOC15_REG_OFFSET(MMHUB, i, 597 regVM_L2_PROTECTION_FAULT_STATUS); 598 hub->vm_l2_pro_fault_cntl = SOC15_REG_OFFSET(MMHUB, i, 599 regVM_L2_PROTECTION_FAULT_CNTL); 600 601 hub->ctx_distance = regVM_CONTEXT1_CNTL - regVM_CONTEXT0_CNTL; 602 hub->ctx_addr_distance = 603 regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 604 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 605 hub->eng_distance = regVM_INVALIDATE_ENG1_REQ - 606 regVM_INVALIDATE_ENG0_REQ; 607 hub->eng_addr_distance = regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 608 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 609 } 610 } 611 612 static int mmhub_v1_8_set_clockgating(struct amdgpu_device *adev, 613 enum amd_clockgating_state state) 614 { 615 return 0; 616 } 617 618 static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags) 619 { 620 621 } 622 623 const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = { 624 .get_fb_location = mmhub_v1_8_get_fb_location, 625 .init = mmhub_v1_8_init, 626 .gart_enable = mmhub_v1_8_gart_enable, 627 .set_fault_enable_default = mmhub_v1_8_set_fault_enable_default, 628 .gart_disable = mmhub_v1_8_gart_disable, 629 .setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs, 630 .set_clockgating = mmhub_v1_8_set_clockgating, 631 .get_clockgating = mmhub_v1_8_get_clockgating, 632 }; 633 634 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = { 635 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_CE_ERR_STATUS_LO, regMMEA0_CE_ERR_STATUS_HI), 636 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, 637 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_CE_ERR_STATUS_LO, regMMEA1_CE_ERR_STATUS_HI), 638 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, 639 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_CE_ERR_STATUS_LO, regMMEA2_CE_ERR_STATUS_HI), 640 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, 641 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_CE_ERR_STATUS_LO, regMMEA3_CE_ERR_STATUS_HI), 642 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, 643 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_CE_ERR_STATUS_LO, regMMEA4_CE_ERR_STATUS_HI), 644 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, 645 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_CE_ERR_STATUS_LO, regMM_CANE_CE_ERR_STATUS_HI), 646 1, 0, "MM_CANE"}, 647 }; 648 649 static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ue_reg_list[] = { 650 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_UE_ERR_STATUS_LO, regMMEA0_UE_ERR_STATUS_HI), 651 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, 652 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_UE_ERR_STATUS_LO, regMMEA1_UE_ERR_STATUS_HI), 653 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, 654 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_UE_ERR_STATUS_LO, regMMEA2_UE_ERR_STATUS_HI), 655 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, 656 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_UE_ERR_STATUS_LO, regMMEA3_UE_ERR_STATUS_HI), 657 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, 658 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_UE_ERR_STATUS_LO, regMMEA4_UE_ERR_STATUS_HI), 659 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, 660 {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_UE_ERR_STATUS_LO, regMM_CANE_UE_ERR_STATUS_HI), 661 1, 0, "MM_CANE"}, 662 }; 663 664 static const struct amdgpu_ras_memory_id_entry mmhub_v1_8_ras_memory_list[] = { 665 {AMDGPU_MMHUB_WGMI_PAGEMEM, "MMEA_WGMI_PAGEMEM"}, 666 {AMDGPU_MMHUB_RGMI_PAGEMEM, "MMEA_RGMI_PAGEMEM"}, 667 {AMDGPU_MMHUB_WDRAM_PAGEMEM, "MMEA_WDRAM_PAGEMEM"}, 668 {AMDGPU_MMHUB_RDRAM_PAGEMEM, "MMEA_RDRAM_PAGEMEM"}, 669 {AMDGPU_MMHUB_WIO_CMDMEM, "MMEA_WIO_CMDMEM"}, 670 {AMDGPU_MMHUB_RIO_CMDMEM, "MMEA_RIO_CMDMEM"}, 671 {AMDGPU_MMHUB_WGMI_CMDMEM, "MMEA_WGMI_CMDMEM"}, 672 {AMDGPU_MMHUB_RGMI_CMDMEM, "MMEA_RGMI_CMDMEM"}, 673 {AMDGPU_MMHUB_WDRAM_CMDMEM, "MMEA_WDRAM_CMDMEM"}, 674 {AMDGPU_MMHUB_RDRAM_CMDMEM, "MMEA_RDRAM_CMDMEM"}, 675 {AMDGPU_MMHUB_MAM_DMEM0, "MMEA_MAM_DMEM0"}, 676 {AMDGPU_MMHUB_MAM_DMEM1, "MMEA_MAM_DMEM1"}, 677 {AMDGPU_MMHUB_MAM_DMEM2, "MMEA_MAM_DMEM2"}, 678 {AMDGPU_MMHUB_MAM_DMEM3, "MMEA_MAM_DMEM3"}, 679 {AMDGPU_MMHUB_WRET_TAGMEM, "MMEA_WRET_TAGMEM"}, 680 {AMDGPU_MMHUB_RRET_TAGMEM, "MMEA_RRET_TAGMEM"}, 681 {AMDGPU_MMHUB_WIO_DATAMEM, "MMEA_WIO_DATAMEM"}, 682 {AMDGPU_MMHUB_WGMI_DATAMEM, "MMEA_WGMI_DATAMEM"}, 683 {AMDGPU_MMHUB_WDRAM_DATAMEM, "MMEA_WDRAM_DATAMEM"}, 684 }; 685 686 static void mmhub_v1_8_inst_query_ras_error_count(struct amdgpu_device *adev, 687 uint32_t mmhub_inst, 688 void *ras_err_status) 689 { 690 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; 691 unsigned long ue_count = 0, ce_count = 0; 692 693 /* NOTE: mmhub is converted by aid_mask and the range is 0-3, 694 * which can be used as die ID directly */ 695 struct amdgpu_smuio_mcm_config_info mcm_info = { 696 .socket_id = adev->smuio.funcs->get_socket_id(adev), 697 .die_id = mmhub_inst, 698 }; 699 700 amdgpu_ras_inst_query_ras_error_count(adev, 701 mmhub_v1_8_ce_reg_list, 702 ARRAY_SIZE(mmhub_v1_8_ce_reg_list), 703 mmhub_v1_8_ras_memory_list, 704 ARRAY_SIZE(mmhub_v1_8_ras_memory_list), 705 mmhub_inst, 706 AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE, 707 &ce_count); 708 amdgpu_ras_inst_query_ras_error_count(adev, 709 mmhub_v1_8_ue_reg_list, 710 ARRAY_SIZE(mmhub_v1_8_ue_reg_list), 711 mmhub_v1_8_ras_memory_list, 712 ARRAY_SIZE(mmhub_v1_8_ras_memory_list), 713 mmhub_inst, 714 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 715 &ue_count); 716 717 amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count); 718 amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count); 719 } 720 721 static void mmhub_v1_8_query_ras_error_count(struct amdgpu_device *adev, 722 void *ras_err_status) 723 { 724 uint32_t inst_mask; 725 uint32_t i; 726 727 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 728 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 729 return; 730 } 731 732 inst_mask = adev->aid_mask; 733 for_each_inst(i, inst_mask) 734 mmhub_v1_8_inst_query_ras_error_count(adev, i, ras_err_status); 735 } 736 737 static void mmhub_v1_8_inst_reset_ras_error_count(struct amdgpu_device *adev, 738 uint32_t mmhub_inst) 739 { 740 amdgpu_ras_inst_reset_ras_error_count(adev, 741 mmhub_v1_8_ce_reg_list, 742 ARRAY_SIZE(mmhub_v1_8_ce_reg_list), 743 mmhub_inst); 744 amdgpu_ras_inst_reset_ras_error_count(adev, 745 mmhub_v1_8_ue_reg_list, 746 ARRAY_SIZE(mmhub_v1_8_ue_reg_list), 747 mmhub_inst); 748 } 749 750 static void mmhub_v1_8_reset_ras_error_count(struct amdgpu_device *adev) 751 { 752 uint32_t inst_mask; 753 uint32_t i; 754 755 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { 756 dev_warn(adev->dev, "MMHUB RAS is not supported\n"); 757 return; 758 } 759 760 inst_mask = adev->aid_mask; 761 for_each_inst(i, inst_mask) 762 mmhub_v1_8_inst_reset_ras_error_count(adev, i); 763 } 764 765 static const struct amdgpu_ras_block_hw_ops mmhub_v1_8_ras_hw_ops = { 766 .query_ras_error_count = mmhub_v1_8_query_ras_error_count, 767 .reset_ras_error_count = mmhub_v1_8_reset_ras_error_count, 768 }; 769 770 static int mmhub_v1_8_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, 771 enum aca_smu_type type, void *data) 772 { 773 struct aca_bank_info info; 774 u64 misc0; 775 int ret; 776 777 ret = aca_bank_info_decode(bank, &info); 778 if (ret) 779 return ret; 780 781 misc0 = bank->regs[ACA_REG_IDX_MISC0]; 782 switch (type) { 783 case ACA_SMU_TYPE_UE: 784 bank->aca_err_type = ACA_ERROR_TYPE_UE; 785 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, 786 1ULL); 787 break; 788 case ACA_SMU_TYPE_CE: 789 bank->aca_err_type = ACA_ERROR_TYPE_CE; 790 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 791 ACA_REG__MISC0__ERRCNT(misc0)); 792 break; 793 default: 794 return -EINVAL; 795 } 796 797 return ret; 798 } 799 800 /* reference to smu driver if header file */ 801 static int mmhub_v1_8_err_codes[] = { 802 0, 1, 2, 3, 4, /* CODE_DAGB0 - 4 */ 803 5, 6, 7, 8, 9, /* CODE_EA0 - 4 */ 804 10, /* CODE_UTCL2_ROUTER */ 805 11, /* CODE_VML2 */ 806 12, /* CODE_VML2_WALKER */ 807 13, /* CODE_MMCANE */ 808 }; 809 810 static bool mmhub_v1_8_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, 811 enum aca_smu_type type, void *data) 812 { 813 u32 instlo; 814 815 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); 816 instlo &= GENMASK(31, 1); 817 818 if (instlo != mmSMNAID_AID0_MCA_SMU) 819 return false; 820 821 if (aca_bank_check_error_codes(handle->adev, bank, 822 mmhub_v1_8_err_codes, 823 ARRAY_SIZE(mmhub_v1_8_err_codes))) 824 return false; 825 826 return true; 827 } 828 829 static const struct aca_bank_ops mmhub_v1_8_aca_bank_ops = { 830 .aca_bank_parser = mmhub_v1_8_aca_bank_parser, 831 .aca_bank_is_valid = mmhub_v1_8_aca_bank_is_valid, 832 }; 833 834 static const struct aca_info mmhub_v1_8_aca_info = { 835 .hwip = ACA_HWIP_TYPE_SMU, 836 .mask = ACA_ERROR_UE_MASK, 837 .bank_ops = &mmhub_v1_8_aca_bank_ops, 838 }; 839 840 static int mmhub_v1_8_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 841 { 842 int r; 843 844 r = amdgpu_ras_block_late_init(adev, ras_block); 845 if (r) 846 return r; 847 848 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__MMHUB, 849 &mmhub_v1_8_aca_info, NULL); 850 if (r) 851 goto late_fini; 852 853 return 0; 854 855 late_fini: 856 amdgpu_ras_block_late_fini(adev, ras_block); 857 858 return r; 859 } 860 861 struct amdgpu_mmhub_ras mmhub_v1_8_ras = { 862 .ras_block = { 863 .hw_ops = &mmhub_v1_8_ras_hw_ops, 864 .ras_late_init = mmhub_v1_8_ras_late_init, 865 }, 866 }; 867