1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "mmhub_v1_8.h" 25 26 #include "mmhub/mmhub_1_8_0_offset.h" 27 #include "mmhub/mmhub_1_8_0_sh_mask.h" 28 #include "vega10_enum.h" 29 30 #include "soc15_common.h" 31 #include "soc15.h" 32 #include "amdgpu_psp.h" 33 34 #define regVM_L2_CNTL3_DEFAULT 0x80100007 35 #define regVM_L2_CNTL4_DEFAULT 0x000000c1 36 37 static u64 mmhub_v1_8_get_fb_location(struct amdgpu_device *adev) 38 { 39 u64 base = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_BASE); 40 u64 top = RREG32_SOC15(MMHUB, 0, regMC_VM_FB_LOCATION_TOP); 41 42 base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 43 base <<= 24; 44 45 top &= MC_VM_FB_LOCATION_TOP__FB_TOP_MASK; 46 top <<= 24; 47 48 adev->gmc.fb_start = base; 49 adev->gmc.fb_end = top; 50 51 return base; 52 } 53 54 static void mmhub_v1_8_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 55 uint64_t page_table_base) 56 { 57 struct amdgpu_vmhub *hub; 58 u32 inst_mask; 59 int i; 60 61 inst_mask = adev->aid_mask; 62 for_each_inst(i, inst_mask) { 63 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 64 WREG32_SOC15_OFFSET(MMHUB, i, 65 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 66 hub->ctx_addr_distance * vmid, 67 lower_32_bits(page_table_base)); 68 69 WREG32_SOC15_OFFSET(MMHUB, i, 70 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 71 hub->ctx_addr_distance * vmid, 72 upper_32_bits(page_table_base)); 73 } 74 } 75 76 static void mmhub_v1_8_init_gart_aperture_regs(struct amdgpu_device *adev) 77 { 78 uint64_t gart_start = amdgpu_virt_xgmi_migrate_enabled(adev) ? 79 adev->gmc.vram_start : adev->gmc.fb_start; 80 uint64_t pt_base; 81 u32 inst_mask; 82 int i; 83 84 if (adev->gmc.pdb0_bo) 85 pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo); 86 else 87 pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 88 89 mmhub_v1_8_setup_vm_pt_regs(adev, 0, pt_base); 90 91 /* If use GART for FB translation, vmid0 page table covers both 92 * vram and system memory (gart) 93 */ 94 inst_mask = adev->aid_mask; 95 for_each_inst(i, inst_mask) { 96 if (adev->gmc.pdb0_bo) { 97 WREG32_SOC15(MMHUB, i, 98 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 99 (u32)(gart_start >> 12)); 100 WREG32_SOC15(MMHUB, i, 101 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 102 (u32)(gart_start >> 44)); 103 104 WREG32_SOC15(MMHUB, i, 105 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 106 (u32)(adev->gmc.gart_end >> 12)); 107 WREG32_SOC15(MMHUB, i, 108 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 109 (u32)(adev->gmc.gart_end >> 44)); 110 111 } else { 112 WREG32_SOC15(MMHUB, i, 113 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 114 (u32)(adev->gmc.gart_start >> 12)); 115 WREG32_SOC15(MMHUB, i, 116 regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 117 (u32)(adev->gmc.gart_start >> 44)); 118 119 WREG32_SOC15(MMHUB, i, 120 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 121 (u32)(adev->gmc.gart_end >> 12)); 122 WREG32_SOC15(MMHUB, i, 123 regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 124 (u32)(adev->gmc.gart_end >> 44)); 125 } 126 } 127 } 128 129 static void mmhub_v1_8_init_system_aperture_regs(struct amdgpu_device *adev) 130 { 131 uint32_t tmp, inst_mask; 132 uint64_t value; 133 int i; 134 135 if (amdgpu_sriov_vf(adev)) 136 return; 137 138 inst_mask = adev->aid_mask; 139 for_each_inst(i, inst_mask) { 140 /* Program the AGP BAR */ 141 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BASE, 0); 142 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 143 adev->gmc.agp_start >> 24); 144 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 145 adev->gmc.agp_end >> 24); 146 147 /* Program the system aperture low logical page number. */ 148 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 149 min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); 150 151 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 152 max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18); 153 154 /* In the case squeezing vram into GART aperture, we don't use 155 * FB aperture and AGP aperture. Disable them. 156 */ 157 if (adev->gmc.pdb0_bo) { 158 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_BOT, 0xFFFFFF); 159 WREG32_SOC15(MMHUB, i, regMC_VM_AGP_TOP, 0); 160 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_TOP, 0); 161 WREG32_SOC15(MMHUB, i, regMC_VM_FB_LOCATION_BASE, 162 0x00FFFFFF); 163 WREG32_SOC15(MMHUB, i, 164 regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 165 0x3FFFFFFF); 166 WREG32_SOC15(MMHUB, i, 167 regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0); 168 } 169 170 /* Set default page address. */ 171 value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr); 172 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 173 (u32)(value >> 12)); 174 WREG32_SOC15(MMHUB, i, regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 175 (u32)(value >> 44)); 176 177 /* Program "protection fault". */ 178 WREG32_SOC15(MMHUB, i, 179 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 180 (u32)(adev->dummy_page_addr >> 12)); 181 WREG32_SOC15(MMHUB, i, 182 regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 183 (u32)((u64)adev->dummy_page_addr >> 44)); 184 185 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2); 186 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2, 187 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 188 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL2, tmp); 189 } 190 } 191 192 static void mmhub_v1_8_init_tlb_regs(struct amdgpu_device *adev) 193 { 194 uint32_t tmp, inst_mask; 195 int i; 196 197 if (amdgpu_sriov_reg_indirect_l1_tlb_cntl(adev)) { 198 tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL); 199 200 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 201 1); 202 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 203 SYSTEM_ACCESS_MODE, 3); 204 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 205 ENABLE_ADVANCED_DRIVER_MODEL, 1); 206 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 207 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 208 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 209 MTYPE, MTYPE_UC);/* XXX for emulation. */ 210 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); 211 212 psp_reg_program_no_ring(&adev->psp, tmp, PSP_REG_MMHUB_L1_TLB_CNTL); 213 } else { 214 inst_mask = adev->aid_mask; 215 for_each_inst(i, inst_mask) { 216 tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); 217 218 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 219 1); 220 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 221 SYSTEM_ACCESS_MODE, 3); 222 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 223 ENABLE_ADVANCED_DRIVER_MODEL, 1); 224 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 225 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 226 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 227 MTYPE, MTYPE_UC);/* XXX for emulation. */ 228 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1); 229 230 WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); 231 } 232 } 233 } 234 235 /* Set snoop bit for SDMA so that SDMA writes probe-invalidates RW lines */ 236 static void mmhub_v1_8_init_snoop_override_regs(struct amdgpu_device *adev) 237 { 238 uint32_t tmp, inst_mask; 239 int i, j; 240 uint32_t distance = regDAGB1_WRCLI_GPU_SNOOP_OVERRIDE - 241 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE; 242 243 if (amdgpu_sriov_vf(adev)) 244 return; 245 246 inst_mask = adev->aid_mask; 247 for_each_inst(i, inst_mask) { 248 for (j = 0; j < 5; j++) { /* DAGB instances */ 249 tmp = RREG32_SOC15_OFFSET(MMHUB, i, 250 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance); 251 tmp |= (1 << 15); /* SDMA client is BIT15 */ 252 WREG32_SOC15_OFFSET(MMHUB, i, 253 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE, j * distance, tmp); 254 255 tmp = RREG32_SOC15_OFFSET(MMHUB, i, 256 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance); 257 tmp |= (1 << 15); 258 WREG32_SOC15_OFFSET(MMHUB, i, 259 regDAGB0_WRCLI_GPU_SNOOP_OVERRIDE_VALUE, j * distance, tmp); 260 } 261 } 262 } 263 264 static void mmhub_v1_8_init_cache_regs(struct amdgpu_device *adev) 265 { 266 uint32_t tmp, inst_mask; 267 int i; 268 269 if (amdgpu_sriov_vf(adev)) 270 return; 271 272 /* Setup L2 cache */ 273 inst_mask = adev->aid_mask; 274 for_each_inst(i, inst_mask) { 275 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL); 276 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1); 277 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 278 ENABLE_L2_FRAGMENT_PROCESSING, 1); 279 /* XXX for emulation, Refer to closed source code.*/ 280 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 281 L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 282 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 283 0); 284 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 285 CONTEXT1_IDENTITY_ACCESS_MODE, 1); 286 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, 287 IDENTITY_MODE_FRAGMENT_SIZE, 0); 288 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL, tmp); 289 290 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_CNTL2); 291 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 292 1); 293 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 294 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL2, tmp); 295 296 tmp = regVM_L2_CNTL3_DEFAULT; 297 if (adev->gmc.translate_further) { 298 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12); 299 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 300 L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 301 } else { 302 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9); 303 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, 304 L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 305 } 306 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL3, tmp); 307 308 tmp = regVM_L2_CNTL4_DEFAULT; 309 /* For AMD APP APUs setup WC memory */ 310 if (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu) { 311 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 312 VMC_TAP_PDE_REQUEST_PHYSICAL, 1); 313 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 314 VMC_TAP_PTE_REQUEST_PHYSICAL, 1); 315 } else { 316 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 317 VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 318 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, 319 VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 320 } 321 WREG32_SOC15(MMHUB, i, regVM_L2_CNTL4, tmp); 322 } 323 } 324 325 static void mmhub_v1_8_enable_system_domain(struct amdgpu_device *adev) 326 { 327 uint32_t tmp, inst_mask; 328 int i; 329 330 inst_mask = adev->aid_mask; 331 for_each_inst(i, inst_mask) { 332 tmp = RREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL); 333 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 334 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 335 adev->gmc.vmid0_page_table_depth); 336 tmp = REG_SET_FIELD(tmp, 337 VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE, 338 adev->gmc.vmid0_page_table_block_size); 339 tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, 340 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 341 WREG32_SOC15(MMHUB, i, regVM_CONTEXT0_CNTL, tmp); 342 } 343 } 344 345 static void mmhub_v1_8_disable_identity_aperture(struct amdgpu_device *adev) 346 { 347 u32 inst_mask; 348 int i; 349 350 if (amdgpu_sriov_vf(adev)) 351 return; 352 353 inst_mask = adev->aid_mask; 354 for_each_inst(i, inst_mask) { 355 WREG32_SOC15(MMHUB, i, 356 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 357 0XFFFFFFFF); 358 WREG32_SOC15(MMHUB, i, 359 regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 360 0x0000000F); 361 362 WREG32_SOC15(MMHUB, i, 363 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 364 0); 365 WREG32_SOC15(MMHUB, i, 366 regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 367 0); 368 369 WREG32_SOC15(MMHUB, i, 370 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); 371 WREG32_SOC15(MMHUB, i, 372 regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); 373 } 374 } 375 376 static void mmhub_v1_8_setup_vmid_config(struct amdgpu_device *adev) 377 { 378 struct amdgpu_vmhub *hub; 379 unsigned int num_level, block_size; 380 uint32_t tmp, inst_mask; 381 int i, j; 382 383 num_level = adev->vm_manager.num_level; 384 block_size = adev->vm_manager.block_size; 385 if (adev->gmc.translate_further) 386 num_level -= 1; 387 else 388 block_size -= 9; 389 390 inst_mask = adev->aid_mask; 391 for_each_inst(j, inst_mask) { 392 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 393 for (i = 0; i <= 14; i++) { 394 tmp = RREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 395 i * hub->ctx_distance); 396 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 397 ENABLE_CONTEXT, 1); 398 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 399 PAGE_TABLE_DEPTH, num_level); 400 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 401 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 402 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 403 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 404 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 405 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 406 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 407 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 408 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 409 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 410 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 411 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 412 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 413 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 414 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 415 PAGE_TABLE_BLOCK_SIZE, 416 block_size); 417 /* On 9.4.3, XNACK can be enabled in the SQ 418 * per-process. Retry faults need to be enabled for 419 * that to work. 420 */ 421 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, 422 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 1); 423 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT1_CNTL, 424 i * hub->ctx_distance, tmp); 425 WREG32_SOC15_OFFSET(MMHUB, j, 426 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 427 i * hub->ctx_addr_distance, 0); 428 WREG32_SOC15_OFFSET(MMHUB, j, 429 regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 430 i * hub->ctx_addr_distance, 0); 431 WREG32_SOC15_OFFSET(MMHUB, j, 432 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 433 i * hub->ctx_addr_distance, 434 lower_32_bits(adev->vm_manager.max_pfn - 1)); 435 WREG32_SOC15_OFFSET(MMHUB, j, 436 regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 437 i * hub->ctx_addr_distance, 438 upper_32_bits(adev->vm_manager.max_pfn - 1)); 439 } 440 } 441 } 442 443 static void mmhub_v1_8_program_invalidation(struct amdgpu_device *adev) 444 { 445 struct amdgpu_vmhub *hub; 446 u32 i, j, inst_mask; 447 448 inst_mask = adev->aid_mask; 449 for_each_inst(j, inst_mask) { 450 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 451 for (i = 0; i < 18; ++i) { 452 WREG32_SOC15_OFFSET(MMHUB, j, 453 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 454 i * hub->eng_addr_distance, 0xffffffff); 455 WREG32_SOC15_OFFSET(MMHUB, j, 456 regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 457 i * hub->eng_addr_distance, 0x1f); 458 } 459 } 460 } 461 462 static int mmhub_v1_8_gart_enable(struct amdgpu_device *adev) 463 { 464 /* GART Enable. */ 465 mmhub_v1_8_init_gart_aperture_regs(adev); 466 mmhub_v1_8_init_system_aperture_regs(adev); 467 mmhub_v1_8_init_tlb_regs(adev); 468 mmhub_v1_8_init_cache_regs(adev); 469 mmhub_v1_8_init_snoop_override_regs(adev); 470 471 mmhub_v1_8_enable_system_domain(adev); 472 mmhub_v1_8_disable_identity_aperture(adev); 473 mmhub_v1_8_setup_vmid_config(adev); 474 mmhub_v1_8_program_invalidation(adev); 475 476 return 0; 477 } 478 479 static void mmhub_v1_8_disable_l1_tlb(struct amdgpu_device *adev) 480 { 481 u32 tmp; 482 u32 i, inst_mask; 483 484 if (amdgpu_sriov_reg_indirect_l1_tlb_cntl(adev)) { 485 tmp = RREG32_SOC15(MMHUB, 0, regMC_VM_MX_L1_TLB_CNTL); 486 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 487 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 488 ENABLE_ADVANCED_DRIVER_MODEL, 0); 489 psp_reg_program_no_ring(&adev->psp, tmp, PSP_REG_MMHUB_L1_TLB_CNTL); 490 } else { 491 inst_mask = adev->aid_mask; 492 for_each_inst(i, inst_mask) { 493 tmp = RREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL); 494 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 495 0); 496 tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, 497 ENABLE_ADVANCED_DRIVER_MODEL, 0); 498 WREG32_SOC15(MMHUB, i, regMC_VM_MX_L1_TLB_CNTL, tmp); 499 } 500 } 501 } 502 503 static void mmhub_v1_8_gart_disable(struct amdgpu_device *adev) 504 { 505 struct amdgpu_vmhub *hub; 506 u32 tmp; 507 u32 i, j, inst_mask; 508 509 /* Disable all tables */ 510 inst_mask = adev->aid_mask; 511 for_each_inst(j, inst_mask) { 512 hub = &adev->vmhub[AMDGPU_MMHUB0(j)]; 513 for (i = 0; i < 16; i++) 514 WREG32_SOC15_OFFSET(MMHUB, j, regVM_CONTEXT0_CNTL, 515 i * hub->ctx_distance, 0); 516 if (!amdgpu_sriov_vf(adev)) { 517 /* Setup L2 cache */ 518 tmp = RREG32_SOC15(MMHUB, j, regVM_L2_CNTL); 519 tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 520 0); 521 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL, tmp); 522 WREG32_SOC15(MMHUB, j, regVM_L2_CNTL3, 0); 523 } 524 } 525 526 mmhub_v1_8_disable_l1_tlb(adev); 527 } 528 529 /** 530 * mmhub_v1_8_set_fault_enable_default - update GART/VM fault handling 531 * 532 * @adev: amdgpu_device pointer 533 * @value: true redirects VM faults to the default page 534 */ 535 static void mmhub_v1_8_set_fault_enable_default(struct amdgpu_device *adev, bool value) 536 { 537 u32 tmp, inst_mask; 538 int i; 539 540 if (amdgpu_sriov_vf(adev)) 541 return; 542 543 inst_mask = adev->aid_mask; 544 for_each_inst(i, inst_mask) { 545 tmp = RREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL); 546 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 547 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 548 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 549 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 550 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 551 PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 552 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 553 PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 554 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 555 TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 556 value); 557 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 558 NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 559 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 560 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 561 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 562 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 563 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 564 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 565 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 566 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 567 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 568 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 569 if (!value) { 570 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 571 CRASH_ON_NO_RETRY_FAULT, 1); 572 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL, 573 CRASH_ON_RETRY_FAULT, 1); 574 } 575 576 WREG32_SOC15(MMHUB, i, regVM_L2_PROTECTION_FAULT_CNTL, tmp); 577 } 578 } 579 580 static void mmhub_v1_8_init(struct amdgpu_device *adev) 581 { 582 struct amdgpu_vmhub *hub; 583 u32 inst_mask; 584 int i; 585 586 inst_mask = adev->aid_mask; 587 for_each_inst(i, inst_mask) { 588 hub = &adev->vmhub[AMDGPU_MMHUB0(i)]; 589 590 hub->ctx0_ptb_addr_lo32 = SOC15_REG_OFFSET(MMHUB, i, 591 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 592 hub->ctx0_ptb_addr_hi32 = SOC15_REG_OFFSET(MMHUB, i, 593 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 594 hub->vm_inv_eng0_req = 595 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_REQ); 596 hub->vm_inv_eng0_ack = 597 SOC15_REG_OFFSET(MMHUB, i, regVM_INVALIDATE_ENG0_ACK); 598 hub->vm_context0_cntl = 599 SOC15_REG_OFFSET(MMHUB, i, regVM_CONTEXT0_CNTL); 600 hub->vm_l2_pro_fault_status = SOC15_REG_OFFSET(MMHUB, i, 601 regVM_L2_PROTECTION_FAULT_STATUS); 602 hub->vm_l2_pro_fault_cntl = SOC15_REG_OFFSET(MMHUB, i, 603 regVM_L2_PROTECTION_FAULT_CNTL); 604 605 hub->ctx_distance = regVM_CONTEXT1_CNTL - regVM_CONTEXT0_CNTL; 606 hub->ctx_addr_distance = 607 regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 608 regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 609 hub->eng_distance = regVM_INVALIDATE_ENG1_REQ - 610 regVM_INVALIDATE_ENG0_REQ; 611 hub->eng_addr_distance = regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 612 regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 613 } 614 } 615 616 static int mmhub_v1_8_set_clockgating(struct amdgpu_device *adev, 617 enum amd_clockgating_state state) 618 { 619 return 0; 620 } 621 622 static void mmhub_v1_8_get_clockgating(struct amdgpu_device *adev, u64 *flags) 623 { 624 625 } 626 627 const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = { 628 .get_fb_location = mmhub_v1_8_get_fb_location, 629 .init = mmhub_v1_8_init, 630 .gart_enable = mmhub_v1_8_gart_enable, 631 .set_fault_enable_default = mmhub_v1_8_set_fault_enable_default, 632 .gart_disable = mmhub_v1_8_gart_disable, 633 .setup_vm_pt_regs = mmhub_v1_8_setup_vm_pt_regs, 634 .set_clockgating = mmhub_v1_8_set_clockgating, 635 .get_clockgating = mmhub_v1_8_get_clockgating, 636 }; 637 638 struct amdgpu_mmhub_ras mmhub_v1_8_ras = { 639 .ras_block = { 640 .hw_ops = NULL, 641 }, 642 }; 643