1 /* 2 * Copyright 2025 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "gfxhub_v12_1.h" 26 27 #include "gc/gc_12_1_0_offset.h" 28 #include "gc/gc_12_1_0_sh_mask.h" 29 #include "soc_v1_0_enum.h" 30 31 #include "soc15_common.h" 32 33 #define regGCVM_L2_CNTL3_DEFAULT 0x80120007 34 #define regGCVM_L2_CNTL4_DEFAULT 0x000000c1 35 #define regGCVM_L2_CNTL5_DEFAULT 0x00003fe0 36 #define regGRBM_GFX_INDEX_DEFAULT 0xe0000000 37 38 39 static u64 gfxhub_v12_1_get_fb_location(struct amdgpu_device *adev) 40 { 41 u64 base; 42 43 base = RREG32_SOC15(GC, GET_INST(GC, 0), 44 regGCMC_VM_FB_LOCATION_BASE_LO32); 45 base &= GCMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32_MASK; 46 base <<= 24; 47 48 base |= ((u64)(GCMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1_MASK & 49 RREG32_SOC15(GC, GET_INST(GC, 0), 50 regGCMC_VM_FB_LOCATION_BASE_HI32)) << 56); 51 return base; 52 } 53 54 static u64 gfxhub_v12_1_get_mc_fb_offset(struct amdgpu_device *adev) 55 { 56 return (u64)(RREG32_SOC15(GC, GET_INST(GC, 0), 57 regGCMC_VM_FB_OFFSET) << 24); 58 } 59 60 static void gfxhub_v12_1_xcc_setup_vm_pt_regs(struct amdgpu_device *adev, 61 uint32_t vmid, 62 uint64_t page_table_base, 63 uint32_t xcc_mask) 64 { 65 struct amdgpu_vmhub *hub; 66 int i; 67 68 for_each_inst(i, xcc_mask) { 69 hub = &adev->vmhub[AMDGPU_GFXHUB(i)]; 70 WREG32_SOC15_OFFSET(GC, GET_INST(GC, i), 71 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 72 hub->ctx_addr_distance * vmid, 73 lower_32_bits(page_table_base)); 74 75 WREG32_SOC15_OFFSET(GC, GET_INST(GC, i), 76 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 77 hub->ctx_addr_distance * vmid, 78 upper_32_bits(page_table_base)); 79 } 80 } 81 82 static void gfxhub_v12_1_setup_vm_pt_regs(struct amdgpu_device *adev, 83 uint32_t vmid, 84 uint64_t page_table_base) 85 { 86 uint32_t xcc_mask; 87 88 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 89 gfxhub_v12_1_xcc_setup_vm_pt_regs(adev, vmid, page_table_base, 90 xcc_mask); 91 } 92 93 static void gfxhub_v12_1_xcc_init_gart_aperture_regs(struct amdgpu_device *adev, 94 uint32_t xcc_mask) 95 { 96 uint64_t pt_base; 97 int i; 98 99 if (adev->gmc.pdb0_bo) 100 pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo); 101 else 102 pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 103 104 gfxhub_v12_1_xcc_setup_vm_pt_regs(adev, 0, pt_base, xcc_mask); 105 106 /* If use GART for FB translation, vmid0 page table covers both 107 * vram and system memory (gart) 108 */ 109 for_each_inst(i, xcc_mask) { 110 if (adev->gmc.pdb0_bo) { 111 WREG32_SOC15(GC, GET_INST(GC, i), 112 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 113 (u32)(adev->gmc.fb_start >> 12)); 114 WREG32_SOC15(GC, GET_INST(GC, i), 115 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 116 (u32)(adev->gmc.fb_start >> 44)); 117 118 WREG32_SOC15(GC, GET_INST(GC, i), 119 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 120 (u32)(adev->gmc.gart_end >> 12)); 121 WREG32_SOC15(GC, GET_INST(GC, i), 122 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 123 (u32)(adev->gmc.gart_end >> 44)); 124 } else { 125 WREG32_SOC15(GC, GET_INST(GC, i), 126 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 127 (u32)(adev->gmc.gart_start >> 12)); 128 WREG32_SOC15(GC, GET_INST(GC, i), 129 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 130 (u32)(adev->gmc.gart_start >> 44)); 131 132 WREG32_SOC15(GC, GET_INST(GC, i), 133 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 134 (u32)(adev->gmc.gart_end >> 12)); 135 WREG32_SOC15(GC, GET_INST(GC, i), 136 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 137 (u32)(adev->gmc.gart_end >> 44)); 138 } 139 } 140 } 141 142 static void gfxhub_v12_1_xcc_init_system_aperture_regs(struct amdgpu_device *adev, 143 uint32_t xcc_mask) 144 { 145 uint64_t value; 146 uint32_t tmp; 147 int i; 148 149 for_each_inst(i, xcc_mask) { 150 /* Program the AGP BAR */ 151 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 152 regGCMC_VM_AGP_BASE_LO32, 0); 153 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 154 regGCMC_VM_AGP_BASE_HI32, 0); 155 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 156 regGCMC_VM_AGP_BOT_LO32, 157 lower_32_bits(adev->gmc.agp_start >> 24)); 158 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 159 regGCMC_VM_AGP_BOT_HI32, 160 upper_32_bits(adev->gmc.agp_start >> 24)); 161 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 162 regGCMC_VM_AGP_TOP_LO32, 163 lower_32_bits(adev->gmc.agp_end >> 24)); 164 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 165 regGCMC_VM_AGP_TOP_HI32, 166 upper_32_bits(adev->gmc.agp_end >> 24)); 167 168 if (!amdgpu_sriov_vf(adev)) { 169 /* Program the system aperture low logical page number. */ 170 WREG32_SOC15(GC, GET_INST(GC, i), 171 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32, 172 lower_32_bits(min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18)); 173 WREG32_SOC15(GC, GET_INST(GC, i), 174 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32, 175 upper_32_bits(min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18)); 176 WREG32_SOC15(GC, GET_INST(GC, i), 177 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 178 lower_32_bits(max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18)); 179 WREG32_SOC15(GC, GET_INST(GC, i), 180 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 181 upper_32_bits(max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18)); 182 183 /* Set default page address. */ 184 value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr); 185 WREG32_SOC15(GC, GET_INST(GC, i), 186 regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 187 (u32)(value >> 12)); 188 WREG32_SOC15(GC, GET_INST(GC, i), 189 regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 190 (u32)(value >> 44)); 191 192 /* Program "protection fault". */ 193 WREG32_SOC15(GC, GET_INST(GC, i), 194 regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 195 (u32)(adev->dummy_page_addr >> 12)); 196 WREG32_SOC15(GC, GET_INST(GC, i), 197 regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 198 (u32)((u64)adev->dummy_page_addr >> 44)); 199 200 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 201 regGCVM_L2_PROTECTION_FAULT_CNTL2); 202 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2, 203 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 204 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2, 205 ENABLE_RETRY_FAULT_INTERRUPT, 0x1); 206 WREG32_SOC15(GC, GET_INST(GC, i), 207 regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp); 208 } 209 210 /* In the case squeezing vram into GART aperture, we don't use 211 * FB aperture and AGP aperture. Disable them. 212 */ 213 if (adev->gmc.pdb0_bo) { 214 WREG32_SOC15(GC, GET_INST(GC, i), 215 regGCMC_VM_FB_LOCATION_TOP_LO32, 0); 216 WREG32_SOC15(GC, GET_INST(GC, i), 217 regGCMC_VM_FB_LOCATION_TOP_HI32, 0); 218 WREG32_SOC15(GC, GET_INST(GC, i), 219 regGCMC_VM_FB_LOCATION_BASE_LO32, 220 0xFFFFFFFF); 221 WREG32_SOC15(GC, GET_INST(GC, i), 222 regGCMC_VM_FB_LOCATION_BASE_HI32, 1); 223 WREG32_SOC15(GC, GET_INST(GC, i), 224 regGCMC_VM_AGP_TOP_LO32, 0); 225 WREG32_SOC15(GC, GET_INST(GC, i), 226 regGCMC_VM_AGP_TOP_HI32, 0); 227 WREG32_SOC15(GC, GET_INST(GC, i), 228 regGCMC_VM_AGP_BOT_LO32, 0xFFFFFFFF); 229 WREG32_SOC15(GC, GET_INST(GC, i), 230 regGCMC_VM_AGP_BOT_HI32, 1); 231 WREG32_SOC15(GC, GET_INST(GC, i), 232 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32, 233 0xFFFFFFFF); 234 WREG32_SOC15(GC, GET_INST(GC, i), 235 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32, 236 0x7F); 237 WREG32_SOC15(GC, GET_INST(GC, i), 238 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0); 239 WREG32_SOC15(GC, GET_INST(GC, i), 240 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0); 241 } 242 } 243 } 244 245 static void gfxhub_v12_1_xcc_init_tlb_regs(struct amdgpu_device *adev, 246 uint32_t xcc_mask) 247 { 248 uint32_t tmp; 249 int i; 250 251 for_each_inst(i, xcc_mask) { 252 /* Setup TLB control */ 253 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 254 regGCMC_VM_MX_L1_TLB_CNTL); 255 256 tmp = REG_SET_FIELD(tmp, 257 GCMC_VM_MX_L1_TLB_CNTL, 258 ENABLE_L1_TLB, 1); 259 tmp = REG_SET_FIELD(tmp, 260 GCMC_VM_MX_L1_TLB_CNTL, 261 SYSTEM_ACCESS_MODE, 3); 262 tmp = REG_SET_FIELD(tmp, 263 GCMC_VM_MX_L1_TLB_CNTL, 264 ENABLE_ADVANCED_DRIVER_MODEL, 1); 265 tmp = REG_SET_FIELD(tmp, 266 GCMC_VM_MX_L1_TLB_CNTL, 267 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 268 tmp = REG_SET_FIELD(tmp, 269 GCMC_VM_MX_L1_TLB_CNTL, 270 ECO_BITS, 0); 271 tmp = REG_SET_FIELD(tmp, 272 GCMC_VM_MX_L1_TLB_CNTL, 273 MTYPE, MTYPE_UC); 274 275 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 276 regGCMC_VM_MX_L1_TLB_CNTL, tmp); 277 } 278 } 279 280 static void gfxhub_v12_1_xcc_init_cache_regs(struct amdgpu_device *adev, 281 uint32_t xcc_mask) 282 { 283 uint32_t tmp; 284 int i; 285 286 for_each_inst(i, xcc_mask) { 287 /* Setup L2 cache */ 288 tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL); 289 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 290 ENABLE_L2_CACHE, 1); 291 /*TODO: set ENABLE_L2_FRAGMENT_PROCESSING to 1? */ 292 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 293 ENABLE_L2_FRAGMENT_PROCESSING, 0); 294 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 295 ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); 296 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 297 L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 298 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 299 PDE_FAULT_CLASSIFICATION, 0); 300 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 301 CONTEXT1_IDENTITY_ACCESS_MODE, 1); 302 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 303 IDENTITY_MODE_FRAGMENT_SIZE, 0); 304 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 305 regGCVM_L2_CNTL, tmp); 306 307 tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL2); 308 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, 309 INVALIDATE_ALL_L1_TLBS, 1); 310 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, 311 INVALIDATE_L2_CACHE, 1); 312 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 313 regGCVM_L2_CNTL2, tmp); 314 315 tmp = regGCVM_L2_CNTL3_DEFAULT; 316 if (adev->gmc.translate_further) { 317 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12); 318 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 319 L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 320 } else { 321 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9); 322 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 323 L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 324 } 325 WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL3, tmp); 326 327 tmp = regGCVM_L2_CNTL4_DEFAULT; 328 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, 329 VMC_TAP_PDE_REQUEST_PHYSICAL, 1); 330 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, 331 VMC_TAP_PTE_REQUEST_PHYSICAL, 1); 332 WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL4, tmp); 333 334 tmp = regGCVM_L2_CNTL5_DEFAULT; 335 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, 336 L2_CACHE_SMALLK_FRAGMENT_SIZE, 0); 337 WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL5, tmp); 338 } 339 } 340 341 static void gfxhub_v12_1_xcc_enable_system_domain(struct amdgpu_device *adev, 342 uint32_t xcc_mask) 343 { 344 uint32_t tmp; 345 int i; 346 347 for_each_inst(i, xcc_mask) { 348 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 349 regGCVM_CONTEXT0_CNTL); 350 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 351 ENABLE_CONTEXT, 1); 352 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 353 PAGE_TABLE_DEPTH, 354 adev->gmc.vmid0_page_table_depth); 355 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 356 PAGE_TABLE_BLOCK_SIZE, 357 adev->gmc.vmid0_page_table_block_size); 358 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 359 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 360 WREG32_SOC15(GC, GET_INST(GC, i), 361 regGCVM_CONTEXT0_CNTL, tmp); 362 } 363 } 364 365 static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *adev, 366 uint32_t xcc_mask) 367 { 368 int i; 369 370 for_each_inst(i, xcc_mask) { 371 WREG32_SOC15(GC, GET_INST(GC, i), 372 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 373 0XFFFFFFFF); 374 WREG32_SOC15(GC, GET_INST(GC, i), 375 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 376 0x00001FFF); 377 378 WREG32_SOC15(GC, GET_INST(GC, i), 379 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 380 0); 381 WREG32_SOC15(GC, GET_INST(GC, i), 382 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 383 0); 384 385 WREG32_SOC15(GC, GET_INST(GC, i), 386 regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 387 0); 388 WREG32_SOC15(GC, GET_INST(GC, i), 389 regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 390 0); 391 } 392 } 393 394 static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, 395 uint32_t xcc_mask) 396 { 397 struct amdgpu_vmhub *hub; 398 unsigned int num_level, block_size; 399 uint32_t tmp; 400 int i, j; 401 402 num_level = adev->vm_manager.num_level; 403 block_size = adev->vm_manager.block_size; 404 block_size -= 9; 405 406 for_each_inst(j, xcc_mask) { 407 hub = &adev->vmhub[AMDGPU_GFXHUB(j)]; 408 for (i = 0; i <= 14; i++) { 409 tmp = RREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 410 regGCVM_CONTEXT1_CNTL, 411 i * hub->ctx_distance); 412 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 413 ENABLE_CONTEXT, 1); 414 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 415 PAGE_TABLE_DEPTH, num_level); 416 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 417 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 418 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 419 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 420 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 421 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 422 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 423 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 424 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 425 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 426 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 427 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 428 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 429 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 430 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 431 PAGE_TABLE_BLOCK_SIZE, block_size); 432 /* Send no-retry XNACK on fault to suppress VM fault storm */ 433 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 434 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 435 1); 436 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regGCVM_CONTEXT1_CNTL, 437 i * hub->ctx_distance, tmp); 438 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 439 regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 440 i * hub->ctx_addr_distance, 0); 441 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 442 regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 443 i * hub->ctx_addr_distance, 0); 444 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 445 regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 446 i * hub->ctx_addr_distance, 447 lower_32_bits(adev->vm_manager.max_pfn - 1)); 448 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 449 regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 450 i * hub->ctx_addr_distance, 451 upper_32_bits(adev->vm_manager.max_pfn - 1)); 452 } 453 454 hub->vm_cntx_cntl = tmp; 455 } 456 } 457 458 static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, 459 uint32_t xcc_mask) 460 { 461 struct amdgpu_vmhub *hub; 462 unsigned int i, j; 463 464 for_each_inst(j, xcc_mask) { 465 hub = &adev->vmhub[AMDGPU_GFXHUB(j)]; 466 467 for (i = 0 ; i < 18; ++i) { 468 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 469 regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 470 i * hub->eng_addr_distance, 0xFFFFFFFF); 471 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 472 regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 473 i * hub->eng_addr_distance, 0x3FFF); 474 } 475 } 476 } 477 478 static int gfxhub_v12_1_xcc_gart_enable(struct amdgpu_device *adev, 479 uint32_t xcc_mask) 480 { 481 uint32_t i; 482 483 if (amdgpu_sriov_vf(adev)) { 484 /* GCMC_VM_FB_LOCATION_BASE/TOP are VF copy registers 485 * VBIO post does not program them at boot up phase 486 * Need driver to program them from guest side */ 487 for_each_inst(i, xcc_mask) { 488 WREG32_SOC15(GC, GET_INST(GC, i), 489 regGCMC_VM_FB_LOCATION_BASE_LO32, 490 lower_32_bits(adev->gmc.vram_start >> 24)); 491 WREG32_SOC15(GC, GET_INST(GC, i), 492 regGCMC_VM_FB_LOCATION_BASE_HI32, 493 upper_32_bits(adev->gmc.vram_start >> 24)); 494 WREG32_SOC15(GC, GET_INST(GC, i), 495 regGCMC_VM_FB_LOCATION_TOP_LO32, 496 lower_32_bits(adev->gmc.vram_end >> 24)); 497 WREG32_SOC15(GC, GET_INST(GC, i), 498 regGCMC_VM_FB_LOCATION_TOP_HI32, 499 upper_32_bits(adev->gmc.vram_end >> 24)); 500 } 501 } 502 /* GART Enable. */ 503 gfxhub_v12_1_xcc_init_gart_aperture_regs(adev, xcc_mask); 504 gfxhub_v12_1_xcc_init_system_aperture_regs(adev, xcc_mask); 505 gfxhub_v12_1_xcc_init_tlb_regs(adev, xcc_mask); 506 if (!amdgpu_sriov_vf(adev)) 507 gfxhub_v12_1_xcc_init_cache_regs(adev, xcc_mask); 508 509 gfxhub_v12_1_xcc_enable_system_domain(adev, xcc_mask); 510 if (!amdgpu_sriov_vf(adev)) 511 gfxhub_v12_1_xcc_disable_identity_aperture(adev, xcc_mask); 512 gfxhub_v12_1_xcc_setup_vmid_config(adev, xcc_mask); 513 gfxhub_v12_1_xcc_program_invalidation(adev, xcc_mask); 514 515 return 0; 516 } 517 518 static int gfxhub_v12_1_gart_enable(struct amdgpu_device *adev) 519 { 520 uint32_t xcc_mask; 521 522 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 523 return gfxhub_v12_1_xcc_gart_enable(adev, xcc_mask); 524 } 525 526 static void gfxhub_v12_1_xcc_gart_disable(struct amdgpu_device *adev, 527 uint32_t xcc_mask) 528 { 529 struct amdgpu_vmhub *hub; 530 u32 tmp; 531 u32 i, j; 532 533 for_each_inst(j, xcc_mask) { 534 hub = &adev->vmhub[AMDGPU_GFXHUB(j)]; 535 /* Disable all tables */ 536 for (i = 0; i < 16; i++) 537 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 538 regGCVM_CONTEXT0_CNTL, 539 i * hub->ctx_distance, 0); 540 541 /* Setup TLB control */ 542 tmp = RREG32_SOC15(GC, GET_INST(GC, j), 543 regGCMC_VM_MX_L1_TLB_CNTL); 544 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 545 ENABLE_L1_TLB, 0); 546 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 547 ENABLE_ADVANCED_DRIVER_MODEL, 0); 548 WREG32_SOC15_RLC(GC, GET_INST(GC, j), 549 regGCMC_VM_MX_L1_TLB_CNTL, tmp); 550 551 /* Setup L2 cache */ 552 if (!amdgpu_sriov_vf(adev)) { 553 tmp = RREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL); 554 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0); 555 WREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL, tmp); 556 WREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL3, 0); 557 } 558 } 559 } 560 561 static void gfxhub_v12_1_gart_disable(struct amdgpu_device *adev) 562 { 563 uint32_t xcc_mask; 564 565 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 566 gfxhub_v12_1_xcc_gart_disable(adev, xcc_mask); 567 } 568 569 static void gfxhub_v12_1_xcc_set_fault_enable_default(struct amdgpu_device *adev, 570 bool value, uint32_t xcc_mask) 571 { 572 u32 tmp; 573 int i; 574 575 for_each_inst(i, xcc_mask) { 576 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 577 regGCVM_L2_PROTECTION_FAULT_CNTL_LO32); 578 tmp = REG_SET_FIELD(tmp, 579 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 580 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 581 tmp = REG_SET_FIELD(tmp, 582 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 583 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 584 tmp = REG_SET_FIELD(tmp, 585 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 586 PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 587 tmp = REG_SET_FIELD(tmp, 588 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 589 PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 590 tmp = REG_SET_FIELD(tmp, 591 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 592 PDE3_PROTECTION_FAULT_ENABLE_DEFAULT, value); 593 tmp = REG_SET_FIELD(tmp, 594 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 595 TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 596 value); 597 tmp = REG_SET_FIELD(tmp, 598 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 599 NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 600 tmp = REG_SET_FIELD(tmp, 601 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 602 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 603 tmp = REG_SET_FIELD(tmp, 604 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 605 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 606 tmp = REG_SET_FIELD(tmp, 607 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 608 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 609 tmp = REG_SET_FIELD(tmp, 610 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 611 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 612 tmp = REG_SET_FIELD(tmp, 613 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 614 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 615 tmp = REG_SET_FIELD(tmp, 616 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 617 CLIENT_ID_NO_RETRY_FAULT_INTERRUPT, value ? 0xFFFF:0); 618 tmp = REG_SET_FIELD(tmp, 619 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 620 OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT, value); 621 if (!value) 622 tmp = REG_SET_FIELD(tmp, 623 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 624 CRASH_ON_NO_RETRY_FAULT, 1); 625 WREG32_SOC15(GC, GET_INST(GC, i), 626 regGCVM_L2_PROTECTION_FAULT_CNTL_LO32, tmp); 627 628 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 629 regGCVM_L2_PROTECTION_FAULT_CNTL_HI32); 630 if (!value) 631 tmp = REG_SET_FIELD(tmp, 632 GCVM_L2_PROTECTION_FAULT_CNTL_HI32, 633 CRASH_ON_RETRY_FAULT, 1); 634 WREG32_SOC15(GC, GET_INST(GC, i), 635 regGCVM_L2_PROTECTION_FAULT_CNTL_HI32, tmp); 636 } 637 } 638 639 /** 640 * gfxhub_v12_1_set_fault_enable_default - update GART/VM fault handling 641 * 642 * @adev: amdgpu_device pointer 643 * @value: true redirects VM faults to the default page 644 */ 645 static void gfxhub_v12_1_set_fault_enable_default(struct amdgpu_device *adev, 646 bool value) 647 { 648 uint32_t xcc_mask; 649 650 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 651 gfxhub_v12_1_xcc_set_fault_enable_default(adev, value, xcc_mask); 652 } 653 654 static uint32_t gfxhub_v12_1_get_invalidate_req(unsigned int vmid, 655 uint32_t flush_type) 656 { 657 u32 req = 0; 658 659 /* invalidate using legacy mode on vmid*/ 660 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 661 PER_VMID_INVALIDATE_REQ, 1 << vmid); 662 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 663 FLUSH_TYPE, flush_type); 664 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 665 INVALIDATE_L2_PTES, 1); 666 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 667 INVALIDATE_L2_PDE0, 1); 668 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 669 INVALIDATE_L2_PDE1, 1); 670 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 671 INVALIDATE_L2_PDE2, 1); 672 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 673 INVALIDATE_L2_PDE3, 1); 674 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 675 INVALIDATE_L1_PTES, 1); 676 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 677 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 678 679 return req; 680 } 681 682 static const char *gfxhub_v12_1_client_ids[] = { 683 "CB", 684 "DB", 685 "GE1", 686 "GE2", 687 "CPF", 688 "CPC", 689 "CPG", 690 "RLC", 691 "TCP", 692 "SQC (inst)", 693 "SQC (data)", 694 "SQG/PC/SC", 695 "Reserved", 696 "SDMA0", 697 "SDMA1", 698 "GCR", 699 "Reserved", 700 "Reserved", 701 "WGS", 702 "DSM", 703 "PA" 704 }; 705 706 /*TODO: l2 protection fault status is increased to 64bits. 707 * some critical fields like FED are moved to STATUS_HI32 */ 708 static void gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device *adev, 709 uint32_t status) 710 { 711 u32 cid = REG_GET_FIELD(status, 712 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, 713 CID); 714 715 dev_err(adev->dev, 716 "GCVM_L2_PROTECTION_FAULT_STATUS_LO32:0x%08X\n", 717 status); 718 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 719 cid >= ARRAY_SIZE(gfxhub_v12_1_client_ids) ? 720 "unknown" : gfxhub_v12_1_client_ids[cid], cid); 721 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 722 REG_GET_FIELD(status, 723 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MORE_FAULTS)); 724 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 725 REG_GET_FIELD(status, 726 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, WALKER_ERROR)); 727 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 728 REG_GET_FIELD(status, 729 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, PERMISSION_FAULTS)); 730 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 731 REG_GET_FIELD(status, 732 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MAPPING_ERROR)); 733 dev_err(adev->dev, "\t RW: 0x%lx\n", 734 REG_GET_FIELD(status, 735 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, RW)); 736 } 737 738 static const struct amdgpu_vmhub_funcs gfxhub_v12_1_vmhub_funcs = { 739 .print_l2_protection_fault_status = gfxhub_v12_1_print_l2_protection_fault_status, 740 .get_invalidate_req = gfxhub_v12_1_get_invalidate_req, 741 }; 742 743 static void gfxhub_v12_1_xcc_init(struct amdgpu_device *adev, uint32_t xcc_mask) 744 { 745 struct amdgpu_vmhub *hub; 746 int i; 747 748 for_each_inst(i, xcc_mask) { 749 hub = &adev->vmhub[AMDGPU_GFXHUB(i)]; 750 751 hub->ctx0_ptb_addr_lo32 = 752 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 753 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 754 hub->ctx0_ptb_addr_hi32 = 755 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 756 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 757 hub->vm_inv_eng0_sem = 758 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 759 regGCVM_INVALIDATE_ENG0_SEM); 760 hub->vm_inv_eng0_req = 761 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 762 regGCVM_INVALIDATE_ENG0_REQ); 763 hub->vm_inv_eng0_ack = 764 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 765 regGCVM_INVALIDATE_ENG0_ACK); 766 hub->vm_context0_cntl = 767 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 768 regGCVM_CONTEXT0_CNTL); 769 /* TODO: add a new member to accomandate additional fault status/cntl reg */ 770 hub->vm_l2_pro_fault_status = 771 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 772 regGCVM_L2_PROTECTION_FAULT_STATUS_LO32); 773 hub->vm_l2_pro_fault_cntl = 774 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 775 regGCVM_L2_PROTECTION_FAULT_CNTL_LO32); 776 hub->ctx_distance = 777 regGCVM_CONTEXT1_CNTL - 778 regGCVM_CONTEXT0_CNTL; 779 hub->ctx_addr_distance = 780 regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 781 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 782 hub->eng_distance = 783 regGCVM_INVALIDATE_ENG1_REQ - 784 regGCVM_INVALIDATE_ENG0_REQ; 785 hub->eng_addr_distance = 786 regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 787 regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 788 789 hub->vm_cntx_cntl_vm_fault = 790 GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 791 GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 792 GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 793 GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 794 GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 795 GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 796 GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 797 798 hub->vmhub_funcs = &gfxhub_v12_1_vmhub_funcs; 799 } 800 } 801 802 static void gfxhub_v12_1_init(struct amdgpu_device *adev) 803 { 804 uint32_t xcc_mask; 805 806 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 807 gfxhub_v12_1_xcc_init(adev, xcc_mask); 808 } 809 810 static int gfxhub_v12_1_get_xgmi_info(struct amdgpu_device *adev) 811 { 812 u32 max_num_physical_nodes; 813 u32 max_physical_node_id; 814 u32 xgmi_lfb_cntl; 815 u32 max_region; 816 u64 seg_size; 817 818 xgmi_lfb_cntl = RREG32_SOC15(GC, GET_INST(GC, 0), 819 regGCMC_VM_XGMI_LFB_CNTL); 820 seg_size = REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, 0), 821 regGCMC_VM_XGMI_LFB_SIZE), 822 GCMC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24; 823 max_region = REG_GET_FIELD(xgmi_lfb_cntl, 824 GCMC_VM_XGMI_LFB_CNTL, 825 PF_MAX_REGION); 826 827 max_num_physical_nodes = 8; 828 max_physical_node_id = 7; 829 830 /* PF_MAX_REGION=0 means xgmi is disabled */ 831 if (max_region || adev->gmc.xgmi.connected_to_cpu) { 832 adev->gmc.xgmi.num_physical_nodes = max_region + 1; 833 834 if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes) 835 return -EINVAL; 836 837 adev->gmc.xgmi.physical_node_id = 838 REG_GET_FIELD(xgmi_lfb_cntl, 839 GCMC_VM_XGMI_LFB_CNTL, 840 PF_LFB_REGION); 841 842 if (adev->gmc.xgmi.physical_node_id > max_physical_node_id) 843 return -EINVAL; 844 845 adev->gmc.xgmi.node_segment_size = seg_size; 846 } 847 848 return 0; 849 } 850 851 const struct amdgpu_gfxhub_funcs gfxhub_v12_1_funcs = { 852 .get_fb_location = gfxhub_v12_1_get_fb_location, 853 .get_mc_fb_offset = gfxhub_v12_1_get_mc_fb_offset, 854 .setup_vm_pt_regs = gfxhub_v12_1_setup_vm_pt_regs, 855 .gart_enable = gfxhub_v12_1_gart_enable, 856 .gart_disable = gfxhub_v12_1_gart_disable, 857 .set_fault_enable_default = gfxhub_v12_1_set_fault_enable_default, 858 .init = gfxhub_v12_1_init, 859 .get_xgmi_info = gfxhub_v12_1_get_xgmi_info, 860 }; 861 862 static int gfxhub_v12_1_xcp_resume(void *handle, uint32_t inst_mask) 863 { 864 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 865 bool value; 866 867 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 868 value = false; 869 else 870 value = true; 871 872 gfxhub_v12_1_xcc_set_fault_enable_default(adev, value, inst_mask); 873 874 if (!amdgpu_sriov_vf(adev)) 875 return gfxhub_v12_1_xcc_gart_enable(adev, inst_mask); 876 877 return 0; 878 } 879 880 static int gfxhub_v12_1_xcp_suspend(void *handle, uint32_t inst_mask) 881 { 882 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 883 884 if (!amdgpu_sriov_vf(adev)) 885 gfxhub_v12_1_xcc_gart_disable(adev, inst_mask); 886 887 return 0; 888 } 889 890 struct amdgpu_xcp_ip_funcs gfxhub_v12_1_xcp_funcs = { 891 .suspend = &gfxhub_v12_1_xcp_suspend, 892 .resume = &gfxhub_v12_1_xcp_resume 893 }; 894