1 /* 2 * Copyright 2025 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "amdgpu_xcp.h" 25 #include "gfxhub_v12_1.h" 26 27 #include "gc/gc_12_1_0_offset.h" 28 #include "gc/gc_12_1_0_sh_mask.h" 29 #include "soc_v1_0_enum.h" 30 31 #include "soc15_common.h" 32 33 #define regGCVM_L2_CNTL3_DEFAULT 0x80120007 34 #define regGCVM_L2_CNTL4_DEFAULT 0x000000c1 35 #define regGCVM_L2_CNTL5_DEFAULT 0x00003fe0 36 #define regGRBM_GFX_INDEX_DEFAULT 0xe0000000 37 38 39 static u64 gfxhub_v12_1_get_fb_location(struct amdgpu_device *adev) 40 { 41 u64 base; 42 43 base = RREG32_SOC15(GC, GET_INST(GC, 0), 44 regGCMC_VM_FB_LOCATION_BASE_LO32); 45 base &= GCMC_VM_FB_LOCATION_BASE_LO32__FB_BASE_LO32_MASK; 46 base <<= 24; 47 48 base |= ((u64)(GCMC_VM_FB_LOCATION_BASE_HI32__FB_BASE_HI1_MASK & 49 RREG32_SOC15(GC, GET_INST(GC, 0), 50 regGCMC_VM_FB_LOCATION_BASE_HI32)) << 56); 51 return base; 52 } 53 54 static u64 gfxhub_v12_1_get_mc_fb_offset(struct amdgpu_device *adev) 55 { 56 return (u64)(RREG32_SOC15(GC, GET_INST(GC, 0), 57 regGCMC_VM_FB_OFFSET) << 24); 58 } 59 60 static void gfxhub_v12_1_xcc_setup_vm_pt_regs(struct amdgpu_device *adev, 61 uint32_t vmid, 62 uint64_t page_table_base, 63 uint32_t xcc_mask) 64 { 65 struct amdgpu_vmhub *hub; 66 int i; 67 68 for_each_inst(i, xcc_mask) { 69 hub = &adev->vmhub[AMDGPU_GFXHUB(i)]; 70 WREG32_SOC15_OFFSET(GC, GET_INST(GC, i), 71 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 72 hub->ctx_addr_distance * vmid, 73 lower_32_bits(page_table_base)); 74 75 WREG32_SOC15_OFFSET(GC, GET_INST(GC, i), 76 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 77 hub->ctx_addr_distance * vmid, 78 upper_32_bits(page_table_base)); 79 } 80 } 81 82 static void gfxhub_v12_1_setup_vm_pt_regs(struct amdgpu_device *adev, 83 uint32_t vmid, 84 uint64_t page_table_base) 85 { 86 uint32_t xcc_mask; 87 88 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 89 gfxhub_v12_1_xcc_setup_vm_pt_regs(adev, vmid, page_table_base, 90 xcc_mask); 91 } 92 93 static void gfxhub_v12_1_xcc_init_gart_aperture_regs(struct amdgpu_device *adev, 94 uint32_t xcc_mask) 95 { 96 uint64_t pt_base; 97 int i; 98 99 if (adev->gmc.pdb0_bo) 100 pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo); 101 else 102 pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 103 104 gfxhub_v12_1_xcc_setup_vm_pt_regs(adev, 0, pt_base, xcc_mask); 105 106 /* If use GART for FB translation, vmid0 page table covers both 107 * vram and system memory (gart) 108 */ 109 for_each_inst(i, xcc_mask) { 110 if (adev->gmc.pdb0_bo) { 111 WREG32_SOC15(GC, GET_INST(GC, i), 112 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 113 (u32)(adev->gmc.fb_start >> 12)); 114 WREG32_SOC15(GC, GET_INST(GC, i), 115 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 116 (u32)(adev->gmc.fb_start >> 44)); 117 118 WREG32_SOC15(GC, GET_INST(GC, i), 119 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 120 (u32)(adev->gmc.gart_end >> 12)); 121 WREG32_SOC15(GC, GET_INST(GC, i), 122 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 123 (u32)(adev->gmc.gart_end >> 44)); 124 } else { 125 WREG32_SOC15(GC, GET_INST(GC, i), 126 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 127 (u32)(adev->gmc.gart_start >> 12)); 128 WREG32_SOC15(GC, GET_INST(GC, i), 129 regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 130 (u32)(adev->gmc.gart_start >> 44)); 131 132 WREG32_SOC15(GC, GET_INST(GC, i), 133 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 134 (u32)(adev->gmc.gart_end >> 12)); 135 WREG32_SOC15(GC, GET_INST(GC, i), 136 regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 137 (u32)(adev->gmc.gart_end >> 44)); 138 } 139 } 140 } 141 142 static void gfxhub_v12_1_xcc_init_system_aperture_regs(struct amdgpu_device *adev, 143 uint32_t xcc_mask) 144 { 145 uint64_t value; 146 uint32_t tmp; 147 int i; 148 149 /*TODO: revisit whether the SRIOV guest access to theseregisters 150 * is blocked by security policy or not */ 151 if (amdgpu_sriov_vf(adev)) 152 return; 153 154 for_each_inst(i, xcc_mask) { 155 if (adev->gmc.pdb0_bo) { 156 /* Disable agp and system aperture 157 * when vmid0 page table is enabled */ 158 WREG32_SOC15(GC, GET_INST(GC, i), 159 regGCMC_VM_FB_LOCATION_TOP_LO32, 0); 160 WREG32_SOC15(GC, GET_INST(GC, i), 161 regGCMC_VM_FB_LOCATION_TOP_HI32, 0); 162 WREG32_SOC15(GC, GET_INST(GC, i), 163 regGCMC_VM_FB_LOCATION_BASE_LO32, 164 0xFFFFFFFF); 165 WREG32_SOC15(GC, GET_INST(GC, i), 166 regGCMC_VM_FB_LOCATION_BASE_HI32, 1); 167 WREG32_SOC15(GC, GET_INST(GC, i), 168 regGCMC_VM_AGP_TOP_LO32, 0); 169 WREG32_SOC15(GC, GET_INST(GC, i), 170 regGCMC_VM_AGP_TOP_HI32, 0); 171 WREG32_SOC15(GC, GET_INST(GC, i), 172 regGCMC_VM_AGP_BOT_LO32, 173 0xFFFFFFFF); 174 WREG32_SOC15(GC, GET_INST(GC, i), 175 regGCMC_VM_AGP_BOT_HI32, 1); 176 WREG32_SOC15(GC, GET_INST(GC, i), 177 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32, 178 0xFFFFFFFF); 179 WREG32_SOC15(GC, GET_INST(GC, i), 180 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32, 181 0x7F); 182 WREG32_SOC15(GC, GET_INST(GC, i), 183 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 0); 184 WREG32_SOC15(GC, GET_INST(GC, i), 185 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 0); 186 } else { 187 /* Program the AGP BAR */ 188 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 189 regGCMC_VM_AGP_BASE_LO32, 0); 190 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 191 regGCMC_VM_AGP_BASE_HI32, 0); 192 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 193 regGCMC_VM_AGP_BOT_LO32, 194 lower_32_bits(adev->gmc.agp_start >> 24)); 195 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 196 regGCMC_VM_AGP_BOT_HI32, 197 upper_32_bits(adev->gmc.agp_start >> 24)); 198 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 199 regGCMC_VM_AGP_TOP_LO32, 200 lower_32_bits(adev->gmc.agp_end >> 24)); 201 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 202 regGCMC_VM_AGP_TOP_HI32, 203 upper_32_bits(adev->gmc.agp_end >> 24)); 204 205 /* Program the system aperture low logical page number. */ 206 WREG32_SOC15(GC, GET_INST(GC, i), 207 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_LO32, 208 lower_32_bits(min(adev->gmc.fb_start, 209 adev->gmc.agp_start) >> 18)); 210 WREG32_SOC15(GC, GET_INST(GC, i), 211 regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_HI32, 212 upper_32_bits(min(adev->gmc.fb_start, 213 adev->gmc.agp_start) >> 18)); 214 WREG32_SOC15(GC, GET_INST(GC, i), 215 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_LO32, 216 lower_32_bits(max(adev->gmc.fb_end, 217 adev->gmc.agp_end) >> 18)); 218 WREG32_SOC15(GC, GET_INST(GC, i), 219 regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_HI32, 220 upper_32_bits(max(adev->gmc.fb_end, 221 adev->gmc.agp_end) >> 18)); 222 } 223 224 /* Set default page address. */ 225 value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr); 226 WREG32_SOC15(GC, GET_INST(GC, i), 227 regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 228 (u32)(value >> 12)); 229 WREG32_SOC15(GC, GET_INST(GC, i), 230 regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 231 (u32)(value >> 44)); 232 233 /* Program "protection fault". */ 234 WREG32_SOC15(GC, GET_INST(GC, i), 235 regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 236 (u32)(adev->dummy_page_addr >> 12)); 237 WREG32_SOC15(GC, GET_INST(GC, i), 238 regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 239 (u32)((u64)adev->dummy_page_addr >> 44)); 240 241 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 242 regGCVM_L2_PROTECTION_FAULT_CNTL2); 243 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2, 244 ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 245 tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL2, 246 ENABLE_RETRY_FAULT_INTERRUPT, 0x1); 247 WREG32_SOC15(GC, GET_INST(GC, i), 248 regGCVM_L2_PROTECTION_FAULT_CNTL2, tmp); 249 } 250 } 251 252 static void gfxhub_v12_1_xcc_init_tlb_regs(struct amdgpu_device *adev, 253 uint32_t xcc_mask) 254 { 255 uint32_t tmp; 256 int i; 257 258 for_each_inst(i, xcc_mask) { 259 /* Setup TLB control */ 260 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 261 regGCMC_VM_MX_L1_TLB_CNTL); 262 263 tmp = REG_SET_FIELD(tmp, 264 GCMC_VM_MX_L1_TLB_CNTL, 265 ENABLE_L1_TLB, 1); 266 tmp = REG_SET_FIELD(tmp, 267 GCMC_VM_MX_L1_TLB_CNTL, 268 SYSTEM_ACCESS_MODE, 3); 269 tmp = REG_SET_FIELD(tmp, 270 GCMC_VM_MX_L1_TLB_CNTL, 271 ENABLE_ADVANCED_DRIVER_MODEL, 1); 272 tmp = REG_SET_FIELD(tmp, 273 GCMC_VM_MX_L1_TLB_CNTL, 274 SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 275 tmp = REG_SET_FIELD(tmp, 276 GCMC_VM_MX_L1_TLB_CNTL, 277 ECO_BITS, 0); 278 tmp = REG_SET_FIELD(tmp, 279 GCMC_VM_MX_L1_TLB_CNTL, 280 MTYPE, MTYPE_UC); 281 282 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 283 regGCMC_VM_MX_L1_TLB_CNTL, tmp); 284 } 285 } 286 287 static void gfxhub_v12_1_xcc_init_cache_regs(struct amdgpu_device *adev, 288 uint32_t xcc_mask) 289 { 290 uint32_t tmp; 291 int i; 292 293 for_each_inst(i, xcc_mask) { 294 /* Setup L2 cache */ 295 tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL); 296 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 297 ENABLE_L2_CACHE, 1); 298 /*TODO: set ENABLE_L2_FRAGMENT_PROCESSING to 1? */ 299 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 300 ENABLE_L2_FRAGMENT_PROCESSING, 0); 301 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 302 ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); 303 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 304 L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 305 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 306 PDE_FAULT_CLASSIFICATION, 0); 307 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 308 CONTEXT1_IDENTITY_ACCESS_MODE, 1); 309 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 310 IDENTITY_MODE_FRAGMENT_SIZE, 0); 311 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 312 regGCVM_L2_CNTL, tmp); 313 314 tmp = RREG32_SOC15(GC, GET_INST(GC, i), regGCVM_L2_CNTL2); 315 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, 316 INVALIDATE_ALL_L1_TLBS, 1); 317 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, 318 INVALIDATE_L2_CACHE, 1); 319 WREG32_SOC15_RLC(GC, GET_INST(GC, i), 320 regGCVM_L2_CNTL2, tmp); 321 322 tmp = regGCVM_L2_CNTL3_DEFAULT; 323 if (adev->gmc.translate_further) { 324 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12); 325 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 326 L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 327 } else { 328 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9); 329 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 330 L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 331 } 332 WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL3, tmp); 333 334 tmp = regGCVM_L2_CNTL4_DEFAULT; 335 if (adev->gmc.xgmi.connected_to_cpu) { 336 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, 337 VMC_TAP_PDE_REQUEST_PHYSICAL, 1); 338 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, 339 VMC_TAP_PTE_REQUEST_PHYSICAL, 1); 340 } else { 341 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, 342 VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 343 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, 344 VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 345 } 346 347 WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL4, tmp); 348 349 tmp = regGCVM_L2_CNTL5_DEFAULT; 350 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, 351 L2_CACHE_SMALLK_FRAGMENT_SIZE, 0); 352 WREG32_SOC15_RLC(GC, GET_INST(GC, i), regGCVM_L2_CNTL5, tmp); 353 } 354 } 355 356 static void gfxhub_v12_1_xcc_enable_system_domain(struct amdgpu_device *adev, 357 uint32_t xcc_mask) 358 { 359 uint32_t tmp; 360 int i; 361 362 for_each_inst(i, xcc_mask) { 363 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 364 regGCVM_CONTEXT0_CNTL); 365 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 366 ENABLE_CONTEXT, 1); 367 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 368 PAGE_TABLE_DEPTH, 369 adev->gmc.vmid0_page_table_depth); 370 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 371 PAGE_TABLE_BLOCK_SIZE, 372 adev->gmc.vmid0_page_table_block_size); 373 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 374 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 375 WREG32_SOC15(GC, GET_INST(GC, i), 376 regGCVM_CONTEXT0_CNTL, tmp); 377 } 378 } 379 380 static void gfxhub_v12_1_xcc_disable_identity_aperture(struct amdgpu_device *adev, 381 uint32_t xcc_mask) 382 { 383 int i; 384 385 for_each_inst(i, xcc_mask) { 386 WREG32_SOC15(GC, GET_INST(GC, i), 387 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 388 0XFFFFFFFF); 389 WREG32_SOC15(GC, GET_INST(GC, i), 390 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 391 0x00001FFF); 392 393 WREG32_SOC15(GC, GET_INST(GC, i), 394 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 395 0); 396 WREG32_SOC15(GC, GET_INST(GC, i), 397 regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 398 0); 399 400 WREG32_SOC15(GC, GET_INST(GC, i), 401 regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 402 0); 403 WREG32_SOC15(GC, GET_INST(GC, i), 404 regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 405 0); 406 } 407 } 408 409 static void gfxhub_v12_1_xcc_setup_vmid_config(struct amdgpu_device *adev, 410 uint32_t xcc_mask) 411 { 412 struct amdgpu_vmhub *hub; 413 unsigned int num_level, block_size; 414 uint32_t tmp; 415 int i, j; 416 417 num_level = adev->vm_manager.num_level; 418 block_size = adev->vm_manager.block_size; 419 block_size -= 9; 420 421 for_each_inst(j, xcc_mask) { 422 hub = &adev->vmhub[AMDGPU_GFXHUB(j)]; 423 for (i = 0; i <= 14; i++) { 424 tmp = RREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 425 regGCVM_CONTEXT1_CNTL, 426 i * hub->ctx_distance); 427 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 428 ENABLE_CONTEXT, 1); 429 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 430 PAGE_TABLE_DEPTH, num_level); 431 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 432 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 433 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 434 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 435 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 436 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 437 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 438 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 439 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 440 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 441 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 442 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 443 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 444 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 445 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 446 PAGE_TABLE_BLOCK_SIZE, block_size); 447 /* Send no-retry XNACK on fault to suppress VM fault storm */ 448 tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 449 RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 450 1); 451 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regGCVM_CONTEXT1_CNTL, 452 i * hub->ctx_distance, tmp); 453 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 454 regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 455 i * hub->ctx_addr_distance, 0); 456 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 457 regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 458 i * hub->ctx_addr_distance, 0); 459 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 460 regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 461 i * hub->ctx_addr_distance, 462 lower_32_bits(adev->vm_manager.max_pfn - 1)); 463 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 464 regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 465 i * hub->ctx_addr_distance, 466 upper_32_bits(adev->vm_manager.max_pfn - 1)); 467 } 468 469 hub->vm_cntx_cntl = tmp; 470 } 471 } 472 473 static void gfxhub_v12_1_xcc_program_invalidation(struct amdgpu_device *adev, 474 uint32_t xcc_mask) 475 { 476 struct amdgpu_vmhub *hub; 477 unsigned int i, j; 478 479 for_each_inst(j, xcc_mask) { 480 hub = &adev->vmhub[AMDGPU_GFXHUB(j)]; 481 482 for (i = 0 ; i < 18; ++i) { 483 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 484 regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 485 i * hub->eng_addr_distance, 0xFFFFFFFF); 486 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 487 regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 488 i * hub->eng_addr_distance, 0x3FFF); 489 } 490 } 491 } 492 493 static int gfxhub_v12_1_xcc_gart_enable(struct amdgpu_device *adev, 494 uint32_t xcc_mask) 495 { 496 uint32_t i; 497 498 if (amdgpu_sriov_vf(adev)) { 499 /* GCMC_VM_FB_LOCATION_BASE/TOP are VF copy registers 500 * VBIO post does not program them at boot up phase 501 * Need driver to program them from guest side */ 502 for_each_inst(i, xcc_mask) { 503 WREG32_SOC15(GC, GET_INST(GC, i), 504 regGCMC_VM_FB_LOCATION_BASE_LO32, 505 lower_32_bits(adev->gmc.vram_start >> 24)); 506 WREG32_SOC15(GC, GET_INST(GC, i), 507 regGCMC_VM_FB_LOCATION_BASE_HI32, 508 upper_32_bits(adev->gmc.vram_start >> 24)); 509 WREG32_SOC15(GC, GET_INST(GC, i), 510 regGCMC_VM_FB_LOCATION_TOP_LO32, 511 lower_32_bits(adev->gmc.vram_end >> 24)); 512 WREG32_SOC15(GC, GET_INST(GC, i), 513 regGCMC_VM_FB_LOCATION_TOP_HI32, 514 upper_32_bits(adev->gmc.vram_end >> 24)); 515 } 516 } 517 /* GART Enable. */ 518 gfxhub_v12_1_xcc_init_gart_aperture_regs(adev, xcc_mask); 519 gfxhub_v12_1_xcc_init_system_aperture_regs(adev, xcc_mask); 520 gfxhub_v12_1_xcc_init_tlb_regs(adev, xcc_mask); 521 if (!amdgpu_sriov_vf(adev)) 522 gfxhub_v12_1_xcc_init_cache_regs(adev, xcc_mask); 523 524 gfxhub_v12_1_xcc_enable_system_domain(adev, xcc_mask); 525 if (!amdgpu_sriov_vf(adev)) 526 gfxhub_v12_1_xcc_disable_identity_aperture(adev, xcc_mask); 527 gfxhub_v12_1_xcc_setup_vmid_config(adev, xcc_mask); 528 gfxhub_v12_1_xcc_program_invalidation(adev, xcc_mask); 529 530 return 0; 531 } 532 533 static int gfxhub_v12_1_gart_enable(struct amdgpu_device *adev) 534 { 535 uint32_t xcc_mask; 536 537 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 538 return gfxhub_v12_1_xcc_gart_enable(adev, xcc_mask); 539 } 540 541 static void gfxhub_v12_1_xcc_gart_disable(struct amdgpu_device *adev, 542 uint32_t xcc_mask) 543 { 544 struct amdgpu_vmhub *hub; 545 u32 tmp; 546 u32 i, j; 547 548 for_each_inst(j, xcc_mask) { 549 hub = &adev->vmhub[AMDGPU_GFXHUB(j)]; 550 /* Disable all tables */ 551 for (i = 0; i < 16; i++) 552 WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), 553 regGCVM_CONTEXT0_CNTL, 554 i * hub->ctx_distance, 0); 555 556 /* Setup TLB control */ 557 tmp = RREG32_SOC15(GC, GET_INST(GC, j), 558 regGCMC_VM_MX_L1_TLB_CNTL); 559 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 560 ENABLE_L1_TLB, 0); 561 tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 562 ENABLE_ADVANCED_DRIVER_MODEL, 0); 563 WREG32_SOC15_RLC(GC, GET_INST(GC, j), 564 regGCMC_VM_MX_L1_TLB_CNTL, tmp); 565 566 /* Setup L2 cache */ 567 if (!amdgpu_sriov_vf(adev)) { 568 tmp = RREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL); 569 tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0); 570 WREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL, tmp); 571 WREG32_SOC15(GC, GET_INST(GC, j), regGCVM_L2_CNTL3, 0); 572 } 573 } 574 } 575 576 static void gfxhub_v12_1_gart_disable(struct amdgpu_device *adev) 577 { 578 uint32_t xcc_mask; 579 580 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 581 gfxhub_v12_1_xcc_gart_disable(adev, xcc_mask); 582 } 583 584 static void gfxhub_v12_1_xcc_set_fault_enable_default(struct amdgpu_device *adev, 585 bool value, uint32_t xcc_mask) 586 { 587 u32 tmp; 588 int i; 589 590 for_each_inst(i, xcc_mask) { 591 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 592 regGCVM_L2_PROTECTION_FAULT_CNTL_LO32); 593 tmp = REG_SET_FIELD(tmp, 594 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 595 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 596 tmp = REG_SET_FIELD(tmp, 597 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 598 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 599 tmp = REG_SET_FIELD(tmp, 600 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 601 PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 602 tmp = REG_SET_FIELD(tmp, 603 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 604 PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 605 tmp = REG_SET_FIELD(tmp, 606 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 607 PDE3_PROTECTION_FAULT_ENABLE_DEFAULT, value); 608 tmp = REG_SET_FIELD(tmp, 609 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 610 TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 611 value); 612 tmp = REG_SET_FIELD(tmp, 613 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 614 NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 615 tmp = REG_SET_FIELD(tmp, 616 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 617 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 618 tmp = REG_SET_FIELD(tmp, 619 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 620 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 621 tmp = REG_SET_FIELD(tmp, 622 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 623 READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 624 tmp = REG_SET_FIELD(tmp, 625 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 626 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 627 tmp = REG_SET_FIELD(tmp, 628 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 629 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 630 tmp = REG_SET_FIELD(tmp, 631 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 632 CLIENT_ID_NO_RETRY_FAULT_INTERRUPT, value ? 0xFFFF:0); 633 tmp = REG_SET_FIELD(tmp, 634 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 635 OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT, value); 636 if (!value) 637 tmp = REG_SET_FIELD(tmp, 638 GCVM_L2_PROTECTION_FAULT_CNTL_LO32, 639 CRASH_ON_NO_RETRY_FAULT, 1); 640 WREG32_SOC15(GC, GET_INST(GC, i), 641 regGCVM_L2_PROTECTION_FAULT_CNTL_LO32, tmp); 642 643 tmp = RREG32_SOC15(GC, GET_INST(GC, i), 644 regGCVM_L2_PROTECTION_FAULT_CNTL_HI32); 645 if (!value) 646 tmp = REG_SET_FIELD(tmp, 647 GCVM_L2_PROTECTION_FAULT_CNTL_HI32, 648 CRASH_ON_RETRY_FAULT, 1); 649 WREG32_SOC15(GC, GET_INST(GC, i), 650 regGCVM_L2_PROTECTION_FAULT_CNTL_HI32, tmp); 651 } 652 } 653 654 /** 655 * gfxhub_v12_1_set_fault_enable_default - update GART/VM fault handling 656 * 657 * @adev: amdgpu_device pointer 658 * @value: true redirects VM faults to the default page 659 */ 660 static void gfxhub_v12_1_set_fault_enable_default(struct amdgpu_device *adev, 661 bool value) 662 { 663 uint32_t xcc_mask; 664 665 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 666 gfxhub_v12_1_xcc_set_fault_enable_default(adev, value, xcc_mask); 667 } 668 669 static uint32_t gfxhub_v12_1_get_invalidate_req(unsigned int vmid, 670 uint32_t flush_type) 671 { 672 u32 req = 0; 673 674 /* invalidate using legacy mode on vmid*/ 675 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 676 PER_VMID_INVALIDATE_REQ, 1 << vmid); 677 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 678 FLUSH_TYPE, flush_type); 679 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 680 INVALIDATE_L2_PTES, 1); 681 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 682 INVALIDATE_L2_PDE0, 1); 683 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 684 INVALIDATE_L2_PDE1, 1); 685 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 686 INVALIDATE_L2_PDE2, 1); 687 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 688 INVALIDATE_L2_PDE3, 1); 689 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 690 INVALIDATE_L1_PTES, 1); 691 req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 692 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 693 694 return req; 695 } 696 697 static const char *gfxhub_v12_1_client_ids[] = { 698 "CB", 699 "DB", 700 "GE1", 701 "GE2", 702 "CPF", 703 "CPC", 704 "CPG", 705 "RLC", 706 "TCP", 707 "SQC (inst)", 708 "SQC (data)", 709 "SQG/PC/SC", 710 "Reserved", 711 "SDMA0", 712 "SDMA1", 713 "GCR", 714 "Reserved", 715 "Reserved", 716 "WGS", 717 "DSM", 718 "PA" 719 }; 720 721 /*TODO: l2 protection fault status is increased to 64bits. 722 * some critical fields like FED are moved to STATUS_HI32 */ 723 static void gfxhub_v12_1_print_l2_protection_fault_status(struct amdgpu_device *adev, 724 uint32_t status) 725 { 726 u32 cid = REG_GET_FIELD(status, 727 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, 728 CID); 729 730 dev_err(adev->dev, 731 "GCVM_L2_PROTECTION_FAULT_STATUS_LO32:0x%08X\n", 732 status); 733 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 734 cid >= ARRAY_SIZE(gfxhub_v12_1_client_ids) ? 735 "unknown" : gfxhub_v12_1_client_ids[cid], cid); 736 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 737 REG_GET_FIELD(status, 738 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MORE_FAULTS)); 739 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 740 REG_GET_FIELD(status, 741 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, WALKER_ERROR)); 742 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 743 REG_GET_FIELD(status, 744 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, PERMISSION_FAULTS)); 745 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 746 REG_GET_FIELD(status, 747 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, MAPPING_ERROR)); 748 dev_err(adev->dev, "\t RW: 0x%lx\n", 749 REG_GET_FIELD(status, 750 GCVM_L2_PROTECTION_FAULT_STATUS_LO32, RW)); 751 } 752 753 static const struct amdgpu_vmhub_funcs gfxhub_v12_1_vmhub_funcs = { 754 .print_l2_protection_fault_status = gfxhub_v12_1_print_l2_protection_fault_status, 755 .get_invalidate_req = gfxhub_v12_1_get_invalidate_req, 756 }; 757 758 static void gfxhub_v12_1_xcc_init(struct amdgpu_device *adev, uint32_t xcc_mask) 759 { 760 struct amdgpu_vmhub *hub; 761 int i; 762 763 for_each_inst(i, xcc_mask) { 764 hub = &adev->vmhub[AMDGPU_GFXHUB(i)]; 765 766 hub->ctx0_ptb_addr_lo32 = 767 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 768 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 769 hub->ctx0_ptb_addr_hi32 = 770 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 771 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 772 hub->vm_inv_eng0_sem = 773 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 774 regGCVM_INVALIDATE_ENG0_SEM); 775 hub->vm_inv_eng0_req = 776 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 777 regGCVM_INVALIDATE_ENG0_REQ); 778 hub->vm_inv_eng0_ack = 779 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 780 regGCVM_INVALIDATE_ENG0_ACK); 781 hub->vm_context0_cntl = 782 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 783 regGCVM_CONTEXT0_CNTL); 784 /* TODO: add a new member to accomandate additional fault status/cntl reg */ 785 hub->vm_l2_pro_fault_status = 786 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 787 regGCVM_L2_PROTECTION_FAULT_STATUS_LO32); 788 hub->vm_l2_pro_fault_cntl = 789 SOC15_REG_OFFSET(GC, GET_INST(GC, i), 790 regGCVM_L2_PROTECTION_FAULT_CNTL_LO32); 791 hub->ctx_distance = 792 regGCVM_CONTEXT1_CNTL - 793 regGCVM_CONTEXT0_CNTL; 794 hub->ctx_addr_distance = 795 regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 796 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 797 hub->eng_distance = 798 regGCVM_INVALIDATE_ENG1_REQ - 799 regGCVM_INVALIDATE_ENG0_REQ; 800 hub->eng_addr_distance = 801 regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 802 regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 803 804 hub->vm_cntx_cntl_vm_fault = 805 GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 806 GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 807 GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 808 GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 809 GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 810 GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 811 GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 812 813 hub->vmhub_funcs = &gfxhub_v12_1_vmhub_funcs; 814 } 815 } 816 817 static void gfxhub_v12_1_init(struct amdgpu_device *adev) 818 { 819 uint32_t xcc_mask; 820 821 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 822 gfxhub_v12_1_xcc_init(adev, xcc_mask); 823 } 824 825 const struct amdgpu_gfxhub_funcs gfxhub_v12_1_funcs = { 826 .get_fb_location = gfxhub_v12_1_get_fb_location, 827 .get_mc_fb_offset = gfxhub_v12_1_get_mc_fb_offset, 828 .setup_vm_pt_regs = gfxhub_v12_1_setup_vm_pt_regs, 829 .gart_enable = gfxhub_v12_1_gart_enable, 830 .gart_disable = gfxhub_v12_1_gart_disable, 831 .set_fault_enable_default = gfxhub_v12_1_set_fault_enable_default, 832 .init = gfxhub_v12_1_init, 833 }; 834 835 static int gfxhub_v12_1_xcp_resume(void *handle, uint32_t inst_mask) 836 { 837 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 838 bool value; 839 840 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 841 value = false; 842 else 843 value = true; 844 845 gfxhub_v12_1_xcc_set_fault_enable_default(adev, value, inst_mask); 846 847 if (!amdgpu_sriov_vf(adev)) 848 return gfxhub_v12_1_xcc_gart_enable(adev, inst_mask); 849 850 return 0; 851 } 852 853 static int gfxhub_v12_1_xcp_suspend(void *handle, uint32_t inst_mask) 854 { 855 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 856 857 if (!amdgpu_sriov_vf(adev)) 858 gfxhub_v12_1_xcc_gart_disable(adev, inst_mask); 859 860 return 0; 861 } 862 863 struct amdgpu_xcp_ip_funcs gfxhub_v12_1_xcp_funcs = { 864 .suspend = &gfxhub_v12_1_xcp_suspend, 865 .resume = &gfxhub_v12_1_xcp_resume 866 }; 867