1 /* 2 * Copyright 2025 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "gmc_v12_1.h" 25 #include "soc15_common.h" 26 #include "soc_v1_0_enum.h" 27 #include "oss/osssys_7_1_0_offset.h" 28 #include "oss/osssys_7_1_0_sh_mask.h" 29 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 30 31 static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev, 32 struct amdgpu_irq_src *src, 33 unsigned int type, 34 enum amdgpu_interrupt_state state) 35 { 36 struct amdgpu_vmhub *hub; 37 u32 tmp, reg, i, j; 38 39 switch (state) { 40 case AMDGPU_IRQ_STATE_DISABLE: 41 for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 42 hub = &adev->vmhub[j]; 43 for (i = 0; i < 16; i++) { 44 reg = hub->vm_context0_cntl + i; 45 46 /* This works because this interrupt is only 47 * enabled at init/resume and disabled in 48 * fini/suspend, so the overall state doesn't 49 * change over the course of suspend/resume. 50 */ 51 if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) 52 continue; 53 54 if (j >= AMDGPU_MMHUB0(0)) 55 tmp = RREG32_SOC15_IP(MMHUB, reg); 56 else 57 tmp = RREG32_XCC(reg, j); 58 59 tmp &= ~hub->vm_cntx_cntl_vm_fault; 60 61 if (j >= AMDGPU_MMHUB0(0)) 62 WREG32_SOC15_IP(MMHUB, reg, tmp); 63 else 64 WREG32_XCC(reg, tmp, j); 65 } 66 } 67 break; 68 case AMDGPU_IRQ_STATE_ENABLE: 69 for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 70 hub = &adev->vmhub[j]; 71 for (i = 0; i < 16; i++) { 72 reg = hub->vm_context0_cntl + i; 73 74 /* This works because this interrupt is only 75 * enabled at init/resume and disabled in 76 * fini/suspend, so the overall state doesn't 77 * change over the course of suspend/resume. 78 */ 79 if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) 80 continue; 81 82 if (j >= AMDGPU_MMHUB0(0)) 83 tmp = RREG32_SOC15_IP(MMHUB, reg); 84 else 85 tmp = RREG32_XCC(reg, j); 86 87 tmp |= hub->vm_cntx_cntl_vm_fault; 88 89 if (j >= AMDGPU_MMHUB0(0)) 90 WREG32_SOC15_IP(MMHUB, reg, tmp); 91 else 92 WREG32_XCC(reg, tmp, j); 93 } 94 } 95 break; 96 default: 97 break; 98 } 99 100 return 0; 101 } 102 103 static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev, 104 struct amdgpu_irq_src *source, 105 struct amdgpu_iv_entry *entry) 106 { 107 struct amdgpu_task_info *task_info; 108 bool retry_fault = false, write_fault = false; 109 unsigned int vmhub, node_id; 110 struct amdgpu_vmhub *hub; 111 uint32_t cam_index = 0; 112 const char *hub_name; 113 int ret, xcc_id = 0; 114 uint32_t status = 0; 115 const char *die_name; 116 char die_name_buf[32]; 117 u64 addr; 118 119 node_id = entry->node_id; 120 121 addr = (u64)entry->src_data[0] << 12; 122 addr |= ((u64)entry->src_data[1] & 0x1fff) << 44; 123 124 if (entry->src_id == UTCL2_1_0__SRCID__RETRY) { 125 retry_fault = true; 126 write_fault = !!(entry->src_data[1] & AMDGPU_GMC121_FAULT_SOURCE_DATA_WRITE); 127 } 128 129 if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) { 130 hub_name = "mmhub0"; 131 vmhub = AMDGPU_MMHUB0(node_id / 4); 132 } else { 133 hub_name = "gfxhub0"; 134 if (adev->gfx.funcs->ih_node_to_logical_xcc) { 135 xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, 136 node_id); 137 if (xcc_id < 0) 138 xcc_id = 0; 139 } 140 vmhub = xcc_id; 141 } 142 143 hub = &adev->vmhub[vmhub]; 144 145 if (retry_fault) { 146 if (adev->irq.retry_cam_enabled) { 147 /* Delegate it to a different ring if the hardware hasn't 148 * already done it. 149 */ 150 if (entry->ih == &adev->irq.ih) { 151 amdgpu_irq_delegate(adev, entry, 8); 152 return 1; 153 } 154 155 cam_index = entry->src_data[3] & 0x3ff; 156 157 ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 158 addr, entry->timestamp, write_fault); 159 WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); 160 if (ret) 161 return 1; 162 } else { 163 /* Process it onyl if it's the first fault for this address */ 164 if (entry->ih != &adev->irq.ih_soft && 165 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 166 entry->timestamp)) 167 return 1; 168 169 /* Delegate it to a different ring if the hardware hasn't 170 * already done it. 171 */ 172 if (entry->ih == &adev->irq.ih) { 173 amdgpu_irq_delegate(adev, entry, 8); 174 return 1; 175 } 176 177 /* Try to handle the recoverable page faults by filling page 178 * tables 179 */ 180 if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 181 addr, entry->timestamp, write_fault)) 182 return 1; 183 } 184 } 185 186 if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault)) 187 return 1; 188 189 if (!printk_ratelimit()) 190 return 0; 191 192 dev_err(adev->dev, 193 "[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name, 194 retry_fault ? "retry" : "no-retry", 195 entry->src_id, entry->ring_id, entry->vmid, entry->pasid); 196 197 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid); 198 if (task_info) { 199 amdgpu_vm_print_task_info(adev, task_info); 200 amdgpu_vm_put_task_info(task_info); 201 } 202 203 dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n", 204 addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]); 205 206 if (adev->irq.ih_funcs && 207 adev->irq.ih_funcs->node_id_to_die_name) { 208 die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id, 209 die_name_buf, 210 sizeof(die_name_buf)); 211 if (die_name) 212 dev_err(adev->dev, 213 " cookie node_id %d fault from die %s\n", 214 node_id, die_name); 215 } 216 217 if (amdgpu_sriov_vf(adev)) 218 return 0; 219 220 /* 221 * Issue a dummy read to wait for the status register to 222 * be updated to avoid reading an incorrect value due to 223 * the new fast GRBM interface. 224 */ 225 if (entry->vmid_src == AMDGPU_GFXHUB(0)) 226 RREG32(hub->vm_l2_pro_fault_status); 227 228 status = RREG32(hub->vm_l2_pro_fault_status); 229 230 /* Only print L2 fault status if the status register could be read and 231 * contains useful information 232 */ 233 if (!status) 234 return 0; 235 236 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 237 238 amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub); 239 240 hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); 241 242 return 0; 243 } 244 245 static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev, 246 uint8_t vmid, uint8_t inst, 247 uint16_t *p_pasid) 248 { 249 uint16_t index; 250 251 if (inst/4) 252 index = 0xA + inst%4; 253 else 254 index = 0x2 + inst%4; 255 256 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_LUT_INDEX), index); 257 258 *p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff; 259 260 return !!(*p_pasid); 261 } 262 263 /* 264 * GART 265 * VMID 0 is the physical GPU addresses as used by the kernel. 266 * VMIDs 1-15 are used for userspace clients and are handled 267 * by the amdgpu vm/hsa code. 268 */ 269 270 static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, 271 unsigned int vmhub, uint32_t flush_type) 272 { 273 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 274 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); 275 u32 tmp; 276 /* Use register 17 for GART */ 277 const unsigned eng = 17; 278 unsigned int i; 279 unsigned char hub_ip = 0; 280 281 hub_ip = (AMDGPU_IS_GFXHUB(vmhub)) ? 282 GC_HWIP : MMHUB_HWIP; 283 284 spin_lock(&adev->gmc.invalidate_lock); 285 286 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip); 287 288 /* Wait for ACK with a delay.*/ 289 for (i = 0; i < adev->usec_timeout; i++) { 290 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack + 291 hub->eng_distance * eng, hub_ip); 292 tmp &= 1 << vmid; 293 if (tmp) 294 break; 295 296 udelay(1); 297 } 298 299 /* Issue additional private vm invalidation to MMHUB */ 300 if (!AMDGPU_IS_GFXHUB(vmhub) && 301 (hub->vm_l2_bank_select_reserved_cid2) && 302 !amdgpu_sriov_vf(adev)) { 303 inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); 304 /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ 305 inv_req |= (1 << 25); 306 /* Issue private invalidation */ 307 WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req); 308 /* Read back to ensure invalidation is done*/ 309 RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); 310 } 311 312 spin_unlock(&adev->gmc.invalidate_lock); 313 314 if (i < adev->usec_timeout) 315 return; 316 317 dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n"); 318 } 319 320 /** 321 * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback 322 * 323 * @adev: amdgpu_device pointer 324 * @vmid: vm instance to flush 325 * @vmhub: which hub to flush 326 * @flush_type: the flush type 327 * 328 * Flush the TLB for the requested page table. 329 */ 330 static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 331 uint32_t vmhub, uint32_t flush_type) 332 { 333 u32 inst; 334 335 if (AMDGPU_IS_GFXHUB(vmhub) && 336 !adev->gfx.is_poweron) 337 return; 338 339 if (vmhub >= AMDGPU_MMHUB0(0)) 340 inst = 0; 341 else 342 inst = vmhub; 343 344 /* This is necessary for SRIOV as well as for GFXOFF to function 345 * properly under bare metal 346 */ 347 if (((adev->gfx.kiq[inst].ring.sched.ready || 348 adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready) && 349 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) { 350 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 351 const unsigned eng = 17; 352 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); 353 u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 354 u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 355 356 amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, 357 1 << vmid, inst); 358 return; 359 } 360 361 gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0); 362 return; 363 } 364 365 /** 366 * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid 367 * 368 * @adev: amdgpu_device pointer 369 * @pasid: pasid to be flush 370 * @flush_type: the flush type 371 * @all_hub: flush all hubs 372 * @inst: is used to select which instance of KIQ to use for the invalidation 373 * 374 * Flush the TLB for the requested pasid. 375 */ 376 static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 377 uint16_t pasid, uint32_t flush_type, 378 bool all_hub, uint32_t inst) 379 { 380 uint16_t queried; 381 int vmid, i; 382 383 if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready && 384 (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) { 385 struct mes_inv_tlbs_pasid_input input = {0}; 386 input.xcc_id = inst; 387 input.pasid = pasid; 388 input.flush_type = flush_type; 389 390 /* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */ 391 if (!amdgpu_gfx_is_master_xcc(adev, inst)) 392 return; 393 394 input.hub_id = AMDGPU_GFXHUB(0); 395 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); 396 397 if (all_hub) { 398 /* invalidate mm_hub */ 399 if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { 400 input.hub_id = AMDGPU_MMHUB0(0); 401 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); 402 } 403 if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { 404 input.hub_id = AMDGPU_MMHUB1(0); 405 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); 406 } 407 } 408 return; 409 } 410 411 for (vmid = 1; vmid < 16; vmid++) { 412 bool valid; 413 414 valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid, inst, 415 &queried); 416 if (!valid || queried != pasid) 417 continue; 418 419 if (all_hub) { 420 for_each_set_bit(i, adev->vmhubs_mask, 421 AMDGPU_MAX_VMHUBS) 422 gmc_v12_1_flush_gpu_tlb(adev, vmid, i, 423 flush_type); 424 } else { 425 gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst), 426 flush_type); 427 } 428 } 429 } 430 431 static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 432 unsigned vmid, uint64_t pd_addr) 433 { 434 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; 435 uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); 436 unsigned eng = ring->vm_inv_eng; 437 438 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 439 (hub->ctx_addr_distance * vmid), 440 lower_32_bits(pd_addr)); 441 442 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 443 (hub->ctx_addr_distance * vmid), 444 upper_32_bits(pd_addr)); 445 446 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 447 hub->eng_distance * eng, 448 hub->vm_inv_eng0_ack + 449 hub->eng_distance * eng, 450 req, 1 << vmid); 451 452 return pd_addr; 453 } 454 455 static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring, 456 unsigned vmid, unsigned pasid) 457 { 458 struct amdgpu_device *adev = ring->adev; 459 uint32_t reg; 460 461 if (ring->vm_hub == AMDGPU_GFXHUB(0)) 462 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid; 463 else 464 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid; 465 466 amdgpu_ring_emit_wreg(ring, reg, pasid); 467 } 468 469 /* 470 * PTE format: 471 * 63 P 472 * 62:59 reserved 473 * 58 D 474 * 57 G 475 * 56 T 476 * 55:54 M 477 * 53:52 SW 478 * 51:48 reserved for future 479 * 47:12 4k physical page base address 480 * 11:7 fragment 481 * 6 write 482 * 5 read 483 * 4 exe 484 * 3 Z 485 * 2 snooped 486 * 1 system 487 * 0 valid 488 * 489 * PDE format: 490 * 63 P 491 * 62:58 block fragment size 492 * 57 reserved 493 * 56 A 494 * 55:54 M 495 * 53:52 reserved 496 * 51:48 reserved for future 497 * 47:6 physical base address of PD or PTE 498 * 5:3 reserved 499 * 2 C 500 * 1 system 501 * 0 valid 502 */ 503 504 static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level, 505 uint64_t *addr, uint64_t *flags) 506 { 507 if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM)) 508 *addr = adev->vm_manager.vram_base_offset + *addr - 509 adev->gmc.vram_start; 510 BUG_ON(*addr & 0xFFFF00000000003FULL); 511 512 *flags |= AMDGPU_PTE_SNOOPED; 513 514 if (!adev->gmc.translate_further) 515 return; 516 517 if (level == AMDGPU_VM_PDB1) { 518 /* Set the block fragment size */ 519 if (!(*flags & AMDGPU_PDE_PTE_GFX12)) 520 *flags |= AMDGPU_PDE_BFS_GFX12(0x9); 521 522 } else if (level == AMDGPU_VM_PDB0) { 523 if (*flags & AMDGPU_PDE_PTE_GFX12) 524 *flags &= ~AMDGPU_PDE_PTE_GFX12; 525 } 526 } 527 528 static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev, 529 struct amdgpu_bo *bo, 530 uint64_t *flags) 531 { 532 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 533 bool is_vram = bo->tbo.resource && 534 bo->tbo.resource->mem_type == TTM_PL_VRAM; 535 bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT | 536 AMDGPU_GEM_CREATE_EXT_COHERENT); 537 bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT; 538 uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0); 539 bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED; 540 unsigned int mtype, mtype_local, mtype_remote; 541 bool snoop = false; 542 bool is_local = false; 543 bool is_aid_a1; 544 545 switch (gc_ip_version) { 546 case IP_VERSION(12, 1, 0): 547 is_aid_a1 = (adev->rev_id & 0x10); 548 549 mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC; 550 mtype_remote = is_aid_a1 ? MTYPE_NC : MTYPE_UC; 551 if (amdgpu_mtype_local == 0) { 552 DRM_INFO_ONCE("Using MTYPE_RW for local memory\n"); 553 mtype_local = MTYPE_RW; 554 } else if (amdgpu_mtype_local == 1) { 555 DRM_INFO_ONCE("Using MTYPE_NC for local memory\n"); 556 mtype_local = MTYPE_NC; 557 } else if (amdgpu_mtype_local == 2) { 558 DRM_INFO_ONCE("MTYPE_CC not supported, using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC"); 559 } else { 560 DRM_INFO_ONCE("Using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC"); 561 } 562 563 is_local = (is_vram && adev == bo_adev); 564 snoop = true; 565 if (uncached) { 566 mtype = MTYPE_UC; 567 } else if (ext_coherent) { 568 mtype = is_local ? mtype_local : MTYPE_UC; 569 } else { 570 mtype = is_local ? mtype_local : mtype_remote; 571 } 572 break; 573 default: 574 if (uncached || coherent) 575 mtype = MTYPE_UC; 576 else 577 mtype = MTYPE_NC; 578 } 579 580 if (mtype != MTYPE_NC) 581 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype); 582 583 if (is_local || adev->have_atomics_support) 584 *flags |= AMDGPU_PTE_BUS_ATOMICS; 585 586 *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; 587 } 588 589 static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev, 590 struct amdgpu_vm *vm, 591 struct amdgpu_bo *bo, 592 uint32_t vm_flags, 593 uint64_t *flags) 594 { 595 if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE) 596 *flags |= AMDGPU_PTE_EXECUTABLE; 597 else 598 *flags &= ~AMDGPU_PTE_EXECUTABLE; 599 600 switch (vm_flags & AMDGPU_VM_MTYPE_MASK) { 601 case AMDGPU_VM_MTYPE_DEFAULT: 602 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); 603 break; 604 case AMDGPU_VM_MTYPE_NC: 605 default: 606 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); 607 break; 608 case AMDGPU_VM_MTYPE_RW: 609 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_RW); 610 break; 611 case AMDGPU_VM_MTYPE_UC: 612 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC); 613 break; 614 } 615 616 if ((*flags & AMDGPU_PTE_VALID) && bo) 617 gmc_v12_1_get_coherence_flags(adev, bo, flags); 618 } 619 620 static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = { 621 .flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb, 622 .flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid, 623 .emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb, 624 .emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping, 625 .get_vm_pde = gmc_v12_1_get_vm_pde, 626 .get_vm_pte = gmc_v12_1_get_vm_pte, 627 .query_mem_partition_mode = &amdgpu_gmc_query_memory_partition, 628 .request_mem_partition_mode = &amdgpu_gmc_request_memory_partition, 629 }; 630 631 void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev) 632 { 633 adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs; 634 } 635 636 static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = { 637 .set = gmc_v12_1_vm_fault_interrupt_state, 638 .process = gmc_v12_1_process_interrupt, 639 }; 640 641 static const struct amdgpu_irq_src_funcs gmc_v12_1_ecc_funcs = { 642 .process = amdgpu_umc_uniras_process_ecc_irq, 643 }; 644 645 void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev) 646 { 647 adev->gmc.vm_fault.num_types = 1; 648 adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs; 649 650 adev->gmc.ecc_irq.num_types = 1; 651 adev->gmc.ecc_irq.funcs = &gmc_v12_1_ecc_funcs; 652 } 653 654 void gmc_v12_1_init_vram_info(struct amdgpu_device *adev) 655 { 656 /* TODO: query vram_info from ip discovery binary */ 657 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4; 658 adev->gmc.vram_width = 384 * 64; 659 } 660