/* * Copyright 2025 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include "amdgpu.h" #include "gmc_v12_1.h" #include "soc15_common.h" #include "soc_v1_0_enum.h" #include "oss/osssys_7_1_0_offset.h" #include "oss/osssys_7_1_0_sh_mask.h" #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev, struct amdgpu_irq_src *src, unsigned int type, enum amdgpu_interrupt_state state) { struct amdgpu_vmhub *hub; u32 tmp, reg, i, j; switch (state) { case AMDGPU_IRQ_STATE_DISABLE: for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { hub = &adev->vmhub[j]; for (i = 0; i < 16; i++) { reg = hub->vm_context0_cntl + i; /* This works because this interrupt is only * enabled at init/resume and disabled in * fini/suspend, so the overall state doesn't * change over the course of suspend/resume. */ if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) continue; if (j >= AMDGPU_MMHUB0(0)) tmp = RREG32_SOC15_IP(MMHUB, reg); else tmp = RREG32_XCC(reg, j); tmp &= ~hub->vm_cntx_cntl_vm_fault; if (j >= AMDGPU_MMHUB0(0)) WREG32_SOC15_IP(MMHUB, reg, tmp); else WREG32_XCC(reg, tmp, j); } } break; case AMDGPU_IRQ_STATE_ENABLE: for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { hub = &adev->vmhub[j]; for (i = 0; i < 16; i++) { reg = hub->vm_context0_cntl + i; /* This works because this interrupt is only * enabled at init/resume and disabled in * fini/suspend, so the overall state doesn't * change over the course of suspend/resume. */ if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) continue; if (j >= AMDGPU_MMHUB0(0)) tmp = RREG32_SOC15_IP(MMHUB, reg); else tmp = RREG32_XCC(reg, j); tmp |= hub->vm_cntx_cntl_vm_fault; if (j >= AMDGPU_MMHUB0(0)) WREG32_SOC15_IP(MMHUB, reg, tmp); else WREG32_XCC(reg, tmp, j); } } break; default: break; } return 0; } static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev, struct amdgpu_irq_src *source, struct amdgpu_iv_entry *entry) { struct amdgpu_task_info *task_info; bool retry_fault = false, write_fault = false; unsigned int vmhub, node_id; struct amdgpu_vmhub *hub; uint32_t cam_index = 0; const char *hub_name; int ret, xcc_id = 0; uint32_t status = 0; u64 addr; node_id = entry->node_id; addr = (u64)entry->src_data[0] << 12; addr |= ((u64)entry->src_data[1] & 0x1fff) << 44; if (entry->src_id == UTCL2_1_0__SRCID__RETRY) { retry_fault = true; write_fault = !!(entry->src_data[1] & 0x200000); } if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) { hub_name = "mmhub0"; vmhub = AMDGPU_MMHUB0(node_id / 4); } else { hub_name = "gfxhub0"; if (adev->gfx.funcs->ih_node_to_logical_xcc) { xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, node_id); if (xcc_id < 0) xcc_id = 0; } vmhub = xcc_id; } hub = &adev->vmhub[vmhub]; if (retry_fault) { if (adev->irq.retry_cam_enabled) { /* Delegate it to a different ring if the hardware hasn't * already done it. */ if (entry->ih == &adev->irq.ih) { amdgpu_irq_delegate(adev, entry, 8); return 1; } cam_index = entry->src_data[3] & 0x3ff; ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, addr, entry->timestamp, write_fault); WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); if (ret) return 1; } else { /* Process it onyl if it's the first fault for this address */ if (entry->ih != &adev->irq.ih_soft && amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, entry->timestamp)) return 1; /* Delegate it to a different ring if the hardware hasn't * already done it. */ if (entry->ih == &adev->irq.ih) { amdgpu_irq_delegate(adev, entry, 8); return 1; } /* Try to handle the recoverable page faults by filling page * tables */ if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, addr, entry->timestamp, write_fault)) return 1; } } if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault)) return 1; if (!printk_ratelimit()) return 0; dev_err(adev->dev, "[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name, retry_fault ? "retry" : "no-retry", entry->src_id, entry->ring_id, entry->vmid, entry->pasid); task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid); if (task_info) { amdgpu_vm_print_task_info(adev, task_info); amdgpu_vm_put_task_info(task_info); } dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n", addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]); if (amdgpu_sriov_vf(adev)) return 0; /* * Issue a dummy read to wait for the status register to * be updated to avoid reading an incorrect value due to * the new fast GRBM interface. */ if (entry->vmid_src == AMDGPU_GFXHUB(0)) RREG32(hub->vm_l2_pro_fault_status); status = RREG32(hub->vm_l2_pro_fault_status); /* Only print L2 fault status if the status register could be read and * contains useful information */ if (!status) return 0; WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub); hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); return 0; } static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev, uint8_t vmid, uint8_t inst, uint16_t *p_pasid) { uint16_t index; if (inst/4) index = 0xA + inst%4; else index = 0x2 + inst%4; WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_LUT_INDEX), index); *p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff; return !!(*p_pasid); } /* * GART * VMID 0 is the physical GPU addresses as used by the kernel. * VMIDs 1-15 are used for userspace clients and are handled * by the amdgpu vm/hsa code. */ static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, unsigned int vmhub, uint32_t flush_type) { struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); u32 tmp; /* Use register 17 for GART */ const unsigned eng = 17; unsigned int i; unsigned char hub_ip = 0; hub_ip = (AMDGPU_IS_GFXHUB(vmhub)) ? GC_HWIP : MMHUB_HWIP; spin_lock(&adev->gmc.invalidate_lock); WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip); /* Wait for ACK with a delay.*/ for (i = 0; i < adev->usec_timeout; i++) { tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack + hub->eng_distance * eng, hub_ip); tmp &= 1 << vmid; if (tmp) break; udelay(1); } /* Issue additional private vm invalidation to MMHUB */ if (!AMDGPU_IS_GFXHUB(vmhub) && (hub->vm_l2_bank_select_reserved_cid2) && !amdgpu_sriov_vf(adev)) { inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ inv_req |= (1 << 25); /* Issue private invalidation */ WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req); /* Read back to ensure invalidation is done*/ RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); } spin_unlock(&adev->gmc.invalidate_lock); if (i < adev->usec_timeout) return; dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n"); } /** * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback * * @adev: amdgpu_device pointer * @vmid: vm instance to flush * @vmhub: which hub to flush * @flush_type: the flush type * * Flush the TLB for the requested page table. */ static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, uint32_t vmhub, uint32_t flush_type) { u32 inst; if (AMDGPU_IS_GFXHUB(vmhub) && !adev->gfx.is_poweron) return; if (vmhub >= AMDGPU_MMHUB0(0)) inst = 0; else inst = vmhub; /* This is necessary for SRIOV as well as for GFXOFF to function * properly under bare metal */ if (((adev->gfx.kiq[inst].ring.sched.ready || adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready) && (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) { struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; const unsigned eng = 17; u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, 1 << vmid, inst); return; } mutex_lock(&adev->mman.gtt_window_lock); gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0); mutex_unlock(&adev->mman.gtt_window_lock); return; } /** * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid * * @adev: amdgpu_device pointer * @pasid: pasid to be flush * @flush_type: the flush type * @all_hub: flush all hubs * @inst: is used to select which instance of KIQ to use for the invalidation * * Flush the TLB for the requested pasid. */ static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev, uint16_t pasid, uint32_t flush_type, bool all_hub, uint32_t inst) { uint16_t queried; int vmid, i; if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready && (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) { struct mes_inv_tlbs_pasid_input input = {0}; input.xcc_id = inst; input.pasid = pasid; input.flush_type = flush_type; /* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */ if (!amdgpu_gfx_is_master_xcc(adev, inst)) return; input.hub_id = AMDGPU_GFXHUB(0); adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); if (all_hub) { /* invalidate mm_hub */ if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { input.hub_id = AMDGPU_MMHUB0(0); adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); } if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { input.hub_id = AMDGPU_MMHUB1(0); adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); } } return; } for (vmid = 1; vmid < 16; vmid++) { bool valid; valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid, inst, &queried); if (!valid || queried != pasid) continue; if (all_hub) { for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) gmc_v12_1_flush_gpu_tlb(adev, vmid, i, flush_type); } else { gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst), flush_type); } } } static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring, unsigned vmid, uint64_t pd_addr) { struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); unsigned eng = ring->vm_inv_eng; amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (hub->ctx_addr_distance * vmid), lower_32_bits(pd_addr)); amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (hub->ctx_addr_distance * vmid), upper_32_bits(pd_addr)); amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + hub->eng_distance * eng, hub->vm_inv_eng0_ack + hub->eng_distance * eng, req, 1 << vmid); return pd_addr; } static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, unsigned pasid) { struct amdgpu_device *adev = ring->adev; uint32_t reg; if (ring->vm_hub == AMDGPU_GFXHUB(0)) reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid; else reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid; amdgpu_ring_emit_wreg(ring, reg, pasid); } /* * PTE format: * 63 P * 62:59 reserved * 58 D * 57 G * 56 T * 55:54 M * 53:52 SW * 51:48 reserved for future * 47:12 4k physical page base address * 11:7 fragment * 6 write * 5 read * 4 exe * 3 Z * 2 snooped * 1 system * 0 valid * * PDE format: * 63 P * 62:58 block fragment size * 57 reserved * 56 A * 55:54 M * 53:52 reserved * 51:48 reserved for future * 47:6 physical base address of PD or PTE * 5:3 reserved * 2 C * 1 system * 0 valid */ static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level, uint64_t *addr, uint64_t *flags) { if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM)) *addr = adev->vm_manager.vram_base_offset + *addr - adev->gmc.vram_start; BUG_ON(*addr & 0xFFFF00000000003FULL); *flags |= AMDGPU_PTE_SNOOPED; if (!adev->gmc.translate_further) return; if (level == AMDGPU_VM_PDB1) { /* Set the block fragment size */ if (!(*flags & AMDGPU_PDE_PTE_GFX12)) *flags |= AMDGPU_PDE_BFS_GFX12(0x9); } else if (level == AMDGPU_VM_PDB0) { if (*flags & AMDGPU_PDE_PTE_GFX12) *flags &= ~AMDGPU_PDE_PTE_GFX12; } } static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev, struct amdgpu_bo *bo, uint64_t *flags) { struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); bool is_vram = bo->tbo.resource && bo->tbo.resource->mem_type == TTM_PL_VRAM; bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT | AMDGPU_GEM_CREATE_EXT_COHERENT); bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT; uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0); bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED; unsigned int mtype, mtype_local; bool snoop = false; bool is_local = false; switch (gc_ip_version) { case IP_VERSION(12, 1, 0): mtype_local = MTYPE_RW; if (amdgpu_mtype_local == 1) { DRM_INFO_ONCE("Using MTYPE_NC for local memory\n"); mtype_local = MTYPE_NC; } else if (amdgpu_mtype_local == 2) { DRM_INFO_ONCE("MTYPE_CC not supported, using MTYPE_RW instead for local memory\n"); } else { DRM_INFO_ONCE("Using MTYPE_RW for local memory\n"); } is_local = (is_vram && adev == bo_adev); snoop = true; if (uncached) { mtype = MTYPE_UC; } else if (ext_coherent) { mtype = is_local ? mtype_local : MTYPE_UC; } else { if (is_local) mtype = mtype_local; else mtype = MTYPE_NC; } break; default: if (uncached || coherent) mtype = MTYPE_UC; else mtype = MTYPE_NC; } if (mtype != MTYPE_NC) *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype); if (is_local || adev->have_atomics_support) *flags |= AMDGPU_PTE_BUS_ATOMICS; *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; } static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev, struct amdgpu_vm *vm, struct amdgpu_bo *bo, uint32_t vm_flags, uint64_t *flags) { if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE) *flags |= AMDGPU_PTE_EXECUTABLE; else *flags &= ~AMDGPU_PTE_EXECUTABLE; switch (vm_flags & AMDGPU_VM_MTYPE_MASK) { case AMDGPU_VM_MTYPE_DEFAULT: *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); break; case AMDGPU_VM_MTYPE_NC: default: *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); break; case AMDGPU_VM_MTYPE_RW: *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_RW); break; case AMDGPU_VM_MTYPE_UC: *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC); break; } if ((*flags & AMDGPU_PTE_VALID) && bo) gmc_v12_1_get_coherence_flags(adev, bo, flags); } static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = { .flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb, .flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid, .emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb, .emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping, .get_vm_pde = gmc_v12_1_get_vm_pde, .get_vm_pte = gmc_v12_1_get_vm_pte, .query_mem_partition_mode = &amdgpu_gmc_query_memory_partition, .request_mem_partition_mode = &amdgpu_gmc_request_memory_partition, }; void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev) { adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs; } static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = { .set = gmc_v12_1_vm_fault_interrupt_state, .process = gmc_v12_1_process_interrupt, }; void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev) { adev->gmc.vm_fault.num_types = 1; adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs; } void gmc_v12_1_init_vram_info(struct amdgpu_device *adev) { /* TODO: query vram_info from ip discovery binary */ adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4; adev->gmc.vram_width = 384 * 64; }