1 /* 2 * Copyright 2025 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include "amdgpu.h" 24 #include "gmc_v12_1.h" 25 #include "soc15_common.h" 26 #include "soc_v1_0_enum.h" 27 #include "oss/osssys_7_1_0_offset.h" 28 #include "oss/osssys_7_1_0_sh_mask.h" 29 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 30 31 static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev, 32 struct amdgpu_irq_src *src, 33 unsigned int type, 34 enum amdgpu_interrupt_state state) 35 { 36 struct amdgpu_vmhub *hub; 37 u32 tmp, reg, i, j; 38 39 switch (state) { 40 case AMDGPU_IRQ_STATE_DISABLE: 41 for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 42 hub = &adev->vmhub[j]; 43 for (i = 0; i < 16; i++) { 44 reg = hub->vm_context0_cntl + i; 45 46 /* This works because this interrupt is only 47 * enabled at init/resume and disabled in 48 * fini/suspend, so the overall state doesn't 49 * change over the course of suspend/resume. 50 */ 51 if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) 52 continue; 53 54 if (j >= AMDGPU_MMHUB0(0)) 55 tmp = RREG32_SOC15_IP(MMHUB, reg); 56 else 57 tmp = RREG32_XCC(reg, j); 58 59 tmp &= ~hub->vm_cntx_cntl_vm_fault; 60 61 if (j >= AMDGPU_MMHUB0(0)) 62 WREG32_SOC15_IP(MMHUB, reg, tmp); 63 else 64 WREG32_XCC(reg, tmp, j); 65 } 66 } 67 break; 68 case AMDGPU_IRQ_STATE_ENABLE: 69 for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 70 hub = &adev->vmhub[j]; 71 for (i = 0; i < 16; i++) { 72 reg = hub->vm_context0_cntl + i; 73 74 /* This works because this interrupt is only 75 * enabled at init/resume and disabled in 76 * fini/suspend, so the overall state doesn't 77 * change over the course of suspend/resume. 78 */ 79 if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) 80 continue; 81 82 if (j >= AMDGPU_MMHUB0(0)) 83 tmp = RREG32_SOC15_IP(MMHUB, reg); 84 else 85 tmp = RREG32_XCC(reg, j); 86 87 tmp |= hub->vm_cntx_cntl_vm_fault; 88 89 if (j >= AMDGPU_MMHUB0(0)) 90 WREG32_SOC15_IP(MMHUB, reg, tmp); 91 else 92 WREG32_XCC(reg, tmp, j); 93 } 94 } 95 break; 96 default: 97 break; 98 } 99 100 return 0; 101 } 102 103 static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev, 104 struct amdgpu_irq_src *source, 105 struct amdgpu_iv_entry *entry) 106 { 107 struct amdgpu_task_info *task_info; 108 bool retry_fault = false, write_fault = false; 109 unsigned int vmhub, node_id; 110 struct amdgpu_vmhub *hub; 111 uint32_t cam_index = 0; 112 const char *hub_name; 113 int ret, xcc_id = 0; 114 uint32_t status = 0; 115 const char *die_name; 116 char die_name_buf[32]; 117 u64 addr; 118 119 node_id = entry->node_id; 120 121 addr = (u64)entry->src_data[0] << 12; 122 addr |= ((u64)entry->src_data[1] & 0x1fff) << 44; 123 124 if (entry->src_id == UTCL2_1_0__SRCID__RETRY) { 125 retry_fault = true; 126 write_fault = !!(entry->src_data[1] & AMDGPU_GMC121_FAULT_SOURCE_DATA_WRITE); 127 } 128 129 if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) { 130 hub_name = "mmhub0"; 131 vmhub = AMDGPU_MMHUB0(node_id / 4); 132 } else { 133 hub_name = "gfxhub0"; 134 if (adev->gfx.funcs->ih_node_to_logical_xcc) { 135 xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, 136 node_id); 137 if (xcc_id < 0) 138 xcc_id = 0; 139 } 140 vmhub = xcc_id; 141 } 142 143 hub = &adev->vmhub[vmhub]; 144 145 if (retry_fault) { 146 if (adev->irq.retry_cam_enabled) { 147 /* Delegate it to a different ring if the hardware hasn't 148 * already done it. 149 */ 150 if (entry->ih == &adev->irq.ih) { 151 amdgpu_irq_delegate(adev, entry, 8); 152 return 1; 153 } 154 155 cam_index = entry->src_data[3] & 0x3ff; 156 157 ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 158 addr, entry->timestamp, write_fault); 159 WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); 160 if (ret) 161 return 1; 162 } else { 163 /* Process it onyl if it's the first fault for this address */ 164 if (entry->ih != &adev->irq.ih_soft && 165 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 166 entry->timestamp)) 167 return 1; 168 169 /* Delegate it to a different ring if the hardware hasn't 170 * already done it. 171 */ 172 if (entry->ih == &adev->irq.ih) { 173 amdgpu_irq_delegate(adev, entry, 8); 174 return 1; 175 } 176 177 /* Try to handle the recoverable page faults by filling page 178 * tables 179 */ 180 if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 181 addr, entry->timestamp, write_fault)) 182 return 1; 183 } 184 } 185 186 if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault)) 187 return 1; 188 189 if (!printk_ratelimit()) 190 return 0; 191 192 dev_err(adev->dev, 193 "[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name, 194 retry_fault ? "retry" : "no-retry", 195 entry->src_id, entry->ring_id, entry->vmid, entry->pasid); 196 197 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid); 198 if (task_info) { 199 amdgpu_vm_print_task_info(adev, task_info); 200 amdgpu_vm_put_task_info(task_info); 201 } 202 203 dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n", 204 addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]); 205 206 if (adev->irq.ih_funcs && 207 adev->irq.ih_funcs->node_id_to_die_name) { 208 die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id, 209 die_name_buf, 210 sizeof(die_name_buf)); 211 if (die_name) 212 dev_err(adev->dev, 213 " cookie node_id %d fault from die %s\n", 214 node_id, die_name); 215 } 216 217 if (amdgpu_sriov_vf(adev)) 218 return 0; 219 220 /* 221 * Issue a dummy read to wait for the status register to 222 * be updated to avoid reading an incorrect value due to 223 * the new fast GRBM interface. 224 */ 225 if (entry->vmid_src == AMDGPU_GFXHUB(0)) 226 RREG32(hub->vm_l2_pro_fault_status); 227 228 status = RREG32(hub->vm_l2_pro_fault_status); 229 230 /* Only print L2 fault status if the status register could be read and 231 * contains useful information 232 */ 233 if (!status) 234 return 0; 235 236 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 237 238 amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub); 239 240 hub->vmhub_funcs->print_l2_protection_fault_status(adev, status); 241 242 return 0; 243 } 244 245 static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev, 246 uint8_t vmid, uint8_t inst, 247 uint16_t *p_pasid) 248 { 249 uint16_t index; 250 251 if (inst/4) 252 index = 0xA + inst%4; 253 else 254 index = 0x2 + inst%4; 255 256 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_LUT_INDEX), index); 257 258 *p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff; 259 260 return !!(*p_pasid); 261 } 262 263 /* 264 * GART 265 * VMID 0 is the physical GPU addresses as used by the kernel. 266 * VMIDs 1-15 are used for userspace clients and are handled 267 * by the amdgpu vm/hsa code. 268 */ 269 270 /** 271 * gmc_v12_1_use_invalidate_semaphore - judge whether to use semaphore 272 * 273 * @adev: amdgpu_device pointer 274 * @vmhub: vmhub type 275 * 276 */ 277 static bool gmc_v12_1_use_invalidate_semaphore(struct amdgpu_device *adev, 278 uint32_t vmhub) 279 { 280 return ((!AMDGPU_IS_GFXHUB(vmhub)) && 281 (!amdgpu_sriov_vf(adev))); 282 } 283 284 static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid, 285 unsigned int vmhub, uint32_t flush_type) 286 { 287 bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(adev, vmhub); 288 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 289 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); 290 u32 tmp; 291 /* Use register 17 for GART */ 292 const unsigned eng = 17; 293 unsigned int i; 294 unsigned char hub_ip = 0; 295 296 hub_ip = (AMDGPU_IS_GFXHUB(vmhub)) ? 297 GC_HWIP : MMHUB_HWIP; 298 299 spin_lock(&adev->gmc.invalidate_lock); 300 301 if (use_semaphore) { 302 for (i = 0; i < adev->usec_timeout; i++) { 303 /* a read return value of 1 means semaphore acuqire */ 304 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + hub->eng_distance * eng, hub_ip); 305 if (tmp & 0x1) 306 break; 307 udelay(1); 308 } 309 310 if (i >= adev->usec_timeout) 311 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 312 } 313 314 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip); 315 316 /* Wait for ACK with a delay.*/ 317 for (i = 0; i < adev->usec_timeout; i++) { 318 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack + 319 hub->eng_distance * eng, hub_ip); 320 tmp &= 1 << vmid; 321 if (tmp) 322 break; 323 324 udelay(1); 325 } 326 327 if (use_semaphore) 328 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0, hub_ip); 329 330 /* Issue additional private vm invalidation to MMHUB */ 331 if (!AMDGPU_IS_GFXHUB(vmhub) && 332 (hub->vm_l2_bank_select_reserved_cid2) && 333 !amdgpu_sriov_vf(adev)) { 334 inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); 335 /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */ 336 inv_req |= (1 << 25); 337 /* Issue private invalidation */ 338 WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req); 339 /* Read back to ensure invalidation is done*/ 340 RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2); 341 } 342 343 spin_unlock(&adev->gmc.invalidate_lock); 344 345 if (i < adev->usec_timeout) 346 return; 347 348 dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n"); 349 } 350 351 /** 352 * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback 353 * 354 * @adev: amdgpu_device pointer 355 * @vmid: vm instance to flush 356 * @vmhub: which hub to flush 357 * @flush_type: the flush type 358 * 359 * Flush the TLB for the requested page table. 360 */ 361 static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 362 uint32_t vmhub, uint32_t flush_type) 363 { 364 u32 inst; 365 366 if (AMDGPU_IS_GFXHUB(vmhub) && 367 !adev->gfx.is_poweron) 368 return; 369 370 if (vmhub >= AMDGPU_MMHUB0(0)) 371 inst = 0; 372 else 373 inst = vmhub; 374 375 /* This is necessary for SRIOV as well as for GFXOFF to function 376 * properly under bare metal 377 */ 378 if (((adev->gfx.kiq[inst].ring.sched.ready || 379 adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready) && 380 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) { 381 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub]; 382 const unsigned eng = 17; 383 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type); 384 u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 385 u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 386 387 amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req, 388 1 << vmid, inst); 389 return; 390 } 391 392 gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0); 393 return; 394 } 395 396 /** 397 * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid 398 * 399 * @adev: amdgpu_device pointer 400 * @pasid: pasid to be flush 401 * @flush_type: the flush type 402 * @all_hub: flush all hubs 403 * @inst: is used to select which instance of KIQ to use for the invalidation 404 * 405 * Flush the TLB for the requested pasid. 406 */ 407 static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 408 uint16_t pasid, uint32_t flush_type, 409 bool all_hub, uint32_t inst) 410 { 411 uint16_t queried; 412 int vmid, i; 413 414 if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready && 415 (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) { 416 struct mes_inv_tlbs_pasid_input input = {0}; 417 input.xcc_id = inst; 418 input.pasid = pasid; 419 input.flush_type = flush_type; 420 421 /* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */ 422 if (!amdgpu_gfx_is_master_xcc(adev, inst)) 423 return; 424 425 input.hub_id = AMDGPU_GFXHUB(0); 426 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); 427 428 if (all_hub) { 429 /* invalidate mm_hub */ 430 if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) { 431 input.hub_id = AMDGPU_MMHUB0(0); 432 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); 433 } 434 if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) { 435 input.hub_id = AMDGPU_MMHUB1(0); 436 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input); 437 } 438 } 439 return; 440 } 441 442 for (vmid = 1; vmid < 16; vmid++) { 443 bool valid; 444 445 valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid, inst, 446 &queried); 447 if (!valid || queried != pasid) 448 continue; 449 450 if (all_hub) { 451 for_each_set_bit(i, adev->vmhubs_mask, 452 AMDGPU_MAX_VMHUBS) 453 gmc_v12_1_flush_gpu_tlb(adev, vmid, i, 454 flush_type); 455 } else { 456 gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst), 457 flush_type); 458 } 459 } 460 } 461 462 static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 463 unsigned vmid, uint64_t pd_addr) 464 { 465 bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(ring->adev, ring->vm_hub); 466 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; 467 uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0); 468 unsigned eng = ring->vm_inv_eng; 469 470 if (use_semaphore) 471 /* a read return value of 1 means semaphore acuqire */ 472 amdgpu_ring_emit_reg_wait(ring, 473 hub->vm_inv_eng0_sem + 474 hub->eng_distance * eng, 0x1, 0x1); 475 476 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 477 (hub->ctx_addr_distance * vmid), 478 lower_32_bits(pd_addr)); 479 480 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 481 (hub->ctx_addr_distance * vmid), 482 upper_32_bits(pd_addr)); 483 484 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 485 hub->eng_distance * eng, 486 hub->vm_inv_eng0_ack + 487 hub->eng_distance * eng, 488 req, 1 << vmid); 489 490 if (use_semaphore) 491 /* 492 * add semaphore release after invalidation, 493 * write with 0 means semaphore release 494 */ 495 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 496 hub->eng_distance * eng, 0); 497 498 return pd_addr; 499 } 500 501 static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring, 502 unsigned vmid, unsigned pasid) 503 { 504 struct amdgpu_device *adev = ring->adev; 505 uint32_t reg; 506 507 if (ring->vm_hub == AMDGPU_GFXHUB(0)) 508 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid; 509 else 510 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid; 511 512 amdgpu_ring_emit_wreg(ring, reg, pasid); 513 } 514 515 /* 516 * PTE format: 517 * 63 P 518 * 62:59 reserved 519 * 58 D 520 * 57 G 521 * 56 T 522 * 55:54 M 523 * 53:52 SW 524 * 51:48 reserved for future 525 * 47:12 4k physical page base address 526 * 11:7 fragment 527 * 6 write 528 * 5 read 529 * 4 exe 530 * 3 Z 531 * 2 snooped 532 * 1 system 533 * 0 valid 534 * 535 * PDE format: 536 * 63 P 537 * 62:58 block fragment size 538 * 57 reserved 539 * 56 A 540 * 55:54 M 541 * 53:52 reserved 542 * 51:48 reserved for future 543 * 47:6 physical base address of PD or PTE 544 * 5:3 reserved 545 * 2 C 546 * 1 system 547 * 0 valid 548 */ 549 550 static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level, 551 uint64_t *addr, uint64_t *flags) 552 { 553 if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM)) 554 *addr = adev->vm_manager.vram_base_offset + *addr - 555 adev->gmc.vram_start; 556 BUG_ON(*addr & 0xFFFF00000000003FULL); 557 558 *flags |= AMDGPU_PTE_SNOOPED; 559 560 if (!adev->gmc.translate_further) 561 return; 562 563 if (level == AMDGPU_VM_PDB1) { 564 /* Set the block fragment size */ 565 if (!(*flags & AMDGPU_PDE_PTE_GFX12)) 566 *flags |= AMDGPU_PDE_BFS_GFX12(0x9); 567 568 } else if (level == AMDGPU_VM_PDB0) { 569 if (*flags & AMDGPU_PDE_PTE_GFX12) 570 *flags &= ~AMDGPU_PDE_PTE_GFX12; 571 } 572 } 573 574 static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev, 575 struct amdgpu_bo *bo, 576 uint64_t *flags) 577 { 578 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 579 bool is_vram = bo->tbo.resource && 580 bo->tbo.resource->mem_type == TTM_PL_VRAM; 581 bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT | 582 AMDGPU_GEM_CREATE_EXT_COHERENT); 583 bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT; 584 uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0); 585 bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED; 586 unsigned int mtype, mtype_local, mtype_remote; 587 bool snoop = false; 588 bool is_local = false; 589 bool is_aid_a1; 590 591 switch (gc_ip_version) { 592 case IP_VERSION(12, 1, 0): 593 is_aid_a1 = (adev->rev_id & 0x10); 594 595 mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC; 596 mtype_remote = is_aid_a1 ? MTYPE_NC : MTYPE_UC; 597 if (amdgpu_mtype_local == 0) { 598 DRM_INFO_ONCE("Using MTYPE_RW for local memory\n"); 599 mtype_local = MTYPE_RW; 600 } else if (amdgpu_mtype_local == 1) { 601 DRM_INFO_ONCE("Using MTYPE_NC for local memory\n"); 602 mtype_local = MTYPE_NC; 603 } else if (amdgpu_mtype_local == 2) { 604 DRM_INFO_ONCE("MTYPE_CC not supported, using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC"); 605 } else { 606 DRM_INFO_ONCE("Using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC"); 607 } 608 609 is_local = (is_vram && adev == bo_adev); 610 snoop = true; 611 if (uncached) { 612 mtype = MTYPE_UC; 613 } else if (ext_coherent) { 614 mtype = is_local ? mtype_local : MTYPE_UC; 615 } else { 616 mtype = is_local ? mtype_local : mtype_remote; 617 } 618 break; 619 default: 620 if (uncached || coherent) 621 mtype = MTYPE_UC; 622 else 623 mtype = MTYPE_NC; 624 } 625 626 if (mtype != MTYPE_NC) 627 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype); 628 629 if (is_local || adev->have_atomics_support) 630 *flags |= AMDGPU_PTE_BUS_ATOMICS; 631 632 *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; 633 } 634 635 static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev, 636 struct amdgpu_vm *vm, 637 struct amdgpu_bo *bo, 638 uint32_t vm_flags, 639 uint64_t *flags) 640 { 641 if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE) 642 *flags |= AMDGPU_PTE_EXECUTABLE; 643 else 644 *flags &= ~AMDGPU_PTE_EXECUTABLE; 645 646 switch (vm_flags & AMDGPU_VM_MTYPE_MASK) { 647 case AMDGPU_VM_MTYPE_DEFAULT: 648 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); 649 break; 650 case AMDGPU_VM_MTYPE_NC: 651 default: 652 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC); 653 break; 654 case AMDGPU_VM_MTYPE_RW: 655 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_RW); 656 break; 657 case AMDGPU_VM_MTYPE_UC: 658 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC); 659 break; 660 } 661 662 if ((*flags & AMDGPU_PTE_VALID) && bo) 663 gmc_v12_1_get_coherence_flags(adev, bo, flags); 664 } 665 666 static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = { 667 .flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb, 668 .flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid, 669 .emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb, 670 .emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping, 671 .get_vm_pde = gmc_v12_1_get_vm_pde, 672 .get_vm_pte = gmc_v12_1_get_vm_pte, 673 .query_mem_partition_mode = &amdgpu_gmc_query_memory_partition, 674 .request_mem_partition_mode = &amdgpu_gmc_request_memory_partition, 675 }; 676 677 void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev) 678 { 679 adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs; 680 } 681 682 static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = { 683 .set = gmc_v12_1_vm_fault_interrupt_state, 684 .process = gmc_v12_1_process_interrupt, 685 }; 686 687 static const struct amdgpu_irq_src_funcs gmc_v12_1_ecc_funcs = { 688 .process = amdgpu_umc_uniras_process_ecc_irq, 689 }; 690 691 void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev) 692 { 693 adev->gmc.vm_fault.num_types = 1; 694 adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs; 695 696 adev->gmc.ecc_irq.num_types = 1; 697 adev->gmc.ecc_irq.funcs = &gmc_v12_1_ecc_funcs; 698 } 699 700 void gmc_v12_1_init_vram_info(struct amdgpu_device *adev) 701 { 702 /* TODO: query vram_info from ip discovery binary */ 703 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4; 704 adev->gmc.vram_width = 384 * 64; 705 } 706