xref: /linux/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c (revision e2a6a4e6d408c0466136cecc2439278a6aaf604b)
1 /*
2  * Copyright 2025 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include "amdgpu.h"
24 #include "gmc_v12_1.h"
25 #include "soc15_common.h"
26 #include "soc_v1_0_enum.h"
27 #include "oss/osssys_7_1_0_offset.h"
28 #include "oss/osssys_7_1_0_sh_mask.h"
29 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
30 
31 static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev,
32 					      struct amdgpu_irq_src *src,
33 					      unsigned int type,
34 					      enum amdgpu_interrupt_state state)
35 {
36 	struct amdgpu_vmhub *hub;
37 	u32 tmp, reg, i, j;
38 
39 	switch (state) {
40 	case AMDGPU_IRQ_STATE_DISABLE:
41 		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
42 			hub = &adev->vmhub[j];
43 			for (i = 0; i < 16; i++) {
44 				reg = hub->vm_context0_cntl + i;
45 
46 				/* This works because this interrupt is only
47 				 * enabled at init/resume and disabled in
48 				 * fini/suspend, so the overall state doesn't
49 				 * change over the course of suspend/resume.
50 				 */
51 				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
52 					continue;
53 
54 				if (j >= AMDGPU_MMHUB0(0))
55 					tmp = RREG32_SOC15_IP(MMHUB, reg);
56 				else
57 					tmp = RREG32_XCC(reg, j);
58 
59 				tmp &= ~hub->vm_cntx_cntl_vm_fault;
60 
61 				if (j >= AMDGPU_MMHUB0(0))
62 					WREG32_SOC15_IP(MMHUB, reg, tmp);
63 				else
64 					WREG32_XCC(reg, tmp, j);
65 			}
66 		}
67 		break;
68 	case AMDGPU_IRQ_STATE_ENABLE:
69 		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
70 			hub = &adev->vmhub[j];
71 			for (i = 0; i < 16; i++) {
72 				reg = hub->vm_context0_cntl + i;
73 
74 				/* This works because this interrupt is only
75 				 * enabled at init/resume and disabled in
76 				 * fini/suspend, so the overall state doesn't
77 				 * change over the course of suspend/resume.
78 				 */
79 				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
80 					continue;
81 
82 				if (j >= AMDGPU_MMHUB0(0))
83 					tmp = RREG32_SOC15_IP(MMHUB, reg);
84 				else
85 					tmp = RREG32_XCC(reg, j);
86 
87 				tmp |= hub->vm_cntx_cntl_vm_fault;
88 
89 				if (j >= AMDGPU_MMHUB0(0))
90 					WREG32_SOC15_IP(MMHUB, reg, tmp);
91 				else
92 					WREG32_XCC(reg, tmp, j);
93 			}
94 		}
95 		break;
96 	default:
97 		break;
98 	}
99 
100 	return 0;
101 }
102 
103 static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
104 				       struct amdgpu_irq_src *source,
105 				       struct amdgpu_iv_entry *entry)
106 {
107 	struct amdgpu_task_info *task_info;
108 	bool retry_fault = false, write_fault = false;
109 	unsigned int vmhub, node_id;
110 	struct amdgpu_vmhub *hub;
111 	uint32_t cam_index = 0;
112 	const char *hub_name;
113 	int ret, xcc_id = 0;
114 	uint32_t status = 0;
115 	u64 addr;
116 
117 	node_id = entry->node_id;
118 
119 	addr = (u64)entry->src_data[0] << 12;
120 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
121 
122 	if (entry->src_id == UTCL2_1_0__SRCID__RETRY) {
123 		retry_fault = true;
124 		write_fault = !!(entry->src_data[1] & 0x200000);
125 	}
126 
127 	if (entry->client_id == SOC21_IH_CLIENTID_VMC) {
128 		hub_name = "mmhub0";
129 		vmhub = AMDGPU_MMHUB0(node_id / 4);
130 	} else {
131 		hub_name = "gfxhub0";
132 		if (adev->gfx.funcs->ih_node_to_logical_xcc) {
133 			xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
134 								node_id);
135 			if (xcc_id < 0)
136 				xcc_id = 0;
137 		}
138 		vmhub = xcc_id;
139 	}
140 
141 	hub = &adev->vmhub[vmhub];
142 
143 	if (retry_fault) {
144 		if (adev->irq.retry_cam_enabled) {
145 			/* Delegate it to a different ring if the hardware hasn't
146 			 * already done it.
147 			 */
148 			if (entry->ih == &adev->irq.ih) {
149 				amdgpu_irq_delegate(adev, entry, 8);
150 				return 1;
151 			}
152 
153 			cam_index = entry->src_data[3] & 0x3ff;
154 
155 			ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
156 							addr, entry->timestamp, write_fault);
157 			WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
158 			if (ret)
159 				return 1;
160 		} else {
161 			/* Process it onyl if it's the first fault for this address */
162 			if (entry->ih != &adev->irq.ih_soft &&
163 				amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
164 							 entry->timestamp))
165 				return 1;
166 
167 			/* Delegate it to a different ring if the hardware hasn't
168 			 * already done it.
169 			 */
170 			if (entry->ih == &adev->irq.ih) {
171 				amdgpu_irq_delegate(adev, entry, 8);
172 				return 1;
173 			}
174 
175 			/* Try to handle the recoverable page faults by filling page
176 			 * tables
177 			 */
178 			if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
179 						   addr, entry->timestamp, write_fault))
180 				return 1;
181 		}
182 	}
183 
184 	if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault))
185 		return 1;
186 
187 	if (!printk_ratelimit())
188 		return 0;
189 
190 	dev_err(adev->dev,
191 		"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name,
192 		retry_fault ? "retry" : "no-retry",
193 		entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
194 
195 	task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
196 	if (task_info) {
197 		amdgpu_vm_print_task_info(adev, task_info);
198 		amdgpu_vm_put_task_info(task_info);
199 	}
200 
201 	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client %d\n",
202 		addr, entry->client_id);
203 
204 	if (amdgpu_sriov_vf(adev))
205 		return 0;
206 
207 	/*
208 	 * Issue a dummy read to wait for the status register to
209 	 * be updated to avoid reading an incorrect value due to
210 	 * the new fast GRBM interface.
211 	 */
212 	if (entry->vmid_src == AMDGPU_GFXHUB(0))
213 		RREG32(hub->vm_l2_pro_fault_status);
214 
215 	status = RREG32(hub->vm_l2_pro_fault_status);
216 
217 	/* Only print L2 fault status if the status register could be read and
218 	 * contains useful information
219 	 */
220 	if (!status)
221 		return 0;
222 
223 	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
224 
225 	amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);
226 
227 	hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
228 
229 	return 0;
230 }
231 
232 static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev,
233 						  uint8_t vmid, uint16_t *p_pasid)
234 {
235 	*p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff;
236 
237 	return !!(*p_pasid);
238 }
239 
240 /*
241  * GART
242  * VMID 0 is the physical GPU addresses as used by the kernel.
243  * VMIDs 1-15 are used for userspace clients and are handled
244  * by the amdgpu vm/hsa code.
245  */
246 
247 static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
248 				   unsigned int vmhub, uint32_t flush_type)
249 {
250 	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
251 	u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
252 	u32 tmp;
253 	/* Use register 17 for GART */
254 	const unsigned eng = 17;
255 	unsigned int i;
256 	unsigned char hub_ip = 0;
257 
258 	hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ?
259 		   GC_HWIP : MMHUB_HWIP;
260 
261 	spin_lock(&adev->gmc.invalidate_lock);
262 
263 	WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip);
264 
265 	/* Wait for ACK with a delay.*/
266 	for (i = 0; i < adev->usec_timeout; i++) {
267 		tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +
268 				    hub->eng_distance * eng, hub_ip);
269 		tmp &= 1 << vmid;
270 		if (tmp)
271 			break;
272 
273 		udelay(1);
274 	}
275 
276 	/* Issue additional private vm invalidation to MMHUB */
277 	if ((vmhub != AMDGPU_GFXHUB(0)) &&
278 	    (hub->vm_l2_bank_select_reserved_cid2) &&
279 		!amdgpu_sriov_vf(adev)) {
280 		inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
281 		/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
282 		inv_req |= (1 << 25);
283 		/* Issue private invalidation */
284 		WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
285 		/* Read back to ensure invalidation is done*/
286 		RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
287 	}
288 
289 	spin_unlock(&adev->gmc.invalidate_lock);
290 
291 	if (i < adev->usec_timeout)
292 		return;
293 
294 	dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
295 }
296 
297 /**
298  * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback
299  *
300  * @adev: amdgpu_device pointer
301  * @vmid: vm instance to flush
302  * @vmhub: which hub to flush
303  * @flush_type: the flush type
304  *
305  * Flush the TLB for the requested page table.
306  */
307 static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
308 				    uint32_t vmhub, uint32_t flush_type)
309 {
310 	if ((vmhub == AMDGPU_GFXHUB(0)) && !adev->gfx.is_poweron)
311 		return;
312 
313 	/* This is necessary for SRIOV as well as for GFXOFF to function
314 	 * properly under bare metal
315 	 */
316 	if (((adev->gfx.kiq[0].ring.sched.ready || adev->mes.ring[0].sched.ready) &&
317 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) {
318 		struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
319 		const unsigned eng = 17;
320 		u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
321 		u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
322 		u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
323 
324 		amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
325 				1 << vmid, GET_INST(GC, 0));
326 		return;
327 	}
328 
329 	mutex_lock(&adev->mman.gtt_window_lock);
330 	gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0);
331 	mutex_unlock(&adev->mman.gtt_window_lock);
332 	return;
333 }
334 
335 /**
336  * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid
337  *
338  * @adev: amdgpu_device pointer
339  * @pasid: pasid to be flush
340  * @flush_type: the flush type
341  * @all_hub: flush all hubs
342  * @inst: is used to select which instance of KIQ to use for the invalidation
343  *
344  * Flush the TLB for the requested pasid.
345  */
346 static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
347 					  uint16_t pasid, uint32_t flush_type,
348 					  bool all_hub, uint32_t inst)
349 {
350 	uint16_t queried;
351 	int vmid, i;
352 
353 	for (vmid = 1; vmid < 16; vmid++) {
354 		bool valid;
355 
356 		valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid,
357 							      &queried);
358 		if (!valid || queried != pasid)
359 			continue;
360 
361 		if (all_hub) {
362 			for_each_set_bit(i, adev->vmhubs_mask,
363 					 AMDGPU_MAX_VMHUBS)
364 				gmc_v12_1_flush_gpu_tlb(adev, vmid, i,
365 							flush_type);
366 		} else {
367 			gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0),
368 						flush_type);
369 		}
370 	}
371 }
372 
373 static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
374 					     unsigned vmid, uint64_t pd_addr)
375 {
376 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
377 	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
378 	unsigned eng = ring->vm_inv_eng;
379 
380 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
381 			      (hub->ctx_addr_distance * vmid),
382 			      lower_32_bits(pd_addr));
383 
384 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
385 			      (hub->ctx_addr_distance * vmid),
386 			      upper_32_bits(pd_addr));
387 
388 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
389 					    hub->eng_distance * eng,
390 					    hub->vm_inv_eng0_ack +
391 					    hub->eng_distance * eng,
392 					    req, 1 << vmid);
393 
394 	return pd_addr;
395 }
396 
397 static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring,
398 					 unsigned vmid, unsigned pasid)
399 {
400 	struct amdgpu_device *adev = ring->adev;
401 	uint32_t reg;
402 
403 	if (ring->vm_hub == AMDGPU_GFXHUB(0))
404 		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid;
405 	else
406 		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid;
407 
408 	amdgpu_ring_emit_wreg(ring, reg, pasid);
409 }
410 
411 /*
412  * PTE format:
413  * 63 P
414  * 62:59 reserved
415  * 58 D
416  * 57 G
417  * 56 T
418  * 55:54 M
419  * 53:52 SW
420  * 51:48 reserved for future
421  * 47:12 4k physical page base address
422  * 11:7 fragment
423  * 6 write
424  * 5 read
425  * 4 exe
426  * 3 Z
427  * 2 snooped
428  * 1 system
429  * 0 valid
430  *
431  * PDE format:
432  * 63 P
433  * 62:58 block fragment size
434  * 57 reserved
435  * 56 A
436  * 55:54 M
437  * 53:52 reserved
438  * 51:48 reserved for future
439  * 47:6 physical base address of PD or PTE
440  * 5:3 reserved
441  * 2 C
442  * 1 system
443  * 0 valid
444  */
445 
446 static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level,
447 				 uint64_t *addr, uint64_t *flags)
448 {
449 	if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM))
450 		*addr = adev->vm_manager.vram_base_offset + *addr -
451 			adev->gmc.vram_start;
452 	BUG_ON(*addr & 0xFFFF00000000003FULL);
453 
454 	*flags |= AMDGPU_PTE_SNOOPED;
455 
456 	if (!adev->gmc.translate_further)
457 		return;
458 
459 	if (level == AMDGPU_VM_PDB1) {
460 		/* Set the block fragment size */
461 		if (!(*flags & AMDGPU_PDE_PTE_GFX12))
462 			*flags |= AMDGPU_PDE_BFS_GFX12(0x9);
463 
464 	} else if (level == AMDGPU_VM_PDB0) {
465 		if (*flags & AMDGPU_PDE_PTE_GFX12)
466 			*flags &= ~AMDGPU_PDE_PTE_GFX12;
467 	}
468 }
469 
470 #if 0
471 static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
472 					  struct amdgpu_bo *bo,
473 					  uint64_t *flags)
474 {
475 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
476 	bool is_vram = bo->tbo.resource &&
477 		       bo->tbo.resource->mem_type == TTM_PL_VRAM;
478 	bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
479 				     AMDGPU_GEM_CREATE_EXT_COHERENT);
480 	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
481 	uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
482 	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
483 	unsigned int mtype, mtype_local;
484 	bool snoop = false;
485 	bool is_local;
486 
487 	switch (gc_ip_version) {
488 	case IP_VERSION(12, 1, 0):
489 		mtype_local = MTYPE_RW;
490 		if (amdgpu_mtype_local == 1) {
491 			DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
492 			mtype_local = MTYPE_NC;
493 		} else if (amdgpu_mtype_local == 2) {
494 			DRM_INFO_ONCE("MTYPE_CC not supported, using MTYPE_RW instead for local memory\n");
495 		} else {
496 			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
497 		}
498 
499 		is_local = (is_vram && adev == bo_adev);
500 		snoop = true;
501 		if (uncached) {
502 			mtype = MTYPE_UC;
503 		} else if (ext_coherent) {
504 			mtype = is_local ? mtype_local : MTYPE_UC;
505 		} else {
506 			if (is_local)
507 				mtype = mtype_local;
508 			else
509 				mtype = MTYPE_NC;
510 		}
511 		break;
512 	default:
513 		if (uncached || coherent)
514 			mtype = MTYPE_UC;
515 		else
516 			mtype = MTYPE_NC;
517 	}
518 
519 	if (mtype != MTYPE_NC)
520 		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype);
521 
522 	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
523 }
524 #endif
525 
526 static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev,
527 				 struct amdgpu_vm *vm,
528 				 struct amdgpu_bo *bo,
529 				 uint32_t vm_flags,
530 				 uint64_t *flags)
531 {
532 	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
533 		*flags |= AMDGPU_PTE_EXECUTABLE;
534 	else
535 		*flags &= ~AMDGPU_PTE_EXECUTABLE;
536 
537 	switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
538 	case AMDGPU_VM_MTYPE_DEFAULT:
539 		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
540 		break;
541 	case AMDGPU_VM_MTYPE_NC:
542 	default:
543 		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
544 		break;
545 	case AMDGPU_VM_MTYPE_UC:
546 		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
547 		break;
548 	}
549 
550 	if (vm_flags & AMDGPU_VM_PAGE_NOALLOC)
551 		*flags |= AMDGPU_PTE_NOALLOC;
552 	else
553 		*flags &= ~AMDGPU_PTE_NOALLOC;
554 
555 	if (vm_flags & AMDGPU_VM_PAGE_PRT) {
556 		*flags |= AMDGPU_PTE_SNOOPED;
557 		*flags |= AMDGPU_PTE_SYSTEM;
558 		*flags |= AMDGPU_PTE_IS_PTE;
559 		*flags &= ~AMDGPU_PTE_VALID;
560 	}
561 
562 	if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
563 			       AMDGPU_GEM_CREATE_EXT_COHERENT |
564 			       AMDGPU_GEM_CREATE_UNCACHED))
565 		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
566 
567 	if (adev->have_atomics_support)
568 		*flags |= AMDGPU_PTE_BUS_ATOMICS;
569 
570 	if (bo && bo->flags & AMDGPU_GEM_CREATE_UNCACHED)
571 		*flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
572 }
573 
574 static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = {
575 	.flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb,
576 	.flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid,
577 	.emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb,
578 	.emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping,
579 	.get_vm_pde = gmc_v12_1_get_vm_pde,
580 	.get_vm_pte = gmc_v12_1_get_vm_pte,
581 	.query_mem_partition_mode = &amdgpu_gmc_query_memory_partition,
582 	.request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
583 };
584 
585 void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev)
586 {
587 	adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs;
588 }
589 
590 static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = {
591 	.set = gmc_v12_1_vm_fault_interrupt_state,
592 	.process = gmc_v12_1_process_interrupt,
593 };
594 
595 void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev)
596 {
597 	adev->gmc.vm_fault.num_types = 1;
598 	adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs;
599 }
600 
601 void gmc_v12_1_init_vram_info(struct amdgpu_device *adev)
602 {
603 	/* TODO: query vram_info from ip discovery binary */
604 	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4;
605 	adev->gmc.vram_width = 384 * 64;
606 }
607