1 /*
2 * Copyright 2025 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23 #include "amdgpu.h"
24 #include "gmc_v12_1.h"
25 #include "soc15_common.h"
26 #include "soc_v1_0_enum.h"
27 #include "oss/osssys_7_1_0_offset.h"
28 #include "oss/osssys_7_1_0_sh_mask.h"
29 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
30
gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned int type,enum amdgpu_interrupt_state state)31 static int gmc_v12_1_vm_fault_interrupt_state(struct amdgpu_device *adev,
32 struct amdgpu_irq_src *src,
33 unsigned int type,
34 enum amdgpu_interrupt_state state)
35 {
36 struct amdgpu_vmhub *hub;
37 u32 tmp, reg, i, j;
38
39 switch (state) {
40 case AMDGPU_IRQ_STATE_DISABLE:
41 for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
42 hub = &adev->vmhub[j];
43 for (i = 0; i < 16; i++) {
44 reg = hub->vm_context0_cntl + i;
45
46 /* This works because this interrupt is only
47 * enabled at init/resume and disabled in
48 * fini/suspend, so the overall state doesn't
49 * change over the course of suspend/resume.
50 */
51 if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
52 continue;
53
54 if (j >= AMDGPU_MMHUB0(0))
55 tmp = RREG32_SOC15_IP(MMHUB, reg);
56 else
57 tmp = RREG32_XCC(reg, j);
58
59 tmp &= ~hub->vm_cntx_cntl_vm_fault;
60
61 if (j >= AMDGPU_MMHUB0(0))
62 WREG32_SOC15_IP(MMHUB, reg, tmp);
63 else
64 WREG32_XCC(reg, tmp, j);
65 }
66 }
67 break;
68 case AMDGPU_IRQ_STATE_ENABLE:
69 for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
70 hub = &adev->vmhub[j];
71 for (i = 0; i < 16; i++) {
72 reg = hub->vm_context0_cntl + i;
73
74 /* This works because this interrupt is only
75 * enabled at init/resume and disabled in
76 * fini/suspend, so the overall state doesn't
77 * change over the course of suspend/resume.
78 */
79 if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
80 continue;
81
82 if (j >= AMDGPU_MMHUB0(0))
83 tmp = RREG32_SOC15_IP(MMHUB, reg);
84 else
85 tmp = RREG32_XCC(reg, j);
86
87 tmp |= hub->vm_cntx_cntl_vm_fault;
88
89 if (j >= AMDGPU_MMHUB0(0))
90 WREG32_SOC15_IP(MMHUB, reg, tmp);
91 else
92 WREG32_XCC(reg, tmp, j);
93 }
94 }
95 break;
96 default:
97 break;
98 }
99
100 return 0;
101 }
102
gmc_v12_1_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)103 static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
104 struct amdgpu_irq_src *source,
105 struct amdgpu_iv_entry *entry)
106 {
107 struct amdgpu_task_info *task_info;
108 bool retry_fault = false, write_fault = false;
109 unsigned int vmhub, node_id;
110 struct amdgpu_vmhub *hub;
111 uint32_t cam_index = 0;
112 const char *hub_name;
113 int ret, xcc_id = 0;
114 uint32_t status = 0;
115 const char *die_name;
116 char die_name_buf[32];
117 u64 addr;
118
119 node_id = entry->node_id;
120
121 addr = (u64)entry->src_data[0] << 12;
122 addr |= ((u64)entry->src_data[1] & 0x1fff) << 44;
123
124 if (entry->src_id == UTCL2_1_0__SRCID__RETRY) {
125 retry_fault = true;
126 write_fault = !!(entry->src_data[1] & AMDGPU_GMC121_FAULT_SOURCE_DATA_WRITE);
127 }
128
129 if (entry->client_id == SOC_V1_0_IH_CLIENTID_VMC) {
130 hub_name = "mmhub0";
131 vmhub = AMDGPU_MMHUB0(node_id / 4);
132 } else {
133 hub_name = "gfxhub0";
134 if (adev->gfx.funcs->ih_node_to_logical_xcc) {
135 xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
136 node_id);
137 if (xcc_id < 0)
138 xcc_id = 0;
139 }
140 vmhub = xcc_id;
141 }
142
143 hub = &adev->vmhub[vmhub];
144
145 if (retry_fault) {
146 if (adev->irq.retry_cam_enabled) {
147 /* Delegate it to a different ring if the hardware hasn't
148 * already done it.
149 */
150 if (entry->ih == &adev->irq.ih) {
151 amdgpu_irq_delegate(adev, entry, 8);
152 return 1;
153 }
154
155 cam_index = entry->src_data[3] & 0x3ff;
156
157 ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
158 addr, entry->timestamp, write_fault);
159 WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
160 if (ret)
161 return 1;
162 } else {
163 /* Process it onyl if it's the first fault for this address */
164 if (entry->ih != &adev->irq.ih_soft &&
165 amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
166 entry->timestamp))
167 return 1;
168
169 /* Delegate it to a different ring if the hardware hasn't
170 * already done it.
171 */
172 if (entry->ih == &adev->irq.ih) {
173 amdgpu_irq_delegate(adev, entry, 8);
174 return 1;
175 }
176
177 /* Try to handle the recoverable page faults by filling page
178 * tables
179 */
180 if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
181 addr, entry->timestamp, write_fault))
182 return 1;
183 }
184 }
185
186 if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault))
187 return 1;
188
189 if (!printk_ratelimit())
190 return 0;
191
192 dev_err(adev->dev,
193 "[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name,
194 retry_fault ? "retry" : "no-retry",
195 entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
196
197 task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
198 if (task_info) {
199 amdgpu_vm_print_task_info(adev, task_info);
200 amdgpu_vm_put_task_info(task_info);
201 }
202
203 dev_err(adev->dev, " in page starting at address 0x%016llx from IH client %d (%s)\n",
204 addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]);
205
206 if (adev->irq.ih_funcs &&
207 adev->irq.ih_funcs->node_id_to_die_name) {
208 die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id,
209 die_name_buf,
210 sizeof(die_name_buf));
211 if (die_name)
212 dev_err(adev->dev,
213 " cookie node_id %d fault from die %s\n",
214 node_id, die_name);
215 }
216
217 if (amdgpu_sriov_vf(adev))
218 return 0;
219
220 /*
221 * Issue a dummy read to wait for the status register to
222 * be updated to avoid reading an incorrect value due to
223 * the new fast GRBM interface.
224 */
225 if (entry->vmid_src == AMDGPU_GFXHUB(0))
226 RREG32(hub->vm_l2_pro_fault_status);
227
228 status = RREG32(hub->vm_l2_pro_fault_status);
229
230 /* Only print L2 fault status if the status register could be read and
231 * contains useful information
232 */
233 if (!status)
234 return 0;
235
236 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
237
238 amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);
239
240 hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
241
242 return 0;
243 }
244
gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device * adev,uint8_t vmid,uint8_t inst,uint16_t * p_pasid)245 static bool gmc_v12_1_get_vmid_pasid_mapping_info(struct amdgpu_device *adev,
246 uint8_t vmid, uint8_t inst,
247 uint16_t *p_pasid)
248 {
249 uint16_t index;
250
251 if (inst/4)
252 index = 0xA + inst%4;
253 else
254 index = 0x2 + inst%4;
255
256 WREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_LUT_INDEX), index);
257
258 *p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff;
259
260 return !!(*p_pasid);
261 }
262
263 /*
264 * GART
265 * VMID 0 is the physical GPU addresses as used by the kernel.
266 * VMIDs 1-15 are used for userspace clients and are handled
267 * by the amdgpu vm/hsa code.
268 */
269
270 /**
271 * gmc_v12_1_use_invalidate_semaphore - judge whether to use semaphore
272 *
273 * @adev: amdgpu_device pointer
274 * @vmhub: vmhub type
275 *
276 */
gmc_v12_1_use_invalidate_semaphore(struct amdgpu_device * adev,uint32_t vmhub)277 static bool gmc_v12_1_use_invalidate_semaphore(struct amdgpu_device *adev,
278 uint32_t vmhub)
279 {
280 return ((!AMDGPU_IS_GFXHUB(vmhub)) &&
281 (!amdgpu_sriov_vf(adev)));
282 }
283
gmc_v12_1_flush_vm_hub(struct amdgpu_device * adev,uint32_t vmid,unsigned int vmhub,uint32_t flush_type)284 static void gmc_v12_1_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
285 unsigned int vmhub, uint32_t flush_type)
286 {
287 bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(adev, vmhub);
288 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
289 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
290 u32 tmp;
291 /* Use register 17 for GART */
292 const unsigned eng = 17;
293 unsigned int i;
294 unsigned char hub_ip = 0;
295
296 hub_ip = (AMDGPU_IS_GFXHUB(vmhub)) ?
297 GC_HWIP : MMHUB_HWIP;
298
299 spin_lock(&adev->gmc.invalidate_lock);
300
301 if (use_semaphore) {
302 for (i = 0; i < adev->usec_timeout; i++) {
303 /* a read return value of 1 means semaphore acuqire */
304 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + hub->eng_distance * eng, hub_ip);
305 if (tmp & 0x1)
306 break;
307 udelay(1);
308 }
309
310 if (i >= adev->usec_timeout)
311 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
312 }
313
314 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req, hub_ip);
315
316 /* Wait for ACK with a delay.*/
317 for (i = 0; i < adev->usec_timeout; i++) {
318 tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +
319 hub->eng_distance * eng, hub_ip);
320 tmp &= 1 << vmid;
321 if (tmp)
322 break;
323
324 udelay(1);
325 }
326
327 if (use_semaphore)
328 WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0, hub_ip);
329
330 /* Issue additional private vm invalidation to MMHUB */
331 if (!AMDGPU_IS_GFXHUB(vmhub) &&
332 (hub->vm_l2_bank_select_reserved_cid2) &&
333 !amdgpu_sriov_vf(adev)) {
334 inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
335 /* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
336 inv_req |= (1 << 25);
337 /* Issue private invalidation */
338 WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
339 /* Read back to ensure invalidation is done*/
340 RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
341 }
342
343 spin_unlock(&adev->gmc.invalidate_lock);
344
345 if (i < adev->usec_timeout)
346 return;
347
348 dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
349 }
350
351 /**
352 * gmc_v12_1_flush_gpu_tlb - gart tlb flush callback
353 *
354 * @adev: amdgpu_device pointer
355 * @vmid: vm instance to flush
356 * @vmhub: which hub to flush
357 * @flush_type: the flush type
358 *
359 * Flush the TLB for the requested page table.
360 */
gmc_v12_1_flush_gpu_tlb(struct amdgpu_device * adev,uint32_t vmid,uint32_t vmhub,uint32_t flush_type)361 static void gmc_v12_1_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
362 uint32_t vmhub, uint32_t flush_type)
363 {
364 u32 inst;
365
366 if (AMDGPU_IS_GFXHUB(vmhub) &&
367 !adev->gfx.is_poweron)
368 return;
369
370 if (vmhub >= AMDGPU_MMHUB0(0))
371 inst = 0;
372 else
373 inst = vmhub;
374
375 /* This is necessary for SRIOV as well as for GFXOFF to function
376 * properly under bare metal
377 */
378 if (((adev->gfx.kiq[inst].ring.sched.ready ||
379 adev->mes.ring[MES_PIPE_INST(inst, 0)].sched.ready) &&
380 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)))) {
381 struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
382 const unsigned eng = 17;
383 u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
384 u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
385 u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
386
387 amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
388 1 << vmid, inst);
389 return;
390 }
391
392 gmc_v12_1_flush_vm_hub(adev, vmid, vmhub, 0);
393 return;
394 }
395
396 /**
397 * gmc_v12_1_flush_gpu_tlb_pasid - tlb flush via pasid
398 *
399 * @adev: amdgpu_device pointer
400 * @pasid: pasid to be flush
401 * @flush_type: the flush type
402 * @all_hub: flush all hubs
403 * @inst: is used to select which instance of KIQ to use for the invalidation
404 *
405 * Flush the TLB for the requested pasid.
406 */
gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device * adev,uint16_t pasid,uint32_t flush_type,bool all_hub,uint32_t inst)407 static void gmc_v12_1_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
408 uint16_t pasid, uint32_t flush_type,
409 bool all_hub, uint32_t inst)
410 {
411 uint16_t queried;
412 int vmid, i;
413
414 if (adev->enable_uni_mes && adev->mes.ring[0].sched.ready &&
415 (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x6f) {
416 struct mes_inv_tlbs_pasid_input input = {0};
417 input.xcc_id = inst;
418 input.pasid = pasid;
419 input.flush_type = flush_type;
420
421 /* MES will invalidate hubs for the device(including slave xcc) from master, ignore request from slave */
422 if (!amdgpu_gfx_is_master_xcc(adev, inst))
423 return;
424
425 input.hub_id = AMDGPU_GFXHUB(0);
426 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
427
428 if (all_hub) {
429 /* invalidate mm_hub */
430 if (test_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask)) {
431 input.hub_id = AMDGPU_MMHUB0(0);
432 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
433 }
434 if (test_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask)) {
435 input.hub_id = AMDGPU_MMHUB1(0);
436 adev->mes.funcs->invalidate_tlbs_pasid(&adev->mes, &input);
437 }
438 }
439 return;
440 }
441
442 for (vmid = 1; vmid < 16; vmid++) {
443 bool valid;
444
445 valid = gmc_v12_1_get_vmid_pasid_mapping_info(adev, vmid, inst,
446 &queried);
447 if (!valid || queried != pasid)
448 continue;
449
450 if (all_hub) {
451 for_each_set_bit(i, adev->vmhubs_mask,
452 AMDGPU_MAX_VMHUBS)
453 gmc_v12_1_flush_gpu_tlb(adev, vmid, i,
454 flush_type);
455 } else {
456 gmc_v12_1_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(inst),
457 flush_type);
458 }
459 }
460 }
461
gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)462 static uint64_t gmc_v12_1_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
463 unsigned vmid, uint64_t pd_addr)
464 {
465 bool use_semaphore = gmc_v12_1_use_invalidate_semaphore(ring->adev, ring->vm_hub);
466 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
467 uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
468 unsigned eng = ring->vm_inv_eng;
469
470 if (use_semaphore)
471 /* a read return value of 1 means semaphore acuqire */
472 amdgpu_ring_emit_reg_wait(ring,
473 hub->vm_inv_eng0_sem +
474 hub->eng_distance * eng, 0x1, 0x1);
475
476 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
477 (hub->ctx_addr_distance * vmid),
478 lower_32_bits(pd_addr));
479
480 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
481 (hub->ctx_addr_distance * vmid),
482 upper_32_bits(pd_addr));
483
484 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
485 hub->eng_distance * eng,
486 hub->vm_inv_eng0_ack +
487 hub->eng_distance * eng,
488 req, 1 << vmid);
489
490 if (use_semaphore)
491 /*
492 * add semaphore release after invalidation,
493 * write with 0 means semaphore release
494 */
495 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
496 hub->eng_distance * eng, 0);
497
498 return pd_addr;
499 }
500
gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring * ring,unsigned vmid,unsigned pasid)501 static void gmc_v12_1_emit_pasid_mapping(struct amdgpu_ring *ring,
502 unsigned vmid, unsigned pasid)
503 {
504 struct amdgpu_device *adev = ring->adev;
505 uint32_t reg;
506
507 if (ring->vm_hub == AMDGPU_GFXHUB(0))
508 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid;
509 else
510 reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid;
511
512 amdgpu_ring_emit_wreg(ring, reg, pasid);
513 }
514
515 /*
516 * PTE format:
517 * 63 P
518 * 62:59 reserved
519 * 58 D
520 * 57 G
521 * 56 T
522 * 55:54 M
523 * 53:52 SW
524 * 51:48 reserved for future
525 * 47:12 4k physical page base address
526 * 11:7 fragment
527 * 6 write
528 * 5 read
529 * 4 exe
530 * 3 Z
531 * 2 snooped
532 * 1 system
533 * 0 valid
534 *
535 * PDE format:
536 * 63 P
537 * 62:58 block fragment size
538 * 57 reserved
539 * 56 A
540 * 55:54 M
541 * 53:52 reserved
542 * 51:48 reserved for future
543 * 47:6 physical base address of PD or PTE
544 * 5:3 reserved
545 * 2 C
546 * 1 system
547 * 0 valid
548 */
549
gmc_v12_1_get_vm_pde(struct amdgpu_device * adev,int level,uint64_t * addr,uint64_t * flags)550 static void gmc_v12_1_get_vm_pde(struct amdgpu_device *adev, int level,
551 uint64_t *addr, uint64_t *flags)
552 {
553 if (!(*flags & AMDGPU_PDE_PTE_GFX12) && !(*flags & AMDGPU_PTE_SYSTEM))
554 *addr = adev->vm_manager.vram_base_offset + *addr -
555 adev->gmc.vram_start;
556 BUG_ON(*addr & 0xFFFF00000000003FULL);
557
558 *flags |= AMDGPU_PTE_SNOOPED;
559
560 if (!adev->gmc.translate_further)
561 return;
562
563 if (level == AMDGPU_VM_PDB1) {
564 /* Set the block fragment size */
565 if (!(*flags & AMDGPU_PDE_PTE_GFX12))
566 *flags |= AMDGPU_PDE_BFS_GFX12(0x9);
567
568 } else if (level == AMDGPU_VM_PDB0) {
569 if (*flags & AMDGPU_PDE_PTE_GFX12)
570 *flags &= ~AMDGPU_PDE_PTE_GFX12;
571 }
572 }
573
gmc_v12_1_get_coherence_flags(struct amdgpu_device * adev,struct amdgpu_bo * bo,uint64_t * flags)574 static void gmc_v12_1_get_coherence_flags(struct amdgpu_device *adev,
575 struct amdgpu_bo *bo,
576 uint64_t *flags)
577 {
578 struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
579 bool is_vram = bo->tbo.resource &&
580 bo->tbo.resource->mem_type == TTM_PL_VRAM;
581 bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
582 AMDGPU_GEM_CREATE_EXT_COHERENT);
583 bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
584 uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
585 bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
586 unsigned int mtype, mtype_local, mtype_remote;
587 bool snoop = false;
588 bool is_local = false;
589 bool is_aid_a1;
590
591 switch (gc_ip_version) {
592 case IP_VERSION(12, 1, 0):
593 is_aid_a1 = (adev->rev_id & 0x10);
594
595 mtype_local = is_aid_a1 ? MTYPE_RW : MTYPE_NC;
596 mtype_remote = is_aid_a1 ? MTYPE_NC : MTYPE_UC;
597 if (amdgpu_mtype_local == 0) {
598 DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
599 mtype_local = MTYPE_RW;
600 } else if (amdgpu_mtype_local == 1) {
601 DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
602 mtype_local = MTYPE_NC;
603 } else if (amdgpu_mtype_local == 2) {
604 DRM_INFO_ONCE("MTYPE_CC not supported, using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
605 } else {
606 DRM_INFO_ONCE("Using %s for local memory\n", is_aid_a1 ? "MTYPE_RW" : "MTYPE_NC");
607 }
608
609 is_local = (is_vram && adev == bo_adev);
610 snoop = true;
611 if (uncached) {
612 mtype = MTYPE_UC;
613 } else if (ext_coherent) {
614 mtype = is_local ? mtype_local : MTYPE_UC;
615 } else {
616 mtype = is_local ? mtype_local : mtype_remote;
617 }
618 break;
619 default:
620 if (uncached || coherent)
621 mtype = MTYPE_UC;
622 else
623 mtype = MTYPE_NC;
624 }
625
626 if (mtype != MTYPE_NC)
627 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, mtype);
628
629 if (is_local || adev->have_atomics_support)
630 *flags |= AMDGPU_PTE_BUS_ATOMICS;
631
632 *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
633 }
634
gmc_v12_1_get_vm_pte(struct amdgpu_device * adev,struct amdgpu_vm * vm,struct amdgpu_bo * bo,uint32_t vm_flags,uint64_t * flags)635 static void gmc_v12_1_get_vm_pte(struct amdgpu_device *adev,
636 struct amdgpu_vm *vm,
637 struct amdgpu_bo *bo,
638 uint32_t vm_flags,
639 uint64_t *flags)
640 {
641 if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
642 *flags |= AMDGPU_PTE_EXECUTABLE;
643 else
644 *flags &= ~AMDGPU_PTE_EXECUTABLE;
645
646 switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
647 case AMDGPU_VM_MTYPE_DEFAULT:
648 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
649 break;
650 case AMDGPU_VM_MTYPE_NC:
651 default:
652 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_NC);
653 break;
654 case AMDGPU_VM_MTYPE_RW:
655 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_RW);
656 break;
657 case AMDGPU_VM_MTYPE_UC:
658 *flags = AMDGPU_PTE_MTYPE_GFX12(*flags, MTYPE_UC);
659 break;
660 }
661
662 if ((*flags & AMDGPU_PTE_VALID) && bo)
663 gmc_v12_1_get_coherence_flags(adev, bo, flags);
664 }
665
666 static const struct amdgpu_gmc_funcs gmc_v12_1_gmc_funcs = {
667 .flush_gpu_tlb = gmc_v12_1_flush_gpu_tlb,
668 .flush_gpu_tlb_pasid = gmc_v12_1_flush_gpu_tlb_pasid,
669 .emit_flush_gpu_tlb = gmc_v12_1_emit_flush_gpu_tlb,
670 .emit_pasid_mapping = gmc_v12_1_emit_pasid_mapping,
671 .get_vm_pde = gmc_v12_1_get_vm_pde,
672 .get_vm_pte = gmc_v12_1_get_vm_pte,
673 .query_mem_partition_mode = &amdgpu_gmc_query_memory_partition,
674 .request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
675 };
676
gmc_v12_1_set_gmc_funcs(struct amdgpu_device * adev)677 void gmc_v12_1_set_gmc_funcs(struct amdgpu_device *adev)
678 {
679 adev->gmc.gmc_funcs = &gmc_v12_1_gmc_funcs;
680 }
681
682 static const struct amdgpu_irq_src_funcs gmc_v12_1_irq_funcs = {
683 .set = gmc_v12_1_vm_fault_interrupt_state,
684 .process = gmc_v12_1_process_interrupt,
685 };
686
687 static const struct amdgpu_irq_src_funcs gmc_v12_1_ecc_funcs = {
688 .process = amdgpu_umc_uniras_process_ecc_irq,
689 };
690
gmc_v12_1_set_irq_funcs(struct amdgpu_device * adev)691 void gmc_v12_1_set_irq_funcs(struct amdgpu_device *adev)
692 {
693 adev->gmc.vm_fault.num_types = 1;
694 adev->gmc.vm_fault.funcs = &gmc_v12_1_irq_funcs;
695
696 adev->gmc.ecc_irq.num_types = 1;
697 adev->gmc.ecc_irq.funcs = &gmc_v12_1_ecc_funcs;
698 }
699
gmc_v12_1_init_vram_info(struct amdgpu_device * adev)700 void gmc_v12_1_init_vram_info(struct amdgpu_device *adev)
701 {
702 /* TODO: query vram_info from ip discovery binary */
703 adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM4;
704 adev->gmc.vram_width = 384 * 64;
705 }
706