xref: /linux/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 /*
2  * Copyright 2021 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include <linux/pci.h>
25 
26 #include <drm/drm_cache.h>
27 
28 #include "amdgpu.h"
29 #include "amdgpu_ip.h"
30 #include "amdgpu_atomfirmware.h"
31 #include "gmc_v11_0.h"
32 #include "umc_v8_10.h"
33 #include "athub/athub_3_0_0_sh_mask.h"
34 #include "athub/athub_3_0_0_offset.h"
35 #include "dcn/dcn_3_2_0_offset.h"
36 #include "dcn/dcn_3_2_0_sh_mask.h"
37 #include "oss/osssys_6_0_0_offset.h"
38 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
39 #include "navi10_enum.h"
40 #include "soc15.h"
41 #include "soc15d.h"
42 #include "soc15_common.h"
43 #include "nbio_v4_3.h"
44 #include "gfxhub_v3_0.h"
45 #include "gfxhub_v3_0_3.h"
46 #include "gfxhub_v11_5_0.h"
47 #include "mmhub_v3_0.h"
48 #include "mmhub_v3_0_1.h"
49 #include "mmhub_v3_0_2.h"
50 #include "mmhub_v3_3.h"
51 #include "athub_v3_0.h"
52 
53 
54 static int gmc_v11_0_ecc_interrupt_state(struct amdgpu_device *adev,
55 					 struct amdgpu_irq_src *src,
56 					 unsigned int type,
57 					 enum amdgpu_interrupt_state state)
58 {
59 	return 0;
60 }
61 
62 static int
63 gmc_v11_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
64 				   struct amdgpu_irq_src *src, unsigned int type,
65 				   enum amdgpu_interrupt_state state)
66 {
67 	switch (state) {
68 	case AMDGPU_IRQ_STATE_DISABLE:
69 		/* MM HUB */
70 		amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), false);
71 		/* GFX HUB */
72 		/* This works because this interrupt is only
73 		 * enabled at init/resume and disabled in
74 		 * fini/suspend, so the overall state doesn't
75 		 * change over the course of suspend/resume.
76 		 */
77 		if (!adev->in_s0ix && (adev->in_runpm || adev->in_suspend ||
78 							   amdgpu_in_reset(adev)))
79 			amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), false);
80 		break;
81 	case AMDGPU_IRQ_STATE_ENABLE:
82 		/* MM HUB */
83 		amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), true);
84 		/* GFX HUB */
85 		/* This works because this interrupt is only
86 		 * enabled at init/resume and disabled in
87 		 * fini/suspend, so the overall state doesn't
88 		 * change over the course of suspend/resume.
89 		 */
90 		if (!adev->in_s0ix)
91 			amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), true);
92 		break;
93 	default:
94 		break;
95 	}
96 
97 	return 0;
98 }
99 
100 static int gmc_v11_0_process_interrupt(struct amdgpu_device *adev,
101 				       struct amdgpu_irq_src *source,
102 				       struct amdgpu_iv_entry *entry)
103 {
104 	uint32_t vmhub_index = entry->client_id == SOC21_IH_CLIENTID_VMC ?
105 			       AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0);
106 	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub_index];
107 	bool retry_fault = !!(entry->src_data[1] &
108 			      AMDGPU_GMC9_FAULT_SOURCE_DATA_RETRY);
109 	bool write_fault = !!(entry->src_data[1] &
110 			      AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE);
111 	uint32_t status = 0;
112 	uint32_t cam_index;
113 	u64 addr;
114 
115 	addr = (u64)entry->src_data[0] << 12;
116 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
117 
118 	if (retry_fault) {
119 		cam_index = entry->src_data[2] & 0x3ff;
120 
121 		int ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, cam_index, 0,
122 							write_fault);
123 		/* Returning 1 here also prevents sending the IV to the KFD */
124 		if (ret == 1)
125 			return 1;
126 	}
127 
128 	if (!amdgpu_sriov_vf(adev)) {
129 		/*
130 		 * Issue a dummy read to wait for the status register to
131 		 * be updated to avoid reading an incorrect value due to
132 		 * the new fast GRBM interface.
133 		 */
134 		if (entry->vmid_src == AMDGPU_GFXHUB(0))
135 			RREG32(hub->vm_l2_pro_fault_status);
136 
137 		status = RREG32(hub->vm_l2_pro_fault_status);
138 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
139 
140 		amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status,
141 					     entry->vmid_src ? AMDGPU_MMHUB0(0) : AMDGPU_GFXHUB(0));
142 	}
143 
144 	if (printk_ratelimit()) {
145 		struct amdgpu_task_info *task_info;
146 
147 		dev_err(adev->dev,
148 			"[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n",
149 			entry->vmid_src ? "mmhub" : "gfxhub",
150 			entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
151 		task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
152 		if (task_info) {
153 			amdgpu_vm_print_task_info(adev, task_info);
154 			amdgpu_vm_put_task_info(task_info);
155 		}
156 
157 		dev_err(adev->dev, "  in page starting at address 0x%016llx from client %d\n",
158 				addr, entry->client_id);
159 
160 		/* Only print L2 fault status if the status register could be read and
161 		 * contains useful information
162 		 */
163 		if (status != 0)
164 			hub->vmhub_funcs->print_l2_protection_fault_status(adev, status);
165 	}
166 
167 	return 0;
168 }
169 
170 static const struct amdgpu_irq_src_funcs gmc_v11_0_irq_funcs = {
171 	.set = gmc_v11_0_vm_fault_interrupt_state,
172 	.process = gmc_v11_0_process_interrupt,
173 };
174 
175 static const struct amdgpu_irq_src_funcs gmc_v11_0_ecc_funcs = {
176 	.set = gmc_v11_0_ecc_interrupt_state,
177 	.process = amdgpu_umc_process_ecc_irq,
178 };
179 
180 static void gmc_v11_0_set_irq_funcs(struct amdgpu_device *adev)
181 {
182 	adev->gmc.vm_fault.num_types = 1;
183 	adev->gmc.vm_fault.funcs = &gmc_v11_0_irq_funcs;
184 
185 	if (!amdgpu_sriov_vf(adev)) {
186 		adev->gmc.ecc_irq.num_types = 1;
187 		adev->gmc.ecc_irq.funcs = &gmc_v11_0_ecc_funcs;
188 	}
189 }
190 
191 /**
192  * gmc_v11_0_use_invalidate_semaphore - judge whether to use semaphore
193  *
194  * @adev: amdgpu_device pointer
195  * @vmhub: vmhub type
196  *
197  */
198 static bool gmc_v11_0_use_invalidate_semaphore(struct amdgpu_device *adev,
199 				       uint32_t vmhub)
200 {
201 	return ((vmhub == AMDGPU_MMHUB0(0)) &&
202 		(!amdgpu_sriov_vf(adev)));
203 }
204 
205 static bool gmc_v11_0_get_vmid_pasid_mapping_info(
206 					struct amdgpu_device *adev,
207 					uint8_t vmid, uint16_t *p_pasid)
208 {
209 	*p_pasid = RREG32(SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid) & 0xffff;
210 
211 	return !!(*p_pasid);
212 }
213 
214 /**
215  * gmc_v11_0_flush_gpu_tlb - gart tlb flush callback
216  *
217  * @adev: amdgpu_device pointer
218  * @vmid: vm instance to flush
219  * @vmhub: which hub to flush
220  * @flush_type: the flush type
221  *
222  * Flush the TLB for the requested page table.
223  */
224 static void gmc_v11_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
225 					uint32_t vmhub, uint32_t flush_type)
226 {
227 	bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(adev, vmhub);
228 	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
229 	u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
230 	/* Use register 17 for GART */
231 	const unsigned int eng = 17;
232 	unsigned char hub_ip;
233 	u32 sem, req, ack;
234 	unsigned int i;
235 	u32 tmp;
236 
237 	if ((vmhub == AMDGPU_GFXHUB(0)) && !adev->gfx.is_poweron)
238 		return;
239 
240 	sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;
241 	req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
242 	ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
243 
244 	/* flush hdp cache */
245 	amdgpu_device_flush_hdp(adev, NULL);
246 
247 	/* This is necessary for SRIOV as well as for GFXOFF to function
248 	 * properly under bare metal
249 	 */
250 	if ((adev->gfx.kiq[0].ring.sched.ready || adev->mes.ring[0].sched.ready) &&
251 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
252 		amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
253 						 1 << vmid, GET_INST(GC, 0));
254 		return;
255 	}
256 
257 	/* This path is needed before KIQ/MES/GFXOFF are set up */
258 	hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ? GC_HWIP : MMHUB_HWIP;
259 
260 	spin_lock(&adev->gmc.invalidate_lock);
261 	/*
262 	 * It may lose gpuvm invalidate acknowldege state across power-gating
263 	 * off cycle, add semaphore acquire before invalidation and semaphore
264 	 * release after invalidation to avoid entering power gated state
265 	 * to WA the Issue
266 	 */
267 
268 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
269 	if (use_semaphore) {
270 		for (i = 0; i < adev->usec_timeout; i++) {
271 			/* a read return value of 1 means semaphore acuqire */
272 			tmp = RREG32_RLC_NO_KIQ(sem, hub_ip);
273 			if (tmp & 0x1)
274 				break;
275 			udelay(1);
276 		}
277 
278 		if (i >= adev->usec_timeout)
279 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
280 	}
281 
282 	WREG32_RLC_NO_KIQ(req, inv_req, hub_ip);
283 
284 	/* Wait for ACK with a delay.*/
285 	for (i = 0; i < adev->usec_timeout; i++) {
286 		tmp = RREG32_RLC_NO_KIQ(ack, hub_ip);
287 		tmp &= 1 << vmid;
288 		if (tmp)
289 			break;
290 
291 		udelay(1);
292 	}
293 
294 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
295 	if (use_semaphore)
296 		WREG32_RLC_NO_KIQ(sem, 0, hub_ip);
297 
298 	/* Issue additional private vm invalidation to MMHUB */
299 	if ((vmhub != AMDGPU_GFXHUB(0)) &&
300 	    (hub->vm_l2_bank_select_reserved_cid2) &&
301 		!amdgpu_sriov_vf(adev)) {
302 		inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
303 		/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
304 		inv_req |= (1 << 25);
305 		/* Issue private invalidation */
306 		WREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2, inv_req);
307 		/* Read back to ensure invalidation is done*/
308 		RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
309 	}
310 
311 	spin_unlock(&adev->gmc.invalidate_lock);
312 
313 	if (i >= adev->usec_timeout)
314 		dev_err(adev->dev, "Timeout waiting for VM flush ACK!\n");
315 }
316 
317 /**
318  * gmc_v11_0_flush_gpu_tlb_pasid - tlb flush via pasid
319  *
320  * @adev: amdgpu_device pointer
321  * @pasid: pasid to be flush
322  * @flush_type: the flush type
323  * @all_hub: flush all hubs
324  * @inst: is used to select which instance of KIQ to use for the invalidation
325  *
326  * Flush the TLB for the requested pasid.
327  */
328 static void gmc_v11_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
329 					  uint16_t pasid, uint32_t flush_type,
330 					  bool all_hub, uint32_t inst)
331 {
332 	uint16_t queried;
333 	int vmid, i;
334 
335 	for (vmid = 1; vmid < 16; vmid++) {
336 		bool valid;
337 
338 		valid = gmc_v11_0_get_vmid_pasid_mapping_info(adev, vmid,
339 							      &queried);
340 		if (!valid || queried != pasid)
341 			continue;
342 
343 		if (all_hub) {
344 			for_each_set_bit(i, adev->vmhubs_mask,
345 					 AMDGPU_MAX_VMHUBS)
346 				gmc_v11_0_flush_gpu_tlb(adev, vmid, i,
347 							flush_type);
348 		} else {
349 			gmc_v11_0_flush_gpu_tlb(adev, vmid, AMDGPU_GFXHUB(0),
350 						flush_type);
351 		}
352 	}
353 }
354 
355 static uint64_t gmc_v11_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
356 					     unsigned int vmid, uint64_t pd_addr)
357 {
358 	bool use_semaphore = gmc_v11_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
359 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
360 	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
361 	unsigned int eng = ring->vm_inv_eng;
362 
363 	/*
364 	 * It may lose gpuvm invalidate acknowldege state across power-gating
365 	 * off cycle, add semaphore acquire before invalidation and semaphore
366 	 * release after invalidation to avoid entering power gated state
367 	 * to WA the Issue
368 	 */
369 
370 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
371 	if (use_semaphore)
372 		/* a read return value of 1 means semaphore acuqire */
373 		amdgpu_ring_emit_reg_wait(ring,
374 					  hub->vm_inv_eng0_sem +
375 					  hub->eng_distance * eng, 0x1, 0x1);
376 
377 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
378 			      (hub->ctx_addr_distance * vmid),
379 			      lower_32_bits(pd_addr));
380 
381 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
382 			      (hub->ctx_addr_distance * vmid),
383 			      upper_32_bits(pd_addr));
384 
385 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
386 					    hub->eng_distance * eng,
387 					    hub->vm_inv_eng0_ack +
388 					    hub->eng_distance * eng,
389 					    req, 1 << vmid);
390 
391 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
392 	if (use_semaphore)
393 		/*
394 		 * add semaphore release after invalidation,
395 		 * write with 0 means semaphore release
396 		 */
397 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
398 				      hub->eng_distance * eng, 0);
399 
400 	return pd_addr;
401 }
402 
403 static void gmc_v11_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
404 					 unsigned int pasid)
405 {
406 	struct amdgpu_device *adev = ring->adev;
407 	uint32_t reg;
408 
409 	if (ring->vm_hub == AMDGPU_GFXHUB(0))
410 		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT) + vmid;
411 	else
412 		reg = SOC15_REG_OFFSET(OSSSYS, 0, regIH_VMID_0_LUT_MM) + vmid;
413 
414 	amdgpu_ring_emit_wreg(ring, reg, pasid);
415 }
416 
417 /*
418  * PTE format:
419  * 63:59 reserved
420  * 58:57 reserved
421  * 56 F
422  * 55 L
423  * 54 reserved
424  * 53:52 SW
425  * 51 T
426  * 50:48 mtype
427  * 47:12 4k physical page base address
428  * 11:7 fragment
429  * 6 write
430  * 5 read
431  * 4 exe
432  * 3 Z
433  * 2 snooped
434  * 1 system
435  * 0 valid
436  *
437  * PDE format:
438  * 63:59 block fragment size
439  * 58:55 reserved
440  * 54 P
441  * 53:48 reserved
442  * 47:6 physical base address of PD or PTE
443  * 5:3 reserved
444  * 2 C
445  * 1 system
446  * 0 valid
447  */
448 
449 static void gmc_v11_0_get_vm_pde(struct amdgpu_device *adev, int level,
450 				 uint64_t *addr, uint64_t *flags)
451 {
452 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
453 		*addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
454 	BUG_ON(*addr & 0xFFFF00000000003FULL);
455 
456 	if (!adev->gmc.translate_further)
457 		return;
458 
459 	if (level == AMDGPU_VM_PDB1) {
460 		/* Set the block fragment size */
461 		if (!(*flags & AMDGPU_PDE_PTE))
462 			*flags |= AMDGPU_PDE_BFS(0x9);
463 
464 	} else if (level == AMDGPU_VM_PDB0) {
465 		if (*flags & AMDGPU_PDE_PTE)
466 			*flags &= ~AMDGPU_PDE_PTE;
467 		else
468 			*flags |= AMDGPU_PTE_TF;
469 	}
470 }
471 
472 static void gmc_v11_0_get_vm_pte(struct amdgpu_device *adev,
473 				 struct amdgpu_vm *vm,
474 				 struct amdgpu_bo *bo,
475 				 uint32_t vm_flags,
476 				 uint64_t *flags)
477 {
478 	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
479 		*flags |= AMDGPU_PTE_EXECUTABLE;
480 	else
481 		*flags &= ~AMDGPU_PTE_EXECUTABLE;
482 
483 	switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
484 	case AMDGPU_VM_MTYPE_DEFAULT:
485 	case AMDGPU_VM_MTYPE_NC:
486 	default:
487 		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_NC);
488 		break;
489 	case AMDGPU_VM_MTYPE_WC:
490 		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_WC);
491 		break;
492 	case AMDGPU_VM_MTYPE_CC:
493 		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_CC);
494 		break;
495 	case AMDGPU_VM_MTYPE_UC:
496 		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
497 		break;
498 	}
499 
500 	if (vm_flags & AMDGPU_VM_PAGE_NOALLOC)
501 		*flags |= AMDGPU_PTE_NOALLOC;
502 	else
503 		*flags &= ~AMDGPU_PTE_NOALLOC;
504 
505 	if (vm_flags & AMDGPU_VM_PAGE_PRT) {
506 		*flags |= AMDGPU_PTE_PRT;
507 		*flags |= AMDGPU_PTE_SNOOPED;
508 		*flags |= AMDGPU_PTE_LOG;
509 		*flags |= AMDGPU_PTE_SYSTEM;
510 		*flags &= ~AMDGPU_PTE_VALID;
511 	}
512 
513 	if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
514 			       AMDGPU_GEM_CREATE_EXT_COHERENT |
515 			       AMDGPU_GEM_CREATE_UNCACHED))
516 		*flags = AMDGPU_PTE_MTYPE_NV10(*flags, MTYPE_UC);
517 }
518 
519 static unsigned int gmc_v11_0_get_vbios_fb_size(struct amdgpu_device *adev)
520 {
521 	u32 d1vga_control = RREG32_SOC15(DCE, 0, regD1VGA_CONTROL);
522 	unsigned int size;
523 
524 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
525 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
526 	} else {
527 		u32 viewport;
528 		u32 pitch;
529 
530 		viewport = RREG32_SOC15(DCE, 0, regHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
531 		pitch = RREG32_SOC15(DCE, 0, regHUBPREQ0_DCSURF_SURFACE_PITCH);
532 		size = (REG_GET_FIELD(viewport,
533 					HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
534 				REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) *
535 				4);
536 	}
537 
538 	return size;
539 }
540 
541 static const struct amdgpu_gmc_funcs gmc_v11_0_gmc_funcs = {
542 	.flush_gpu_tlb = gmc_v11_0_flush_gpu_tlb,
543 	.flush_gpu_tlb_pasid = gmc_v11_0_flush_gpu_tlb_pasid,
544 	.emit_flush_gpu_tlb = gmc_v11_0_emit_flush_gpu_tlb,
545 	.emit_pasid_mapping = gmc_v11_0_emit_pasid_mapping,
546 	.get_vm_pde = gmc_v11_0_get_vm_pde,
547 	.get_vm_pte = gmc_v11_0_get_vm_pte,
548 	.get_vbios_fb_size = gmc_v11_0_get_vbios_fb_size,
549 };
550 
551 static void gmc_v11_0_set_gmc_funcs(struct amdgpu_device *adev)
552 {
553 	adev->gmc.gmc_funcs = &gmc_v11_0_gmc_funcs;
554 }
555 
556 static void gmc_v11_0_set_umc_funcs(struct amdgpu_device *adev)
557 {
558 	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
559 	case IP_VERSION(8, 10, 0):
560 		adev->umc.channel_inst_num = UMC_V8_10_CHANNEL_INSTANCE_NUM;
561 		adev->umc.umc_inst_num = UMC_V8_10_UMC_INSTANCE_NUM;
562 		adev->umc.max_ras_err_cnt_per_query = UMC_V8_10_TOTAL_CHANNEL_NUM(adev);
563 		adev->umc.channel_offs = UMC_V8_10_PER_CHANNEL_OFFSET;
564 		adev->umc.retire_unit = UMC_V8_10_NA_COL_2BITS_POWER_OF_2_NUM;
565 		if (adev->umc.node_inst_num == 4)
566 			adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl_ext0[0][0][0];
567 		else
568 			adev->umc.channel_idx_tbl = &umc_v8_10_channel_idx_tbl[0][0][0];
569 		adev->umc.ras = &umc_v8_10_ras;
570 		break;
571 	case IP_VERSION(8, 11, 0):
572 		break;
573 	default:
574 		break;
575 	}
576 }
577 
578 
579 static void gmc_v11_0_set_mmhub_funcs(struct amdgpu_device *adev)
580 {
581 	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
582 	case IP_VERSION(3, 0, 1):
583 		adev->mmhub.funcs = &mmhub_v3_0_1_funcs;
584 		break;
585 	case IP_VERSION(3, 0, 2):
586 		adev->mmhub.funcs = &mmhub_v3_0_2_funcs;
587 		break;
588 	case IP_VERSION(3, 3, 0):
589 	case IP_VERSION(3, 3, 1):
590 	case IP_VERSION(3, 3, 2):
591 	case IP_VERSION(3, 4, 0):
592 	case IP_VERSION(3, 4, 2):
593 		adev->mmhub.funcs = &mmhub_v3_3_funcs;
594 		break;
595 	default:
596 		adev->mmhub.funcs = &mmhub_v3_0_funcs;
597 		break;
598 	}
599 }
600 
601 static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev)
602 {
603 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
604 	case IP_VERSION(11, 0, 3):
605 		adev->gfxhub.funcs = &gfxhub_v3_0_3_funcs;
606 		break;
607 	case IP_VERSION(11, 5, 0):
608 	case IP_VERSION(11, 5, 1):
609 	case IP_VERSION(11, 5, 2):
610 	case IP_VERSION(11, 5, 3):
611 	case IP_VERSION(11, 5, 4):
612 	case IP_VERSION(11, 5, 6):
613 	case IP_VERSION(11, 7, 0):
614 	case IP_VERSION(11, 7, 1):
615 		adev->gfxhub.funcs = &gfxhub_v11_5_0_funcs;
616 		break;
617 	default:
618 		adev->gfxhub.funcs = &gfxhub_v3_0_funcs;
619 		break;
620 	}
621 }
622 
623 static int gmc_v11_0_early_init(struct amdgpu_ip_block *ip_block)
624 {
625 	struct amdgpu_device *adev = ip_block->adev;
626 
627 	gmc_v11_0_set_gfxhub_funcs(adev);
628 	gmc_v11_0_set_mmhub_funcs(adev);
629 	gmc_v11_0_set_gmc_funcs(adev);
630 	gmc_v11_0_set_irq_funcs(adev);
631 	gmc_v11_0_set_umc_funcs(adev);
632 
633 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
634 	adev->gmc.shared_aperture_end =
635 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
636 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
637 	adev->gmc.private_aperture_end =
638 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
639 	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
640 
641 	return 0;
642 }
643 
644 static int gmc_v11_0_late_init(struct amdgpu_ip_block *ip_block)
645 {
646 	struct amdgpu_device *adev = ip_block->adev;
647 	int r;
648 
649 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
650 	if (r)
651 		return r;
652 
653 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
654 }
655 
656 static void gmc_v11_0_vram_gtt_location(struct amdgpu_device *adev,
657 					struct amdgpu_gmc *mc)
658 {
659 	u64 base = 0;
660 
661 	base = adev->mmhub.funcs->get_fb_location(adev);
662 
663 	amdgpu_gmc_set_agp_default(adev, mc);
664 	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
665 	amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_HIGH);
666 	if (!amdgpu_sriov_vf(adev) &&
667 	    (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(11, 5, 0)) &&
668 	    (amdgpu_agp == 1))
669 		amdgpu_gmc_agp_location(adev, mc);
670 
671 	/* base offset of vram pages */
672 	if (amdgpu_sriov_vf(adev))
673 		adev->vm_manager.vram_base_offset = 0;
674 	else
675 		adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev);
676 }
677 
678 /**
679  * gmc_v11_0_mc_init - initialize the memory controller driver params
680  *
681  * @adev: amdgpu_device pointer
682  *
683  * Look up the amount of vram, vram width, and decide how to place
684  * vram and gart within the GPU's physical address space.
685  * Returns 0 for success.
686  */
687 static int gmc_v11_0_mc_init(struct amdgpu_device *adev)
688 {
689 	int r;
690 
691 	/* size in MB on si */
692 	adev->gmc.mc_vram_size =
693 		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
694 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
695 
696 	if (!(adev->flags & AMD_IS_APU)) {
697 		r = amdgpu_device_resize_fb_bar(adev);
698 		if (r)
699 			return r;
700 	}
701 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
702 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
703 
704 #ifdef CONFIG_X86_64
705 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {
706 		adev->gmc.aper_base = adev->mmhub.funcs->get_mc_fb_offset(adev);
707 		adev->gmc.aper_size = adev->gmc.real_vram_size;
708 	}
709 #endif
710 	/* In case the PCI BAR is larger than the actual amount of vram */
711 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
712 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
713 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
714 
715 	/* set the gart size */
716 	amdgpu_gmc_set_gart_size(adev, SZ_512M);
717 
718 	gmc_v11_0_vram_gtt_location(adev, &adev->gmc);
719 
720 	return 0;
721 }
722 
723 static int gmc_v11_0_gart_init(struct amdgpu_device *adev)
724 {
725 	int r;
726 
727 	if (adev->gart.bo) {
728 		WARN(1, "PCIE GART already initialized\n");
729 		return 0;
730 	}
731 
732 	/* Initialize common gart structure */
733 	r = amdgpu_gart_init(adev);
734 	if (r)
735 		return r;
736 
737 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
738 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(0ULL, MTYPE_UC) |
739 				 AMDGPU_PTE_EXECUTABLE;
740 
741 	return amdgpu_gart_table_vram_alloc(adev);
742 }
743 
744 static int gmc_v11_0_sw_init(struct amdgpu_ip_block *ip_block)
745 {
746 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
747 	struct amdgpu_device *adev = ip_block->adev;
748 
749 	adev->mmhub.funcs->init(adev);
750 
751 	adev->gfxhub.funcs->init(adev);
752 
753 	spin_lock_init(&adev->gmc.invalidate_lock);
754 
755 	r = amdgpu_gmc_get_vram_info(adev,
756 					      &vram_width, &vram_type, &vram_vendor);
757 	adev->gmc.vram_width = vram_width;
758 
759 	adev->gmc.vram_type = vram_type;
760 	adev->gmc.vram_vendor = vram_vendor;
761 
762 	/* The mall_size is already calculated as mall_size_per_umc * num_umc.
763 	 * However, for gfx1151, which features a 2-to-1 UMC mapping,
764 	 * the result must be multiplied by 2 to determine the actual mall size.
765 	 */
766 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
767 	case IP_VERSION(11, 5, 1):
768 		adev->gmc.mall_size *= 2;
769 		break;
770 	default:
771 		break;
772 	}
773 
774 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
775 	case IP_VERSION(11, 0, 0):
776 	case IP_VERSION(11, 0, 1):
777 	case IP_VERSION(11, 0, 2):
778 	case IP_VERSION(11, 0, 3):
779 	case IP_VERSION(11, 0, 4):
780 	case IP_VERSION(11, 5, 0):
781 	case IP_VERSION(11, 5, 1):
782 	case IP_VERSION(11, 5, 2):
783 	case IP_VERSION(11, 5, 3):
784 	case IP_VERSION(11, 5, 4):
785 	case IP_VERSION(11, 5, 6):
786 	case IP_VERSION(11, 7, 0):
787 	case IP_VERSION(11, 7, 1):
788 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
789 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
790 		/*
791 		 * To fulfill 4-level page support,
792 		 * vm size is 256TB (48bit), maximum size,
793 		 * block size 512 (9bit)
794 		 */
795 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
796 		break;
797 	default:
798 		break;
799 	}
800 
801 	/* This interrupt is VMC page fault.*/
802 	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VMC,
803 			      VMC_1_0__SRCID__VM_FAULT,
804 			      &adev->gmc.vm_fault);
805 
806 	if (r)
807 		return r;
808 
809 	r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX,
810 			      UTCL2_1_0__SRCID__FAULT,
811 			      &adev->gmc.vm_fault);
812 	if (r)
813 		return r;
814 
815 	if (!amdgpu_sriov_vf(adev)) {
816 		/* interrupt sent to DF. */
817 		r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_DF, 0,
818 				      &adev->gmc.ecc_irq);
819 		if (r)
820 			return r;
821 	}
822 
823 	/*
824 	 * Set the internal MC address mask This is the max address of the GPU's
825 	 * internal address space.
826 	 */
827 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
828 	adev->gmc.pte_addr_mask = 0x0000FFFFFFFFF000ULL; /* 48 bit PA */
829 
830 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
831 	if (r) {
832 		drm_warn(adev_to_drm(adev), "No suitable DMA available.\n");
833 		return r;
834 	}
835 
836 	adev->need_swiotlb = drm_need_swiotlb(44);
837 
838 	r = gmc_v11_0_mc_init(adev);
839 	if (r)
840 		return r;
841 
842 	/* Memory manager */
843 	r = amdgpu_bo_init(adev);
844 	if (r)
845 		return r;
846 
847 	r = gmc_v11_0_gart_init(adev);
848 	if (r)
849 		return r;
850 
851 	/*
852 	 * number of VMs
853 	 * VMID 0 is reserved for System
854 	 * amdgpu graphics/compute will use VMIDs 1-7
855 	 * amdkfd will use VMIDs 8-15
856 	 */
857 	adev->vm_manager.first_kfd_vmid = adev->gfx.disable_kq ? 1 : 8;
858 
859 	amdgpu_vmid_mgr_set_vmid_mask(adev,
860 				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
861 				      false);
862 	amdgpu_vmid_mgr_set_vmid_mask(adev, GENMASK(AMDGPU_NUM_VMID - 1, 1), true);
863 
864 	amdgpu_vm_manager_init(adev);
865 
866 	r = amdgpu_gmc_ras_sw_init(adev);
867 	if (r)
868 		return r;
869 
870 	return 0;
871 }
872 
873 /**
874  * gmc_v11_0_gart_fini - vm fini callback
875  *
876  * @adev: amdgpu_device pointer
877  *
878  * Tears down the driver GART/VM setup (CIK).
879  */
880 static void gmc_v11_0_gart_fini(struct amdgpu_device *adev)
881 {
882 	amdgpu_gart_table_vram_free(adev);
883 }
884 
885 static int gmc_v11_0_sw_fini(struct amdgpu_ip_block *ip_block)
886 {
887 	struct amdgpu_device *adev = ip_block->adev;
888 
889 	amdgpu_vm_manager_fini(adev);
890 	gmc_v11_0_gart_fini(adev);
891 	amdgpu_gem_force_release(adev);
892 	amdgpu_bo_fini(adev);
893 
894 	return 0;
895 }
896 
897 static void gmc_v11_0_init_golden_registers(struct amdgpu_device *adev)
898 {
899 	if (amdgpu_sriov_vf(adev)) {
900 		struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB0(0)];
901 
902 		WREG32(hub->vm_contexts_disable, 0);
903 		return;
904 	}
905 }
906 
907 /**
908  * gmc_v11_0_gart_enable - gart enable
909  *
910  * @adev: amdgpu_device pointer
911  */
912 static int gmc_v11_0_gart_enable(struct amdgpu_device *adev)
913 {
914 	int r;
915 	bool value;
916 
917 	if (adev->gart.bo == NULL) {
918 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
919 		return -EINVAL;
920 	}
921 
922 	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
923 
924 	r = adev->mmhub.funcs->gart_enable(adev);
925 	if (r)
926 		return r;
927 
928 	/* Flush HDP after it is initialized */
929 	amdgpu_device_flush_hdp(adev, NULL);
930 
931 	value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS;
932 
933 	adev->mmhub.funcs->set_fault_enable_default(adev, value);
934 	gmc_v11_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0);
935 
936 	drm_info(adev_to_drm(adev), "PCIE GART of %uM enabled (table at 0x%016llX).\n",
937 		 (unsigned int)(adev->gmc.gart_size >> 20),
938 		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
939 
940 	return 0;
941 }
942 
943 static int gmc_v11_0_hw_init(struct amdgpu_ip_block *ip_block)
944 {
945 	struct amdgpu_device *adev = ip_block->adev;
946 	int r;
947 
948 	adev->gmc.flush_pasid_uses_kiq = !amdgpu_emu_mode;
949 
950 	/* The sequence of these two function calls matters.*/
951 	gmc_v11_0_init_golden_registers(adev);
952 
953 	r = gmc_v11_0_gart_enable(adev);
954 	if (r)
955 		return r;
956 
957 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
958 		adev->umc.funcs->init_registers(adev);
959 
960 	return 0;
961 }
962 
963 /**
964  * gmc_v11_0_gart_disable - gart disable
965  *
966  * @adev: amdgpu_device pointer
967  *
968  * This disables all VM page table.
969  */
970 static void gmc_v11_0_gart_disable(struct amdgpu_device *adev)
971 {
972 	adev->mmhub.funcs->gart_disable(adev);
973 }
974 
975 static int gmc_v11_0_hw_fini(struct amdgpu_ip_block *ip_block)
976 {
977 	struct amdgpu_device *adev = ip_block->adev;
978 
979 	if (amdgpu_sriov_vf(adev)) {
980 		/* full access mode, so don't touch any GMC register */
981 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
982 		return 0;
983 	}
984 
985 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
986 
987 	if (adev->gmc.ecc_irq.funcs &&
988 		amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))
989 		amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
990 
991 	gmc_v11_0_gart_disable(adev);
992 
993 	return 0;
994 }
995 
996 static int gmc_v11_0_suspend(struct amdgpu_ip_block *ip_block)
997 {
998 	gmc_v11_0_hw_fini(ip_block);
999 
1000 	return 0;
1001 }
1002 
1003 static int gmc_v11_0_resume(struct amdgpu_ip_block *ip_block)
1004 {
1005 	int r;
1006 
1007 	r = gmc_v11_0_hw_init(ip_block);
1008 	if (r)
1009 		return r;
1010 
1011 	amdgpu_vmid_reset_all(ip_block->adev);
1012 
1013 	return 0;
1014 }
1015 
1016 static int gmc_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
1017 {
1018 	/* There is no need to wait for MC idle in GMC v11.*/
1019 	return 0;
1020 }
1021 
1022 static int gmc_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
1023 					   enum amd_clockgating_state state)
1024 {
1025 	int r;
1026 	struct amdgpu_device *adev = ip_block->adev;
1027 
1028 	r = adev->mmhub.funcs->set_clockgating(adev, state);
1029 	if (r)
1030 		return r;
1031 
1032 	return athub_v3_0_set_clockgating(adev, state);
1033 }
1034 
1035 static void gmc_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
1036 {
1037 	struct amdgpu_device *adev = ip_block->adev;
1038 
1039 	adev->mmhub.funcs->get_clockgating(adev, flags);
1040 
1041 	athub_v3_0_get_clockgating(adev, flags);
1042 }
1043 
1044 static int gmc_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
1045 					   enum amd_powergating_state state)
1046 {
1047 	return 0;
1048 }
1049 
1050 const struct amd_ip_funcs gmc_v11_0_ip_funcs = {
1051 	.name = "gmc_v11_0",
1052 	.early_init = gmc_v11_0_early_init,
1053 	.sw_init = gmc_v11_0_sw_init,
1054 	.hw_init = gmc_v11_0_hw_init,
1055 	.late_init = gmc_v11_0_late_init,
1056 	.sw_fini = gmc_v11_0_sw_fini,
1057 	.hw_fini = gmc_v11_0_hw_fini,
1058 	.suspend = gmc_v11_0_suspend,
1059 	.resume = gmc_v11_0_resume,
1060 	.wait_for_idle = gmc_v11_0_wait_for_idle,
1061 	.set_clockgating_state = gmc_v11_0_set_clockgating_state,
1062 	.set_powergating_state = gmc_v11_0_set_powergating_state,
1063 	.get_clockgating_state = gmc_v11_0_get_clockgating_state,
1064 };
1065 
1066 const struct amdgpu_ip_block_version gmc_v11_0_ip_block = {
1067 	.type = AMD_IP_BLOCK_TYPE_GMC,
1068 	.major = 11,
1069 	.minor = 0,
1070 	.rev = 0,
1071 	.funcs = &gmc_v11_0_ip_funcs,
1072 };
1073