xref: /linux/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c (revision 7d1f84f1b94e79cb95d974ab3ec54cb3a4d64f5b)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/pci.h>
26 
27 #include <drm/drm_cache.h>
28 
29 #include "amdgpu.h"
30 #include "gmc_v9_0.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "amdgpu_gem.h"
33 
34 #include "gc/gc_9_0_sh_mask.h"
35 #include "dce/dce_12_0_offset.h"
36 #include "dce/dce_12_0_sh_mask.h"
37 #include "vega10_enum.h"
38 #include "mmhub/mmhub_1_0_offset.h"
39 #include "athub/athub_1_0_sh_mask.h"
40 #include "athub/athub_1_0_offset.h"
41 #include "oss/osssys_4_0_offset.h"
42 
43 #include "soc15.h"
44 #include "soc15d.h"
45 #include "soc15_common.h"
46 #include "umc/umc_6_0_sh_mask.h"
47 
48 #include "gfxhub_v1_0.h"
49 #include "mmhub_v1_0.h"
50 #include "athub_v1_0.h"
51 #include "gfxhub_v1_1.h"
52 #include "gfxhub_v1_2.h"
53 #include "mmhub_v9_4.h"
54 #include "mmhub_v1_7.h"
55 #include "mmhub_v1_8.h"
56 #include "umc_v6_1.h"
57 #include "umc_v6_0.h"
58 #include "umc_v6_7.h"
59 #include "umc_v12_0.h"
60 #include "ras_umc_v12_0.h"
61 #include "hdp_v4_0.h"
62 #include "mca_v3_0.h"
63 
64 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
65 
66 #include "amdgpu_ip.h"
67 #include "amdgpu_ras.h"
68 #include "amdgpu_xgmi.h"
69 
70 /* add these here since we already include dce12 headers and these are for DCN */
71 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
72 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
73 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
74 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
75 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
76 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
77 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
78 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
79 
80 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2                                                          0x05ea
81 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX                                                 2
82 
83 static const char * const gfxhub_client_ids[] = {
84 	"CB",
85 	"DB",
86 	"IA",
87 	"WD",
88 	"CPF",
89 	"CPC",
90 	"CPG",
91 	"RLC",
92 	"TCP",
93 	"SQC (inst)",
94 	"SQC (data)",
95 	"SQG",
96 	"PA",
97 };
98 
99 static const char *mmhub_client_ids_raven[][2] = {
100 	[0][0] = "MP1",
101 	[1][0] = "MP0",
102 	[2][0] = "VCN",
103 	[3][0] = "VCNU",
104 	[4][0] = "HDP",
105 	[5][0] = "DCE",
106 	[13][0] = "UTCL2",
107 	[19][0] = "TLS",
108 	[26][0] = "OSS",
109 	[27][0] = "SDMA0",
110 	[0][1] = "MP1",
111 	[1][1] = "MP0",
112 	[2][1] = "VCN",
113 	[3][1] = "VCNU",
114 	[4][1] = "HDP",
115 	[5][1] = "XDP",
116 	[6][1] = "DBGU0",
117 	[7][1] = "DCE",
118 	[8][1] = "DCEDWB0",
119 	[9][1] = "DCEDWB1",
120 	[26][1] = "OSS",
121 	[27][1] = "SDMA0",
122 };
123 
124 static const char *mmhub_client_ids_renoir[][2] = {
125 	[0][0] = "MP1",
126 	[1][0] = "MP0",
127 	[2][0] = "HDP",
128 	[4][0] = "DCEDMC",
129 	[5][0] = "DCEVGA",
130 	[13][0] = "UTCL2",
131 	[19][0] = "TLS",
132 	[26][0] = "OSS",
133 	[27][0] = "SDMA0",
134 	[28][0] = "VCN",
135 	[29][0] = "VCNU",
136 	[30][0] = "JPEG",
137 	[0][1] = "MP1",
138 	[1][1] = "MP0",
139 	[2][1] = "HDP",
140 	[3][1] = "XDP",
141 	[6][1] = "DBGU0",
142 	[7][1] = "DCEDMC",
143 	[8][1] = "DCEVGA",
144 	[9][1] = "DCEDWB",
145 	[26][1] = "OSS",
146 	[27][1] = "SDMA0",
147 	[28][1] = "VCN",
148 	[29][1] = "VCNU",
149 	[30][1] = "JPEG",
150 };
151 
152 static const char *mmhub_client_ids_vega10[][2] = {
153 	[0][0] = "MP0",
154 	[1][0] = "UVD",
155 	[2][0] = "UVDU",
156 	[3][0] = "HDP",
157 	[13][0] = "UTCL2",
158 	[14][0] = "OSS",
159 	[15][0] = "SDMA1",
160 	[32+0][0] = "VCE0",
161 	[32+1][0] = "VCE0U",
162 	[32+2][0] = "XDMA",
163 	[32+3][0] = "DCE",
164 	[32+4][0] = "MP1",
165 	[32+14][0] = "SDMA0",
166 	[0][1] = "MP0",
167 	[1][1] = "UVD",
168 	[2][1] = "UVDU",
169 	[3][1] = "DBGU0",
170 	[4][1] = "HDP",
171 	[5][1] = "XDP",
172 	[14][1] = "OSS",
173 	[15][1] = "SDMA0",
174 	[32+0][1] = "VCE0",
175 	[32+1][1] = "VCE0U",
176 	[32+2][1] = "XDMA",
177 	[32+3][1] = "DCE",
178 	[32+4][1] = "DCEDWB",
179 	[32+5][1] = "MP1",
180 	[32+6][1] = "DBGU1",
181 	[32+14][1] = "SDMA1",
182 };
183 
184 static const char *mmhub_client_ids_vega12[][2] = {
185 	[0][0] = "MP0",
186 	[1][0] = "VCE0",
187 	[2][0] = "VCE0U",
188 	[3][0] = "HDP",
189 	[13][0] = "UTCL2",
190 	[14][0] = "OSS",
191 	[15][0] = "SDMA1",
192 	[32+0][0] = "DCE",
193 	[32+1][0] = "XDMA",
194 	[32+2][0] = "UVD",
195 	[32+3][0] = "UVDU",
196 	[32+4][0] = "MP1",
197 	[32+15][0] = "SDMA0",
198 	[0][1] = "MP0",
199 	[1][1] = "VCE0",
200 	[2][1] = "VCE0U",
201 	[3][1] = "DBGU0",
202 	[4][1] = "HDP",
203 	[5][1] = "XDP",
204 	[14][1] = "OSS",
205 	[15][1] = "SDMA0",
206 	[32+0][1] = "DCE",
207 	[32+1][1] = "DCEDWB",
208 	[32+2][1] = "XDMA",
209 	[32+3][1] = "UVD",
210 	[32+4][1] = "UVDU",
211 	[32+5][1] = "MP1",
212 	[32+6][1] = "DBGU1",
213 	[32+15][1] = "SDMA1",
214 };
215 
216 static const char *mmhub_client_ids_vega20[][2] = {
217 	[0][0] = "XDMA",
218 	[1][0] = "DCE",
219 	[2][0] = "VCE0",
220 	[3][0] = "VCE0U",
221 	[4][0] = "UVD",
222 	[5][0] = "UVD1U",
223 	[13][0] = "OSS",
224 	[14][0] = "HDP",
225 	[15][0] = "SDMA0",
226 	[32+0][0] = "UVD",
227 	[32+1][0] = "UVDU",
228 	[32+2][0] = "MP1",
229 	[32+3][0] = "MP0",
230 	[32+12][0] = "UTCL2",
231 	[32+14][0] = "SDMA1",
232 	[0][1] = "XDMA",
233 	[1][1] = "DCE",
234 	[2][1] = "DCEDWB",
235 	[3][1] = "VCE0",
236 	[4][1] = "VCE0U",
237 	[5][1] = "UVD1",
238 	[6][1] = "UVD1U",
239 	[7][1] = "DBGU0",
240 	[8][1] = "XDP",
241 	[13][1] = "OSS",
242 	[14][1] = "HDP",
243 	[15][1] = "SDMA0",
244 	[32+0][1] = "UVD",
245 	[32+1][1] = "UVDU",
246 	[32+2][1] = "DBGU1",
247 	[32+3][1] = "MP1",
248 	[32+4][1] = "MP0",
249 	[32+14][1] = "SDMA1",
250 };
251 
252 static const char *mmhub_client_ids_arcturus[][2] = {
253 	[0][0] = "DBGU1",
254 	[1][0] = "XDP",
255 	[2][0] = "MP1",
256 	[14][0] = "HDP",
257 	[171][0] = "JPEG",
258 	[172][0] = "VCN",
259 	[173][0] = "VCNU",
260 	[203][0] = "JPEG1",
261 	[204][0] = "VCN1",
262 	[205][0] = "VCN1U",
263 	[256][0] = "SDMA0",
264 	[257][0] = "SDMA1",
265 	[258][0] = "SDMA2",
266 	[259][0] = "SDMA3",
267 	[260][0] = "SDMA4",
268 	[261][0] = "SDMA5",
269 	[262][0] = "SDMA6",
270 	[263][0] = "SDMA7",
271 	[384][0] = "OSS",
272 	[0][1] = "DBGU1",
273 	[1][1] = "XDP",
274 	[2][1] = "MP1",
275 	[14][1] = "HDP",
276 	[171][1] = "JPEG",
277 	[172][1] = "VCN",
278 	[173][1] = "VCNU",
279 	[203][1] = "JPEG1",
280 	[204][1] = "VCN1",
281 	[205][1] = "VCN1U",
282 	[256][1] = "SDMA0",
283 	[257][1] = "SDMA1",
284 	[258][1] = "SDMA2",
285 	[259][1] = "SDMA3",
286 	[260][1] = "SDMA4",
287 	[261][1] = "SDMA5",
288 	[262][1] = "SDMA6",
289 	[263][1] = "SDMA7",
290 	[384][1] = "OSS",
291 };
292 
293 static const char *mmhub_client_ids_aldebaran[][2] = {
294 	[2][0] = "MP1",
295 	[3][0] = "MP0",
296 	[32+1][0] = "DBGU_IO0",
297 	[32+2][0] = "DBGU_IO2",
298 	[32+4][0] = "MPIO",
299 	[96+11][0] = "JPEG0",
300 	[96+12][0] = "VCN0",
301 	[96+13][0] = "VCNU0",
302 	[128+11][0] = "JPEG1",
303 	[128+12][0] = "VCN1",
304 	[128+13][0] = "VCNU1",
305 	[160+1][0] = "XDP",
306 	[160+14][0] = "HDP",
307 	[256+0][0] = "SDMA0",
308 	[256+1][0] = "SDMA1",
309 	[256+2][0] = "SDMA2",
310 	[256+3][0] = "SDMA3",
311 	[256+4][0] = "SDMA4",
312 	[384+0][0] = "OSS",
313 	[2][1] = "MP1",
314 	[3][1] = "MP0",
315 	[32+1][1] = "DBGU_IO0",
316 	[32+2][1] = "DBGU_IO2",
317 	[32+4][1] = "MPIO",
318 	[96+11][1] = "JPEG0",
319 	[96+12][1] = "VCN0",
320 	[96+13][1] = "VCNU0",
321 	[128+11][1] = "JPEG1",
322 	[128+12][1] = "VCN1",
323 	[128+13][1] = "VCNU1",
324 	[160+1][1] = "XDP",
325 	[160+14][1] = "HDP",
326 	[256+0][1] = "SDMA0",
327 	[256+1][1] = "SDMA1",
328 	[256+2][1] = "SDMA2",
329 	[256+3][1] = "SDMA3",
330 	[256+4][1] = "SDMA4",
331 	[384+0][1] = "OSS",
332 };
333 
334 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = {
335 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
336 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
337 };
338 
339 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = {
340 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
341 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
342 };
343 
344 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
345 	(0x000143c0 + 0x00000000),
346 	(0x000143c0 + 0x00000800),
347 	(0x000143c0 + 0x00001000),
348 	(0x000143c0 + 0x00001800),
349 	(0x000543c0 + 0x00000000),
350 	(0x000543c0 + 0x00000800),
351 	(0x000543c0 + 0x00001000),
352 	(0x000543c0 + 0x00001800),
353 	(0x000943c0 + 0x00000000),
354 	(0x000943c0 + 0x00000800),
355 	(0x000943c0 + 0x00001000),
356 	(0x000943c0 + 0x00001800),
357 	(0x000d43c0 + 0x00000000),
358 	(0x000d43c0 + 0x00000800),
359 	(0x000d43c0 + 0x00001000),
360 	(0x000d43c0 + 0x00001800),
361 	(0x001143c0 + 0x00000000),
362 	(0x001143c0 + 0x00000800),
363 	(0x001143c0 + 0x00001000),
364 	(0x001143c0 + 0x00001800),
365 	(0x001543c0 + 0x00000000),
366 	(0x001543c0 + 0x00000800),
367 	(0x001543c0 + 0x00001000),
368 	(0x001543c0 + 0x00001800),
369 	(0x001943c0 + 0x00000000),
370 	(0x001943c0 + 0x00000800),
371 	(0x001943c0 + 0x00001000),
372 	(0x001943c0 + 0x00001800),
373 	(0x001d43c0 + 0x00000000),
374 	(0x001d43c0 + 0x00000800),
375 	(0x001d43c0 + 0x00001000),
376 	(0x001d43c0 + 0x00001800),
377 };
378 
379 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
380 	(0x000143e0 + 0x00000000),
381 	(0x000143e0 + 0x00000800),
382 	(0x000143e0 + 0x00001000),
383 	(0x000143e0 + 0x00001800),
384 	(0x000543e0 + 0x00000000),
385 	(0x000543e0 + 0x00000800),
386 	(0x000543e0 + 0x00001000),
387 	(0x000543e0 + 0x00001800),
388 	(0x000943e0 + 0x00000000),
389 	(0x000943e0 + 0x00000800),
390 	(0x000943e0 + 0x00001000),
391 	(0x000943e0 + 0x00001800),
392 	(0x000d43e0 + 0x00000000),
393 	(0x000d43e0 + 0x00000800),
394 	(0x000d43e0 + 0x00001000),
395 	(0x000d43e0 + 0x00001800),
396 	(0x001143e0 + 0x00000000),
397 	(0x001143e0 + 0x00000800),
398 	(0x001143e0 + 0x00001000),
399 	(0x001143e0 + 0x00001800),
400 	(0x001543e0 + 0x00000000),
401 	(0x001543e0 + 0x00000800),
402 	(0x001543e0 + 0x00001000),
403 	(0x001543e0 + 0x00001800),
404 	(0x001943e0 + 0x00000000),
405 	(0x001943e0 + 0x00000800),
406 	(0x001943e0 + 0x00001000),
407 	(0x001943e0 + 0x00001800),
408 	(0x001d43e0 + 0x00000000),
409 	(0x001d43e0 + 0x00000800),
410 	(0x001d43e0 + 0x00001000),
411 	(0x001d43e0 + 0x00001800),
412 };
413 
414 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
415 		struct amdgpu_irq_src *src,
416 		unsigned int type,
417 		enum amdgpu_interrupt_state state)
418 {
419 	u32 bits, i, tmp, reg;
420 
421 	/* Devices newer then VEGA10/12 shall have these programming
422 	 * sequences performed by PSP BL
423 	 */
424 	if (adev->asic_type >= CHIP_VEGA20)
425 		return 0;
426 
427 	bits = 0x7f;
428 
429 	switch (state) {
430 	case AMDGPU_IRQ_STATE_DISABLE:
431 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
432 			reg = ecc_umc_mcumc_ctrl_addrs[i];
433 			tmp = RREG32(reg);
434 			tmp &= ~bits;
435 			WREG32(reg, tmp);
436 		}
437 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
438 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
439 			tmp = RREG32(reg);
440 			tmp &= ~bits;
441 			WREG32(reg, tmp);
442 		}
443 		break;
444 	case AMDGPU_IRQ_STATE_ENABLE:
445 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
446 			reg = ecc_umc_mcumc_ctrl_addrs[i];
447 			tmp = RREG32(reg);
448 			tmp |= bits;
449 			WREG32(reg, tmp);
450 		}
451 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
452 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
453 			tmp = RREG32(reg);
454 			tmp |= bits;
455 			WREG32(reg, tmp);
456 		}
457 		break;
458 	default:
459 		break;
460 	}
461 
462 	return 0;
463 }
464 
465 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
466 					struct amdgpu_irq_src *src,
467 					unsigned int type,
468 					enum amdgpu_interrupt_state state)
469 {
470 	struct amdgpu_vmhub *hub;
471 	u32 tmp, reg, bits, i, j;
472 
473 	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
474 		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
475 		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476 		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477 		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478 		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479 		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
480 
481 	switch (state) {
482 	case AMDGPU_IRQ_STATE_DISABLE:
483 		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
484 			hub = &adev->vmhub[j];
485 			for (i = 0; i < 16; i++) {
486 				reg = hub->vm_context0_cntl + i;
487 
488 				/* This works because this interrupt is only
489 				 * enabled at init/resume and disabled in
490 				 * fini/suspend, so the overall state doesn't
491 				 * change over the course of suspend/resume.
492 				 */
493 				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
494 					continue;
495 
496 				if (j >= AMDGPU_MMHUB0(0))
497 					tmp = RREG32_SOC15_IP(MMHUB, reg);
498 				else
499 					tmp = RREG32_XCC(reg, j);
500 
501 				tmp &= ~bits;
502 
503 				if (j >= AMDGPU_MMHUB0(0))
504 					WREG32_SOC15_IP(MMHUB, reg, tmp);
505 				else
506 					WREG32_XCC(reg, tmp, j);
507 			}
508 		}
509 		break;
510 	case AMDGPU_IRQ_STATE_ENABLE:
511 		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
512 			hub = &adev->vmhub[j];
513 			for (i = 0; i < 16; i++) {
514 				reg = hub->vm_context0_cntl + i;
515 
516 				/* This works because this interrupt is only
517 				 * enabled at init/resume and disabled in
518 				 * fini/suspend, so the overall state doesn't
519 				 * change over the course of suspend/resume.
520 				 */
521 				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
522 					continue;
523 
524 				if (j >= AMDGPU_MMHUB0(0))
525 					tmp = RREG32_SOC15_IP(MMHUB, reg);
526 				else
527 					tmp = RREG32_XCC(reg, j);
528 
529 				tmp |= bits;
530 
531 				if (j >= AMDGPU_MMHUB0(0))
532 					WREG32_SOC15_IP(MMHUB, reg, tmp);
533 				else
534 					WREG32_XCC(reg, tmp, j);
535 			}
536 		}
537 		break;
538 	default:
539 		break;
540 	}
541 
542 	return 0;
543 }
544 
545 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
546 				      struct amdgpu_irq_src *source,
547 				      struct amdgpu_iv_entry *entry)
548 {
549 	bool retry_fault = !!(entry->src_data[1] &
550 			      AMDGPU_GMC9_FAULT_SOURCE_DATA_RETRY);
551 	bool write_fault = !!(entry->src_data[1] &
552 			      AMDGPU_GMC9_FAULT_SOURCE_DATA_WRITE);
553 	uint32_t status = 0, cid = 0, rw = 0, fed = 0;
554 	struct amdgpu_task_info *task_info;
555 	struct amdgpu_vmhub *hub;
556 	const char *mmhub_cid;
557 	const char *hub_name;
558 	unsigned int vmhub;
559 	u64 addr;
560 	uint32_t cam_index = 0;
561 	int ret, xcc_id = 0;
562 	uint32_t node_id;
563 
564 	node_id = entry->node_id;
565 
566 	addr = (u64)entry->src_data[0] << 12;
567 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
568 
569 	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
570 		hub_name = "mmhub0";
571 		vmhub = AMDGPU_MMHUB0(node_id / 4);
572 	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
573 		hub_name = "mmhub1";
574 		vmhub = AMDGPU_MMHUB1(0);
575 	} else {
576 		hub_name = "gfxhub0";
577 		if (adev->gfx.funcs->ih_node_to_logical_xcc) {
578 			xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
579 				node_id);
580 			if (xcc_id < 0)
581 				xcc_id = 0;
582 		}
583 		vmhub = xcc_id;
584 	}
585 	hub = &adev->vmhub[vmhub];
586 
587 	if (retry_fault) {
588 		cam_index = entry->src_data[2] & 0x3ff;
589 
590 		ret = amdgpu_gmc_handle_retry_fault(adev, entry, addr, cam_index, node_id,
591 						    write_fault);
592 		/* Returning 1 here also prevents sending the IV to the KFD */
593 		if (ret == 1)
594 			return 1;
595 	}
596 
597 	if (kgd2kfd_vmfault_fast_path(adev, entry, retry_fault))
598 		return 1;
599 
600 	if (!printk_ratelimit())
601 		return 0;
602 
603 	dev_err(adev->dev,
604 		"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n", hub_name,
605 		retry_fault ? "retry" : "no-retry",
606 		entry->src_id, entry->ring_id, entry->vmid, entry->pasid);
607 
608 	task_info = amdgpu_vm_get_task_info_pasid(adev, entry->pasid);
609 	if (task_info) {
610 		amdgpu_vm_print_task_info(adev, task_info);
611 		amdgpu_vm_put_task_info(task_info);
612 	}
613 
614 	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
615 		addr, entry->client_id,
616 		soc15_ih_clientid_name[entry->client_id]);
617 
618 	if (amdgpu_is_multi_aid(adev))
619 		dev_err(adev->dev, "  cookie node_id %d fault from die %s%d%s\n",
620 			node_id, node_id % 4 == 3 ? "RSV" : "AID", node_id / 4,
621 			node_id % 4 == 1 ? ".XCD0" : node_id % 4 == 2 ? ".XCD1" : "");
622 
623 	if (amdgpu_sriov_vf(adev))
624 		return 0;
625 
626 	/*
627 	 * Issue a dummy read to wait for the status register to
628 	 * be updated to avoid reading an incorrect value due to
629 	 * the new fast GRBM interface.
630 	 */
631 	if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&
632 	    (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 4, 2)))
633 		RREG32(hub->vm_l2_pro_fault_status);
634 
635 	status = RREG32(hub->vm_l2_pro_fault_status);
636 	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
637 	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
638 	fed = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, FED);
639 
640 	/* for fed error, kfd will handle it, return directly */
641 	if (fed && amdgpu_ras_is_poison_mode_supported(adev) &&
642 	    (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2)))
643 		return 0;
644 
645 	/* Only print L2 fault status if the status register could be read and
646 	 * contains useful information
647 	 */
648 	if (!status)
649 		return 0;
650 
651 	if (!amdgpu_sriov_vf(adev))
652 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
653 
654 	amdgpu_vm_update_fault_cache(adev, entry->pasid, addr, status, vmhub);
655 
656 	dev_err(adev->dev,
657 		"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
658 		status);
659 	if (entry->vmid_src == AMDGPU_GFXHUB(0)) {
660 		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
661 			cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
662 			gfxhub_client_ids[cid],
663 			cid);
664 	} else {
665 		mmhub_cid = amdgpu_mmhub_client_name(&adev->mmhub, cid, rw);
666 		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
667 			mmhub_cid ? mmhub_cid : "unknown", cid);
668 	}
669 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
670 		REG_GET_FIELD(status,
671 		VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
672 	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
673 		REG_GET_FIELD(status,
674 		VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
675 	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
676 		REG_GET_FIELD(status,
677 		VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
678 	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
679 		REG_GET_FIELD(status,
680 		VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
681 	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
682 	return 0;
683 }
684 
685 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
686 	.set = gmc_v9_0_vm_fault_interrupt_state,
687 	.process = gmc_v9_0_process_interrupt,
688 };
689 
690 
691 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
692 	.set = gmc_v9_0_ecc_interrupt_state,
693 	.process = amdgpu_umc_process_ecc_irq,
694 };
695 
696 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
697 {
698 	adev->gmc.vm_fault.num_types = 1;
699 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
700 
701 	if (!amdgpu_sriov_vf(adev) &&
702 	    !adev->gmc.xgmi.connected_to_cpu &&
703 	    !adev->gmc.is_app_apu) {
704 		adev->gmc.ecc_irq.num_types = 1;
705 		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
706 	}
707 }
708 
709 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
710 					uint32_t flush_type)
711 {
712 	u32 req = 0;
713 
714 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
715 			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
716 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
717 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
718 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
719 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
720 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
721 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
722 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
723 			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
724 
725 	return req;
726 }
727 
728 /**
729  * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
730  *
731  * @adev: amdgpu_device pointer
732  * @vmhub: vmhub type
733  *
734  */
735 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
736 				       uint32_t vmhub)
737 {
738 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
739 	    amdgpu_is_multi_aid(adev))
740 		return false;
741 
742 	return ((vmhub == AMDGPU_MMHUB0(0) ||
743 		 vmhub == AMDGPU_MMHUB1(0)) &&
744 		(!amdgpu_sriov_vf(adev)) &&
745 		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
746 		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
747 }
748 
749 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
750 					uint8_t vmid, uint16_t *p_pasid)
751 {
752 	uint32_t value;
753 
754 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
755 		     + vmid);
756 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
757 
758 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
759 }
760 
761 /*
762  * GART
763  * VMID 0 is the physical GPU addresses as used by the kernel.
764  * VMIDs 1-15 are used for userspace clients and are handled
765  * by the amdgpu vm/hsa code.
766  */
767 
768 /**
769  * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
770  *
771  * @adev: amdgpu_device pointer
772  * @vmid: vm instance to flush
773  * @vmhub: which hub to flush
774  * @flush_type: the flush type
775  *
776  * Flush the TLB for the requested page table using certain type.
777  */
778 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
779 					uint32_t vmhub, uint32_t flush_type)
780 {
781 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
782 	u32 j, inv_req, tmp, sem, req, ack, inst;
783 	const unsigned int eng = 17;
784 	struct amdgpu_vmhub *hub;
785 
786 	BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS);
787 
788 	hub = &adev->vmhub[vmhub];
789 	inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
790 	sem = hub->vm_inv_eng0_sem + hub->eng_distance * eng;
791 	req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
792 	ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
793 
794 	if (vmhub >= AMDGPU_MMHUB0(0))
795 		inst = 0;
796 	else
797 		inst = vmhub;
798 
799 	/* This is necessary for SRIOV as well as for GFXOFF to function
800 	 * properly under bare metal
801 	 */
802 	if (adev->gfx.kiq[inst].ring.sched.ready &&
803 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev))) {
804 		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
805 		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
806 
807 		amdgpu_gmc_fw_reg_write_reg_wait(adev, req, ack, inv_req,
808 						 1 << vmid, inst);
809 		return;
810 	}
811 
812 	/* This path is needed before KIQ/MES/GFXOFF are set up */
813 	spin_lock(&adev->gmc.invalidate_lock);
814 
815 	/*
816 	 * It may lose gpuvm invalidate acknowldege state across power-gating
817 	 * off cycle, add semaphore acquire before invalidation and semaphore
818 	 * release after invalidation to avoid entering power gated state
819 	 * to WA the Issue
820 	 */
821 
822 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
823 	if (use_semaphore) {
824 		for (j = 0; j < adev->usec_timeout; j++) {
825 			/* a read return value of 1 means semaphore acquire */
826 			if (vmhub >= AMDGPU_MMHUB0(0))
827 				tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, sem, GET_INST(GC, inst));
828 			else
829 				tmp = RREG32_SOC15_IP_NO_KIQ(GC, sem, GET_INST(GC, inst));
830 			if (tmp & 0x1)
831 				break;
832 			udelay(1);
833 		}
834 
835 		if (j >= adev->usec_timeout)
836 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
837 	}
838 
839 	if (vmhub >= AMDGPU_MMHUB0(0))
840 		WREG32_SOC15_IP_NO_KIQ(MMHUB, req, inv_req, GET_INST(GC, inst));
841 	else
842 		WREG32_SOC15_IP_NO_KIQ(GC, req, inv_req, GET_INST(GC, inst));
843 
844 	/*
845 	 * Issue a dummy read to wait for the ACK register to
846 	 * be cleared to avoid a false ACK due to the new fast
847 	 * GRBM interface.
848 	 */
849 	if ((vmhub == AMDGPU_GFXHUB(0)) &&
850 	    (amdgpu_ip_version(adev, GC_HWIP, 0) < IP_VERSION(9, 4, 2)))
851 		RREG32_NO_KIQ(req);
852 
853 	for (j = 0; j < adev->usec_timeout; j++) {
854 		if (vmhub >= AMDGPU_MMHUB0(0))
855 			tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, ack, GET_INST(GC, inst));
856 		else
857 			tmp = RREG32_SOC15_IP_NO_KIQ(GC, ack, GET_INST(GC, inst));
858 		if (tmp & (1 << vmid))
859 			break;
860 		udelay(1);
861 	}
862 
863 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
864 	if (use_semaphore) {
865 		/*
866 		 * add semaphore release after invalidation,
867 		 * write with 0 means semaphore release
868 		 */
869 		if (vmhub >= AMDGPU_MMHUB0(0))
870 			WREG32_SOC15_IP_NO_KIQ(MMHUB, sem, 0, GET_INST(GC, inst));
871 		else
872 			WREG32_SOC15_IP_NO_KIQ(GC, sem, 0, GET_INST(GC, inst));
873 	}
874 
875 	spin_unlock(&adev->gmc.invalidate_lock);
876 
877 	if (j < adev->usec_timeout)
878 		return;
879 
880 	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
881 }
882 
883 /**
884  * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
885  *
886  * @adev: amdgpu_device pointer
887  * @pasid: pasid to be flush
888  * @flush_type: the flush type
889  * @all_hub: flush all hubs
890  * @inst: is used to select which instance of KIQ to use for the invalidation
891  *
892  * Flush the TLB for the requested pasid.
893  */
894 static void gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
895 					 uint16_t pasid, uint32_t flush_type,
896 					 bool all_hub, uint32_t inst)
897 {
898 	uint16_t queried;
899 	int i, vmid;
900 
901 	for (vmid = 1; vmid < 16; vmid++) {
902 		bool valid;
903 
904 		valid = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
905 								 &queried);
906 		if (!valid || queried != pasid)
907 			continue;
908 
909 		if (all_hub) {
910 			for_each_set_bit(i, adev->vmhubs_mask,
911 					 AMDGPU_MAX_VMHUBS)
912 				gmc_v9_0_flush_gpu_tlb(adev, vmid, i,
913 						       flush_type);
914 		} else {
915 			gmc_v9_0_flush_gpu_tlb(adev, vmid,
916 					       AMDGPU_GFXHUB(0),
917 					       flush_type);
918 		}
919 	}
920 }
921 
922 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
923 					    unsigned int vmid, uint64_t pd_addr)
924 {
925 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
926 	struct amdgpu_device *adev = ring->adev;
927 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub];
928 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
929 	unsigned int eng = ring->vm_inv_eng;
930 
931 	/*
932 	 * It may lose gpuvm invalidate acknowldege state across power-gating
933 	 * off cycle, add semaphore acquire before invalidation and semaphore
934 	 * release after invalidation to avoid entering power gated state
935 	 * to WA the Issue
936 	 */
937 
938 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
939 	if (use_semaphore)
940 		/* a read return value of 1 means semaphore acuqire */
941 		amdgpu_ring_emit_reg_wait(ring,
942 					  hub->vm_inv_eng0_sem +
943 					  hub->eng_distance * eng, 0x1, 0x1);
944 
945 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
946 			      (hub->ctx_addr_distance * vmid),
947 			      lower_32_bits(pd_addr));
948 
949 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
950 			      (hub->ctx_addr_distance * vmid),
951 			      upper_32_bits(pd_addr));
952 
953 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
954 					    hub->eng_distance * eng,
955 					    hub->vm_inv_eng0_ack +
956 					    hub->eng_distance * eng,
957 					    req, 1 << vmid);
958 
959 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
960 	if (use_semaphore)
961 		/*
962 		 * add semaphore release after invalidation,
963 		 * write with 0 means semaphore release
964 		 */
965 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
966 				      hub->eng_distance * eng, 0);
967 
968 	return pd_addr;
969 }
970 
971 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
972 					unsigned int pasid)
973 {
974 	struct amdgpu_device *adev = ring->adev;
975 	uint32_t reg;
976 
977 	/* Do nothing because there's no lut register for mmhub1. */
978 	if (ring->vm_hub == AMDGPU_MMHUB1(0))
979 		return;
980 
981 	if (ring->vm_hub == AMDGPU_GFXHUB(0))
982 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
983 	else
984 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
985 
986 	amdgpu_ring_emit_wreg(ring, reg, pasid);
987 }
988 
989 /*
990  * PTE format on VEGA 10:
991  * 63:59 reserved
992  * 58:57 mtype
993  * 56 F
994  * 55 L
995  * 54 P
996  * 53 SW
997  * 52 T
998  * 50:48 reserved
999  * 47:12 4k physical page base address
1000  * 11:7 fragment
1001  * 6 write
1002  * 5 read
1003  * 4 exe
1004  * 3 Z
1005  * 2 snooped
1006  * 1 system
1007  * 0 valid
1008  *
1009  * PDE format on VEGA 10:
1010  * 63:59 block fragment size
1011  * 58:55 reserved
1012  * 54 P
1013  * 53:48 reserved
1014  * 47:6 physical base address of PD or PTE
1015  * 5:3 reserved
1016  * 2 C
1017  * 1 system
1018  * 0 valid
1019  */
1020 
1021 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1022 				uint64_t *addr, uint64_t *flags)
1023 {
1024 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1025 		*addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
1026 	BUG_ON(*addr & 0xFFFF00000000003FULL);
1027 
1028 	if (!adev->gmc.translate_further)
1029 		return;
1030 
1031 	if (level == AMDGPU_VM_PDB1) {
1032 		/* Set the block fragment size */
1033 		if (!(*flags & AMDGPU_PDE_PTE))
1034 			*flags |= AMDGPU_PDE_BFS(0x9);
1035 
1036 	} else if (level == AMDGPU_VM_PDB0) {
1037 		if (*flags & AMDGPU_PDE_PTE) {
1038 			*flags &= ~AMDGPU_PDE_PTE;
1039 			if (!(*flags & AMDGPU_PTE_VALID))
1040 				*addr |= 1 << PAGE_SHIFT;
1041 		} else {
1042 			*flags |= AMDGPU_PTE_TF;
1043 		}
1044 	}
1045 }
1046 
1047 static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
1048 					 struct amdgpu_vm *vm,
1049 					 struct amdgpu_bo *bo,
1050 					 uint32_t vm_flags,
1051 					 uint64_t *flags)
1052 {
1053 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1054 	bool is_vram = bo->tbo.resource &&
1055 		bo->tbo.resource->mem_type == TTM_PL_VRAM;
1056 	bool coherent = bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
1057 				     AMDGPU_GEM_CREATE_EXT_COHERENT);
1058 	bool ext_coherent = bo->flags & AMDGPU_GEM_CREATE_EXT_COHERENT;
1059 	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
1060 	unsigned int mtype_local, mtype;
1061 	uint32_t gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
1062 	bool snoop = false;
1063 	bool is_local;
1064 
1065 	dma_resv_assert_held(bo->tbo.base.resv);
1066 
1067 	switch (gc_ip_version) {
1068 	case IP_VERSION(9, 4, 1):
1069 	case IP_VERSION(9, 4, 2):
1070 		if (is_vram) {
1071 			if (bo_adev == adev) {
1072 				if (uncached)
1073 					mtype = MTYPE_UC;
1074 				else if (coherent)
1075 					mtype = MTYPE_CC;
1076 				else
1077 					mtype = MTYPE_RW;
1078 				/* FIXME: is this still needed? Or does
1079 				 * amdgpu_ttm_tt_pde_flags already handle this?
1080 				 */
1081 				if (gc_ip_version == IP_VERSION(9, 4, 2) &&
1082 				    adev->gmc.xgmi.connected_to_cpu)
1083 					snoop = true;
1084 			} else {
1085 				if (uncached || coherent)
1086 					mtype = MTYPE_UC;
1087 				else
1088 					mtype = MTYPE_NC;
1089 				if (amdgpu_xgmi_same_hive(adev, bo_adev))
1090 					snoop = true;
1091 			}
1092 		} else {
1093 			if (uncached || coherent)
1094 				mtype = MTYPE_UC;
1095 			else
1096 				mtype = MTYPE_NC;
1097 			/* FIXME: is this still needed? Or does
1098 			 * amdgpu_ttm_tt_pde_flags already handle this?
1099 			 */
1100 			snoop = true;
1101 		}
1102 		break;
1103 	case IP_VERSION(9, 4, 3):
1104 	case IP_VERSION(9, 4, 4):
1105 	case IP_VERSION(9, 5, 0):
1106 		/* Only local VRAM BOs or system memory on non-NUMA APUs
1107 		 * can be assumed to be local in their entirety. Choose
1108 		 * MTYPE_NC as safe fallback for all system memory BOs on
1109 		 * NUMA systems. Their MTYPE can be overridden per-page in
1110 		 * gmc_v9_0_override_vm_pte_flags.
1111 		 */
1112 		mtype_local = MTYPE_RW;
1113 		if (amdgpu_mtype_local == 1) {
1114 			drm_info_once(adev_to_drm(adev), "Using MTYPE_NC for local memory\n");
1115 			mtype_local = MTYPE_NC;
1116 		} else if (amdgpu_mtype_local == 2) {
1117 			drm_info_once(adev_to_drm(adev), "Using MTYPE_CC for local memory\n");
1118 			mtype_local = MTYPE_CC;
1119 		} else {
1120 			drm_info_once(adev_to_drm(adev), "Using MTYPE_RW for local memory\n");
1121 		}
1122 		is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
1123 			    num_possible_nodes() <= 1) ||
1124 			   (is_vram && adev == bo_adev &&
1125 			    KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id);
1126 		snoop = true;
1127 		if (uncached) {
1128 			mtype = MTYPE_UC;
1129 		} else if (ext_coherent) {
1130 			mtype = is_local ? MTYPE_CC : MTYPE_UC;
1131 		} else if (adev->flags & AMD_IS_APU) {
1132 			mtype = is_local ? mtype_local : MTYPE_NC;
1133 		} else {
1134 			/* dGPU */
1135 			if (is_local)
1136 				mtype = mtype_local;
1137 			else if (gc_ip_version < IP_VERSION(9, 5, 0) && !is_vram)
1138 				mtype = MTYPE_UC;
1139 			else
1140 				mtype = MTYPE_NC;
1141 		}
1142 
1143 		break;
1144 	default:
1145 		if (uncached || coherent)
1146 			mtype = MTYPE_UC;
1147 		else
1148 			mtype = MTYPE_NC;
1149 
1150 		/* FIXME: is this still needed? Or does
1151 		 * amdgpu_ttm_tt_pde_flags already handle this?
1152 		 */
1153 		if (!is_vram)
1154 			snoop = true;
1155 	}
1156 
1157 	if (mtype != MTYPE_NC)
1158 		*flags = AMDGPU_PTE_MTYPE_VG10(*flags, mtype);
1159 
1160 	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1161 }
1162 
1163 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1164 				struct amdgpu_vm *vm,
1165 				struct amdgpu_bo *bo,
1166 				uint32_t vm_flags,
1167 				uint64_t *flags)
1168 {
1169 	if (vm_flags & AMDGPU_VM_PAGE_EXECUTABLE)
1170 		*flags |= AMDGPU_PTE_EXECUTABLE;
1171 	else
1172 		*flags &= ~AMDGPU_PTE_EXECUTABLE;
1173 
1174 	switch (vm_flags & AMDGPU_VM_MTYPE_MASK) {
1175 	case AMDGPU_VM_MTYPE_DEFAULT:
1176 	case AMDGPU_VM_MTYPE_NC:
1177 	default:
1178 		*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_NC);
1179 		break;
1180 	case AMDGPU_VM_MTYPE_WC:
1181 		*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_WC);
1182 		break;
1183 	case AMDGPU_VM_MTYPE_RW:
1184 		*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_RW);
1185 		break;
1186 	case AMDGPU_VM_MTYPE_CC:
1187 		*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_CC);
1188 		break;
1189 	case AMDGPU_VM_MTYPE_UC:
1190 		*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_UC);
1191 		break;
1192 	}
1193 
1194 	if (vm_flags & AMDGPU_VM_PAGE_PRT) {
1195 		*flags |= AMDGPU_PTE_PRT;
1196 		*flags &= ~AMDGPU_PTE_VALID;
1197 	}
1198 
1199 	if ((*flags & AMDGPU_PTE_VALID) && bo)
1200 		gmc_v9_0_get_coherence_flags(adev, vm, bo, vm_flags, flags);
1201 }
1202 
1203 static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
1204 					   struct amdgpu_vm *vm,
1205 					   uint64_t addr, uint64_t *flags)
1206 {
1207 	int local_node, nid;
1208 
1209 	/* MTYPE_NC is the same default and can be overridden.
1210 	 * MTYPE_UC will be present if the memory is extended-coherent
1211 	 * and can also be overridden.
1212 	 */
1213 	if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1214 	    AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_NC) &&
1215 	    (*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1216 	    AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_UC)) {
1217 		dev_dbg_ratelimited(adev->dev, "MTYPE is not NC or UC\n");
1218 		return;
1219 	}
1220 
1221 	if (vm->mem_id >= 0) {
1222 		local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
1223 	} else {
1224 		dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n");
1225 		return;
1226 	}
1227 
1228 	/* Only handle real RAM. Mappings of PCIe resources don't have struct
1229 	 * page or NUMA nodes.
1230 	 */
1231 	if (!page_is_ram(addr >> PAGE_SHIFT)) {
1232 		dev_dbg_ratelimited(adev->dev, "Page is not RAM.\n");
1233 		return;
1234 	}
1235 	nid = pfn_to_nid(addr >> PAGE_SHIFT);
1236 	dev_dbg_ratelimited(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n",
1237 			    vm->mem_id, local_node, nid);
1238 	if (nid == local_node) {
1239 		uint64_t old_flags = *flags;
1240 		if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) ==
1241 			AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_NC)) {
1242 			unsigned int mtype_local = MTYPE_RW;
1243 
1244 			if (amdgpu_mtype_local == 1)
1245 				mtype_local = MTYPE_NC;
1246 			else if (amdgpu_mtype_local == 2)
1247 				mtype_local = MTYPE_CC;
1248 
1249 			*flags = AMDGPU_PTE_MTYPE_VG10(*flags, mtype_local);
1250 		} else {
1251 			/* MTYPE_UC case */
1252 			*flags = AMDGPU_PTE_MTYPE_VG10(*flags, MTYPE_CC);
1253 		}
1254 
1255 		dev_dbg_ratelimited(adev->dev, "flags updated from %llx to %llx\n",
1256 				    old_flags, *flags);
1257 	}
1258 }
1259 
1260 static unsigned int gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1261 {
1262 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1263 	unsigned int size;
1264 
1265 	/* TODO move to DC so GMC doesn't need to hard-code DCN registers */
1266 
1267 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1268 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1269 	} else {
1270 		u32 viewport;
1271 
1272 		switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) {
1273 		case IP_VERSION(1, 0, 0):
1274 		case IP_VERSION(1, 0, 1):
1275 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1276 			size = (REG_GET_FIELD(viewport,
1277 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1278 				REG_GET_FIELD(viewport,
1279 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1280 				4);
1281 			break;
1282 		case IP_VERSION(2, 1, 0):
1283 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
1284 			size = (REG_GET_FIELD(viewport,
1285 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1286 				REG_GET_FIELD(viewport,
1287 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1288 				4);
1289 			break;
1290 		default:
1291 			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1292 			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1293 				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1294 				4);
1295 			break;
1296 		}
1297 	}
1298 
1299 	return size;
1300 }
1301 
1302 static bool gmc_v9_0_need_reset_on_init(struct amdgpu_device *adev)
1303 {
1304 	if (adev->nbio.funcs && adev->nbio.funcs->is_nps_switch_requested &&
1305 	    adev->nbio.funcs->is_nps_switch_requested(adev)) {
1306 		adev->gmc.reset_flags |= AMDGPU_GMC_INIT_RESET_NPS;
1307 		return true;
1308 	}
1309 
1310 	return false;
1311 }
1312 
1313 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1314 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1315 	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1316 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1317 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1318 	.get_vm_pde = gmc_v9_0_get_vm_pde,
1319 	.get_vm_pte = gmc_v9_0_get_vm_pte,
1320 	.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
1321 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1322 	.query_mem_partition_mode = &amdgpu_gmc_query_memory_partition,
1323 	.request_mem_partition_mode = &amdgpu_gmc_request_memory_partition,
1324 	.need_reset_on_init = &gmc_v9_0_need_reset_on_init,
1325 };
1326 
1327 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1328 {
1329 	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1330 
1331 	/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes, local system
1332 	 * memory can use more efficient MTYPEs.
1333 	 *
1334 	 * APUs mapping system memory may need different MTYPEs on different
1335 	 * NUMA nodes.
1336 	 *
1337 	 * Only direct-mapped memory allows us to determine the NUMA node from
1338 	 * the DMA address.
1339 	 */
1340 	adev->gmc.override_pte = adev->gmc.is_app_apu &&
1341 				 num_possible_nodes() > 1 &&
1342 				 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
1343 				 adev->ram_is_direct_mapped;
1344 }
1345 
1346 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1347 {
1348 	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
1349 	case IP_VERSION(6, 0, 0):
1350 		adev->umc.funcs = &umc_v6_0_funcs;
1351 		break;
1352 	case IP_VERSION(6, 1, 1):
1353 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1354 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1355 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1356 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1357 		adev->umc.retire_unit = 1;
1358 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1359 		adev->umc.ras = &umc_v6_1_ras;
1360 		break;
1361 	case IP_VERSION(6, 1, 2):
1362 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1363 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1364 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1365 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1366 		adev->umc.retire_unit = 1;
1367 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1368 		adev->umc.ras = &umc_v6_1_ras;
1369 		break;
1370 	case IP_VERSION(6, 7, 0):
1371 		adev->umc.max_ras_err_cnt_per_query =
1372 			UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL;
1373 		adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
1374 		adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
1375 		adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
1376 		adev->umc.retire_unit = (UMC_V6_7_NA_MAP_PA_NUM * 2);
1377 		if (!adev->gmc.xgmi.connected_to_cpu)
1378 			adev->umc.ras = &umc_v6_7_ras;
1379 		if (1 & adev->smuio.funcs->get_die_id(adev))
1380 			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
1381 		else
1382 			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1383 		break;
1384 	case IP_VERSION(12, 0, 0):
1385 	case IP_VERSION(12, 5, 0):
1386 		adev->umc.max_ras_err_cnt_per_query =
1387 			UMC_V12_0_TOTAL_CHANNEL_NUM * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL;
1388 		adev->umc.channel_inst_num = UMC_V12_0_CHANNEL_INSTANCE_NUM;
1389 		adev->umc.umc_inst_num = UMC_V12_0_UMC_INSTANCE_NUM;
1390 		adev->umc.node_inst_num /= UMC_V12_0_UMC_INSTANCE_NUM;
1391 		adev->umc.channel_offs = UMC_V12_0_PER_CHANNEL_OFFSET;
1392 		if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
1393 			adev->umc.ras = &umc_v12_0_ras;
1394 		break;
1395 	default:
1396 		break;
1397 	}
1398 }
1399 
1400 static void gmc_v9_0_init_mmhub_client_info(struct amdgpu_device *adev)
1401 {
1402 	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
1403 	case IP_VERSION(9, 0, 0):
1404 		amdgpu_mmhub_init_client_info(&adev->mmhub,
1405 					     mmhub_client_ids_vega10,
1406 					     ARRAY_SIZE(mmhub_client_ids_vega10));
1407 		break;
1408 	case IP_VERSION(9, 3, 0):
1409 		amdgpu_mmhub_init_client_info(&adev->mmhub,
1410 					     mmhub_client_ids_vega12,
1411 					     ARRAY_SIZE(mmhub_client_ids_vega12));
1412 		break;
1413 	case IP_VERSION(9, 4, 0):
1414 		amdgpu_mmhub_init_client_info(&adev->mmhub,
1415 					     mmhub_client_ids_vega20,
1416 					     ARRAY_SIZE(mmhub_client_ids_vega20));
1417 		break;
1418 	case IP_VERSION(9, 4, 1):
1419 		amdgpu_mmhub_init_client_info(&adev->mmhub,
1420 					     mmhub_client_ids_arcturus,
1421 					     ARRAY_SIZE(mmhub_client_ids_arcturus));
1422 		break;
1423 	case IP_VERSION(9, 1, 0):
1424 	case IP_VERSION(9, 2, 0):
1425 		amdgpu_mmhub_init_client_info(&adev->mmhub,
1426 					     mmhub_client_ids_raven,
1427 					     ARRAY_SIZE(mmhub_client_ids_raven));
1428 		break;
1429 	case IP_VERSION(1, 5, 0):
1430 	case IP_VERSION(2, 4, 0):
1431 		amdgpu_mmhub_init_client_info(&adev->mmhub,
1432 					     mmhub_client_ids_renoir,
1433 					     ARRAY_SIZE(mmhub_client_ids_renoir));
1434 		break;
1435 	case IP_VERSION(1, 8, 0):
1436 	case IP_VERSION(9, 4, 2):
1437 		amdgpu_mmhub_init_client_info(&adev->mmhub,
1438 					     mmhub_client_ids_aldebaran,
1439 					     ARRAY_SIZE(mmhub_client_ids_aldebaran));
1440 		break;
1441 	default:
1442 		break;
1443 	}
1444 }
1445 
1446 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1447 {
1448 	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
1449 	case IP_VERSION(9, 4, 1):
1450 		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1451 		break;
1452 	case IP_VERSION(9, 4, 2):
1453 		adev->mmhub.funcs = &mmhub_v1_7_funcs;
1454 		break;
1455 	case IP_VERSION(1, 8, 0):
1456 	case IP_VERSION(1, 8, 1):
1457 		adev->mmhub.funcs = &mmhub_v1_8_funcs;
1458 		break;
1459 	default:
1460 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1461 		break;
1462 	}
1463 
1464 	gmc_v9_0_init_mmhub_client_info(adev);
1465 }
1466 
1467 static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1468 {
1469 	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
1470 	case IP_VERSION(9, 4, 0):
1471 		adev->mmhub.ras = &mmhub_v1_0_ras;
1472 		break;
1473 	case IP_VERSION(9, 4, 1):
1474 		adev->mmhub.ras = &mmhub_v9_4_ras;
1475 		break;
1476 	case IP_VERSION(9, 4, 2):
1477 		adev->mmhub.ras = &mmhub_v1_7_ras;
1478 		break;
1479 	case IP_VERSION(1, 8, 0):
1480 	case IP_VERSION(1, 8, 1):
1481 		adev->mmhub.ras = &mmhub_v1_8_ras;
1482 		break;
1483 	default:
1484 		/* mmhub ras is not available */
1485 		break;
1486 	}
1487 }
1488 
1489 static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1490 {
1491 	if (amdgpu_is_multi_aid(adev))
1492 		adev->gfxhub.funcs = &gfxhub_v1_2_funcs;
1493 	else
1494 		adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1495 }
1496 
1497 static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
1498 {
1499 	adev->hdp.ras = &hdp_v4_0_ras;
1500 }
1501 
1502 static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev)
1503 {
1504 	struct amdgpu_mca *mca = &adev->mca;
1505 
1506 	/* is UMC the right IP to check for MCA?  Maybe DF? */
1507 	switch (amdgpu_ip_version(adev, UMC_HWIP, 0)) {
1508 	case IP_VERSION(6, 7, 0):
1509 		if (!adev->gmc.xgmi.connected_to_cpu) {
1510 			mca->mp0.ras = &mca_v3_0_mp0_ras;
1511 			mca->mp1.ras = &mca_v3_0_mp1_ras;
1512 			mca->mpio.ras = &mca_v3_0_mpio_ras;
1513 		}
1514 		break;
1515 	default:
1516 		break;
1517 	}
1518 }
1519 
1520 static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
1521 {
1522 	if (!adev->gmc.xgmi.connected_to_cpu)
1523 		adev->gmc.xgmi.ras = &xgmi_ras;
1524 }
1525 
1526 static void gmc_v9_0_init_nps_details(struct amdgpu_device *adev)
1527 {
1528 	enum amdgpu_memory_partition mode;
1529 	uint32_t supp_modes;
1530 	int i;
1531 
1532 	adev->gmc.supported_nps_modes = 0;
1533 
1534 	if (amdgpu_sriov_vf(adev) || (adev->flags & AMD_IS_APU))
1535 		return;
1536 
1537 	mode = amdgpu_gmc_get_memory_partition(adev, &supp_modes);
1538 
1539 	/* Mode detected by hardware and supported modes available */
1540 	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && supp_modes) {
1541 		while ((i = ffs(supp_modes))) {
1542 			if (AMDGPU_ALL_NPS_MASK & BIT(i))
1543 				adev->gmc.supported_nps_modes |= BIT(i);
1544 			supp_modes &= supp_modes - 1;
1545 		}
1546 	} else {
1547 		/*TODO: Check PSP version also which supports NPS switch. Otherwise keep
1548 	 * supported modes as 0.
1549 	 */
1550 		switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1551 		case IP_VERSION(9, 4, 3):
1552 		case IP_VERSION(9, 4, 4):
1553 			adev->gmc.supported_nps_modes =
1554 				BIT(AMDGPU_NPS1_PARTITION_MODE) |
1555 				BIT(AMDGPU_NPS4_PARTITION_MODE);
1556 			break;
1557 		default:
1558 			break;
1559 		}
1560 	}
1561 }
1562 
1563 static int gmc_v9_0_early_init(struct amdgpu_ip_block *ip_block)
1564 {
1565 	struct amdgpu_device *adev = ip_block->adev;
1566 
1567 	/*
1568 	 * 9.4.0, 9.4.1 and 9.4.3 don't have XGMI defined
1569 	 * in their IP discovery tables
1570 	 */
1571 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0) ||
1572 	    amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
1573 	    amdgpu_is_multi_aid(adev))
1574 		adev->gmc.xgmi.supported = true;
1575 
1576 	if (amdgpu_ip_version(adev, XGMI_HWIP, 0) == IP_VERSION(6, 1, 0)) {
1577 		adev->gmc.xgmi.supported = true;
1578 		adev->gmc.xgmi.connected_to_cpu =
1579 			adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
1580 	}
1581 
1582 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3)) {
1583 		enum amdgpu_pkg_type pkg_type =
1584 			adev->smuio.funcs->get_pkg_type(adev);
1585 		/* On GFXIP 9.4.3. APU, there is no physical VRAM domain present
1586 		 * and the APU, can be in used two possible modes:
1587 		 *  - carveout mode
1588 		 *  - native APU mode
1589 		 * "is_app_apu" can be used to identify the APU in the native
1590 		 * mode.
1591 		 */
1592 		adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU &&
1593 					!pci_resource_len(adev->pdev, 0));
1594 	}
1595 
1596 	gmc_v9_0_set_gmc_funcs(adev);
1597 	gmc_v9_0_set_irq_funcs(adev);
1598 	gmc_v9_0_set_umc_funcs(adev);
1599 	gmc_v9_0_set_mmhub_funcs(adev);
1600 	gmc_v9_0_set_mmhub_ras_funcs(adev);
1601 	gmc_v9_0_set_gfxhub_funcs(adev);
1602 	gmc_v9_0_set_hdp_ras_funcs(adev);
1603 	gmc_v9_0_set_mca_ras_funcs(adev);
1604 	gmc_v9_0_set_xgmi_ras_funcs(adev);
1605 
1606 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1607 	adev->gmc.shared_aperture_end =
1608 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1609 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1610 	adev->gmc.private_aperture_end =
1611 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1612 	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
1613 
1614 	return 0;
1615 }
1616 
1617 static int gmc_v9_0_late_init(struct amdgpu_ip_block *ip_block)
1618 {
1619 	struct amdgpu_device *adev = ip_block->adev;
1620 	int r;
1621 
1622 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1623 	if (r)
1624 		return r;
1625 
1626 	/*
1627 	 * Workaround performance drop issue with VBIOS enables partial
1628 	 * writes, while disables HBM ECC for vega10.
1629 	 */
1630 	if (!amdgpu_sriov_vf(adev) &&
1631 	    (amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(6, 0, 0))) {
1632 		if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1633 			if (adev->df.funcs &&
1634 			    adev->df.funcs->enable_ecc_force_par_wr_rmw)
1635 				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1636 		}
1637 	}
1638 
1639 	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1640 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
1641 		amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__HDP);
1642 	}
1643 
1644 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1645 }
1646 
1647 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1648 					struct amdgpu_gmc *mc)
1649 {
1650 	u64 base = adev->mmhub.funcs->get_fb_location(adev);
1651 
1652 	amdgpu_gmc_set_agp_default(adev, mc);
1653 
1654 	/* add the xgmi offset of the physical node */
1655 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1656 	if (amdgpu_gmc_is_pdb0_enabled(adev)) {
1657 		amdgpu_gmc_sysvm_location(adev, mc);
1658 	} else {
1659 		amdgpu_gmc_vram_location(adev, mc, base);
1660 		amdgpu_gmc_gart_location(adev, mc, AMDGPU_GART_PLACEMENT_BEST_FIT);
1661 		if (!amdgpu_sriov_vf(adev) && (amdgpu_agp == 1))
1662 			amdgpu_gmc_agp_location(adev, mc);
1663 	}
1664 	/* base offset of vram pages */
1665 	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1666 
1667 	/* XXX: add the xgmi offset of the physical node? */
1668 	adev->vm_manager.vram_base_offset +=
1669 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1670 }
1671 
1672 /**
1673  * gmc_v9_0_mc_init - initialize the memory controller driver params
1674  *
1675  * @adev: amdgpu_device pointer
1676  *
1677  * Look up the amount of vram, vram width, and decide how to place
1678  * vram and gart within the GPU's physical address space.
1679  * Returns 0 for success.
1680  */
1681 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1682 {
1683 	int r;
1684 
1685 	/* size in MB on si */
1686 	if (!adev->gmc.is_app_apu) {
1687 		adev->gmc.mc_vram_size =
1688 			adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1689 	} else {
1690 		DRM_DEBUG("Set mc_vram_size = 0 for APP APU\n");
1691 		adev->gmc.mc_vram_size = 0;
1692 	}
1693 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1694 
1695 	if (!(adev->flags & AMD_IS_APU) &&
1696 	    !adev->gmc.xgmi.connected_to_cpu) {
1697 		r = amdgpu_device_resize_fb_bar(adev);
1698 		if (r)
1699 			return r;
1700 	}
1701 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1702 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1703 
1704 #ifdef CONFIG_X86_64
1705 	/*
1706 	 * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi
1707 	 * interface can use VRAM through here as it appears system reserved
1708 	 * memory in host address space.
1709 	 *
1710 	 * For APUs, VRAM is just the stolen system memory and can be accessed
1711 	 * directly.
1712 	 *
1713 	 * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR.
1714 	 */
1715 
1716 	/* check whether both host-gpu and gpu-gpu xgmi links exist */
1717 	if ((!amdgpu_sriov_vf(adev) &&
1718 		(adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
1719 	    (adev->gmc.xgmi.supported &&
1720 	     adev->gmc.xgmi.connected_to_cpu)) {
1721 		adev->gmc.aper_base =
1722 			adev->gfxhub.funcs->get_mc_fb_offset(adev) +
1723 			adev->gmc.xgmi.physical_node_id *
1724 			adev->gmc.xgmi.node_segment_size;
1725 		adev->gmc.aper_size = adev->gmc.real_vram_size;
1726 	}
1727 
1728 #endif
1729 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1730 
1731 	/* set the gart size */
1732 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1733 	case IP_VERSION(9, 1, 0):   /* DCE SG support */
1734 	case IP_VERSION(9, 2, 2):   /* DCE SG support */
1735 	case IP_VERSION(9, 3, 0):
1736 		amdgpu_gmc_set_gart_size(adev, SZ_1G);
1737 		break;
1738 	case IP_VERSION(9, 0, 1):  /* all engines support GPUVM */
1739 	case IP_VERSION(9, 2, 1):  /* all engines support GPUVM */
1740 	case IP_VERSION(9, 4, 0):
1741 	case IP_VERSION(9, 4, 1):
1742 	case IP_VERSION(9, 4, 2):
1743 	case IP_VERSION(9, 4, 3):
1744 	case IP_VERSION(9, 4, 4):
1745 	case IP_VERSION(9, 5, 0):
1746 	default:
1747 		amdgpu_gmc_set_gart_size(adev, SZ_512M);
1748 		break;
1749 	}
1750 
1751 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1752 
1753 	return 0;
1754 }
1755 
1756 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1757 {
1758 	int r;
1759 
1760 	if (adev->gart.bo) {
1761 		WARN(1, "VEGA10 PCIE GART already initialized\n");
1762 		return 0;
1763 	}
1764 
1765 	if (amdgpu_gmc_is_pdb0_enabled(adev)) {
1766 		adev->gmc.vmid0_page_table_depth = 1;
1767 		adev->gmc.vmid0_page_table_block_size = 12;
1768 	} else {
1769 		adev->gmc.vmid0_page_table_depth = 0;
1770 		adev->gmc.vmid0_page_table_block_size = 0;
1771 	}
1772 
1773 	/* Initialize common gart structure */
1774 	r = amdgpu_gart_init(adev);
1775 	if (r)
1776 		return r;
1777 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1778 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(0ULL, MTYPE_UC) |
1779 				 AMDGPU_PTE_EXECUTABLE;
1780 
1781 	if (!adev->gmc.real_vram_size) {
1782 		dev_info(adev->dev, "Put GART in system memory for APU\n");
1783 		r = amdgpu_gart_table_ram_alloc(adev);
1784 		if (r)
1785 			dev_err(adev->dev, "Failed to allocate GART in system memory\n");
1786 	} else {
1787 		r = amdgpu_gart_table_vram_alloc(adev);
1788 		if (r)
1789 			return r;
1790 
1791 		if (amdgpu_gmc_is_pdb0_enabled(adev))
1792 			r = amdgpu_gmc_pdb0_alloc(adev);
1793 	}
1794 
1795 	return r;
1796 }
1797 
1798 /**
1799  * gmc_v9_0_save_registers - saves regs
1800  *
1801  * @adev: amdgpu_device pointer
1802  *
1803  * This saves potential register values that should be
1804  * restored upon resume
1805  */
1806 static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1807 {
1808 	if ((amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 0)) ||
1809 	    (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 1)))
1810 		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1811 }
1812 
1813 static void gmc_v9_0_init_vram_info(struct amdgpu_device *adev)
1814 {
1815 	static const u32 regBIF_BIOS_SCRATCH_4 = 0x50;
1816 	int dev_var = adev->pdev->device & 0xF;
1817 	u32 vram_info;
1818 
1819 	if (adev->gmc.is_app_apu) {
1820 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1821 		adev->gmc.vram_width = 128 * 64;
1822 	} else if (adev->flags & AMD_IS_APU) {
1823 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;
1824 		adev->gmc.vram_width = 64 * 64;
1825 	} else if (amdgpu_is_multi_aid(adev)) {
1826 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
1827 		adev->gmc.vram_width = 128 * 64;
1828 
1829 		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0))
1830 			adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM3E;
1831 
1832 		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4) &&
1833 		    adev->rev_id == 0x3)
1834 			adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM3E;
1835 
1836 		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
1837 		    (dev_var == 0x5))
1838 			adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM3E;
1839 
1840 		if (!(adev->flags & AMD_IS_APU) && !amdgpu_sriov_vf(adev)) {
1841 			vram_info = RREG32(regBIF_BIOS_SCRATCH_4);
1842 			adev->gmc.vram_vendor = vram_info & 0xF;
1843 		}
1844 	}
1845 }
1846 
1847 static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block)
1848 {
1849 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
1850 	struct amdgpu_device *adev = ip_block->adev;
1851 	unsigned long inst_mask = adev->aid_mask;
1852 
1853 	adev->gfxhub.funcs->init(adev);
1854 
1855 	adev->mmhub.funcs->init(adev);
1856 
1857 	spin_lock_init(&adev->gmc.invalidate_lock);
1858 
1859 	if (!adev->bios) {
1860 		gmc_v9_0_init_vram_info(adev);
1861 	} else {
1862 		r = amdgpu_gmc_get_vram_info(adev,
1863 				&vram_width, &vram_type, &vram_vendor);
1864 		if (amdgpu_sriov_vf(adev))
1865 			/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1866 			 * and DF related registers is not readable, seems hardcord is the
1867 			 * only way to set the correct vram_width
1868 			 */
1869 			adev->gmc.vram_width = 2048;
1870 		else if (amdgpu_emu_mode != 1)
1871 			adev->gmc.vram_width = vram_width;
1872 
1873 		if (!adev->gmc.vram_width) {
1874 			int chansize, numchan;
1875 
1876 			/* hbm memory channel size */
1877 			if (adev->flags & AMD_IS_APU)
1878 				chansize = 64;
1879 			else
1880 				chansize = 128;
1881 			if (adev->df.funcs &&
1882 			    adev->df.funcs->get_hbm_channel_number) {
1883 				numchan = adev->df.funcs->get_hbm_channel_number(adev);
1884 				adev->gmc.vram_width = numchan * chansize;
1885 			}
1886 		}
1887 
1888 		adev->gmc.vram_type = vram_type;
1889 		adev->gmc.vram_vendor = vram_vendor;
1890 	}
1891 
1892 	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
1893 	case IP_VERSION(9, 1, 0):
1894 	case IP_VERSION(9, 2, 2):
1895 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
1896 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
1897 
1898 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1899 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1900 		} else {
1901 			/* vm_size is 128TB + 512GB for legacy 3-level page support */
1902 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
1903 			adev->gmc.translate_further =
1904 				adev->vm_manager.num_level > 1;
1905 		}
1906 		break;
1907 	case IP_VERSION(9, 0, 1):
1908 	case IP_VERSION(9, 2, 1):
1909 	case IP_VERSION(9, 4, 0):
1910 	case IP_VERSION(9, 3, 0):
1911 	case IP_VERSION(9, 4, 2):
1912 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
1913 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
1914 
1915 		/*
1916 		 * To fulfill 4-level page support,
1917 		 * vm size is 256TB (48bit), maximum size of Vega10,
1918 		 * block size 512 (9bit)
1919 		 */
1920 
1921 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1922 		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2))
1923 			adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1924 		break;
1925 	case IP_VERSION(9, 4, 1):
1926 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
1927 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
1928 		set_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask);
1929 
1930 		/* Keep the vm size same with Vega20 */
1931 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1932 		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1933 		break;
1934 	case IP_VERSION(9, 4, 3):
1935 	case IP_VERSION(9, 4, 4):
1936 	case IP_VERSION(9, 5, 0):
1937 		bitmap_set(adev->vmhubs_mask, AMDGPU_GFXHUB(0),
1938 				  NUM_XCC(adev->gfx.xcc_mask));
1939 
1940 		inst_mask <<= AMDGPU_MMHUB0(0);
1941 		bitmap_or(adev->vmhubs_mask, adev->vmhubs_mask, &inst_mask, 32);
1942 
1943 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1944 		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
1945 		break;
1946 	default:
1947 		break;
1948 	}
1949 
1950 	/* This interrupt is VMC page fault.*/
1951 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1952 				&adev->gmc.vm_fault);
1953 	if (r)
1954 		return r;
1955 
1956 	if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) {
1957 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1958 					&adev->gmc.vm_fault);
1959 		if (r)
1960 			return r;
1961 	}
1962 
1963 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1964 				&adev->gmc.vm_fault);
1965 
1966 	if (r)
1967 		return r;
1968 
1969 	if (!amdgpu_sriov_vf(adev) &&
1970 	    !adev->gmc.xgmi.connected_to_cpu &&
1971 	    !adev->gmc.is_app_apu) {
1972 		/* interrupt sent to DF. */
1973 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1974 				      &adev->gmc.ecc_irq);
1975 		if (r)
1976 			return r;
1977 	}
1978 
1979 	/* Set the internal MC address mask
1980 	 * This is the max address of the GPU's
1981 	 * internal address space.
1982 	 */
1983 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
1984 	adev->gmc.pte_addr_mask = 0x0000FFFFFFFFF000ULL; /* 48 bit PA */
1985 
1986 	dma_addr_bits = amdgpu_ip_version(adev, GC_HWIP, 0) >=
1987 					IP_VERSION(9, 4, 2) ?
1988 				48 :
1989 				44;
1990 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(dma_addr_bits));
1991 	if (r) {
1992 		drm_warn(adev_to_drm(adev), "No suitable DMA available.\n");
1993 		return r;
1994 	}
1995 	adev->need_swiotlb = drm_need_swiotlb(dma_addr_bits);
1996 
1997 	r = gmc_v9_0_mc_init(adev);
1998 	if (r)
1999 		return r;
2000 
2001 	if (amdgpu_is_multi_aid(adev)) {
2002 		r = amdgpu_gmc_init_mem_ranges(adev);
2003 		if (r)
2004 			return r;
2005 	}
2006 
2007 	/* Memory manager */
2008 	r = amdgpu_bo_init(adev);
2009 	if (r)
2010 		return r;
2011 
2012 	r = gmc_v9_0_gart_init(adev);
2013 	if (r)
2014 		return r;
2015 
2016 	gmc_v9_0_init_nps_details(adev);
2017 	/*
2018 	 * number of VMs
2019 	 * VMID 0 is reserved for System
2020 	 * amdgpu graphics/compute will use VMIDs 1..n-1
2021 	 * amdkfd will use VMIDs n..15
2022 	 *
2023 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
2024 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
2025 	 * for video processing.
2026 	 *
2027 	 * If kernel queues are disabled, allow KFD to use all vmids.
2028 	 */
2029 	if (adev->gfx.disable_kq &&
2030 	    adev->jpeg.disable_kq &&
2031 	    adev->vcn.disable_kq &&
2032 	    adev->sdma.no_user_submission)
2033 		adev->vm_manager.first_kfd_vmid = 1;
2034 	else
2035 		adev->vm_manager.first_kfd_vmid =
2036 			(amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) ||
2037 			 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) ||
2038 			 amdgpu_is_multi_aid(adev)) ?
2039 			3 :
2040 			8;
2041 
2042 	amdgpu_vmid_mgr_set_vmid_mask(adev,
2043 				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
2044 				      false);
2045 	amdgpu_vmid_mgr_set_vmid_mask(adev,
2046 				      GENMASK(adev->vm_manager.first_kfd_vmid - 1, 1),
2047 				      true);
2048 	amdgpu_vm_manager_init(adev);
2049 
2050 	gmc_v9_0_save_registers(adev);
2051 
2052 	r = amdgpu_gmc_ras_sw_init(adev);
2053 	if (r)
2054 		return r;
2055 
2056 	if (amdgpu_is_multi_aid(adev))
2057 		amdgpu_gmc_sysfs_init(adev);
2058 
2059 	return 0;
2060 }
2061 
2062 static int gmc_v9_0_sw_fini(struct amdgpu_ip_block *ip_block)
2063 {
2064 	struct amdgpu_device *adev = ip_block->adev;
2065 
2066 	if (amdgpu_is_multi_aid(adev))
2067 		amdgpu_gmc_sysfs_fini(adev);
2068 
2069 	amdgpu_gem_force_release(adev);
2070 	amdgpu_vm_manager_fini(adev);
2071 	if (!adev->gmc.real_vram_size) {
2072 		dev_info(adev->dev, "Put GART in system memory for APU free\n");
2073 		amdgpu_gart_table_ram_free(adev);
2074 	} else {
2075 		amdgpu_gart_table_vram_free(adev);
2076 	}
2077 	amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
2078 	amdgpu_bo_fini(adev);
2079 
2080 	adev->gmc.num_mem_partitions = 0;
2081 	kfree(adev->gmc.mem_partitions);
2082 
2083 	return 0;
2084 }
2085 
2086 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
2087 {
2088 	switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) {
2089 	case IP_VERSION(9, 0, 0):
2090 		if (amdgpu_sriov_vf(adev))
2091 			break;
2092 		fallthrough;
2093 	case IP_VERSION(9, 4, 0):
2094 		soc15_program_register_sequence(adev,
2095 						golden_settings_mmhub_1_0_0,
2096 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
2097 		soc15_program_register_sequence(adev,
2098 						golden_settings_athub_1_0_0,
2099 						ARRAY_SIZE(golden_settings_athub_1_0_0));
2100 		break;
2101 	case IP_VERSION(9, 1, 0):
2102 	case IP_VERSION(9, 2, 0):
2103 		/* TODO for renoir */
2104 		soc15_program_register_sequence(adev,
2105 						golden_settings_athub_1_0_0,
2106 						ARRAY_SIZE(golden_settings_athub_1_0_0));
2107 		break;
2108 	default:
2109 		break;
2110 	}
2111 }
2112 
2113 /**
2114  * gmc_v9_0_restore_registers - restores regs
2115  *
2116  * @adev: amdgpu_device pointer
2117  *
2118  * This restores register values, saved at suspend.
2119  */
2120 void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
2121 {
2122 	if ((amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 0)) ||
2123 	    (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(1, 0, 1))) {
2124 		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
2125 		WARN_ON(adev->gmc.sdpif_register !=
2126 			RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
2127 	}
2128 }
2129 
2130 /**
2131  * gmc_v9_0_gart_enable - gart enable
2132  *
2133  * @adev: amdgpu_device pointer
2134  */
2135 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
2136 {
2137 	int r;
2138 
2139 	if (amdgpu_gmc_is_pdb0_enabled(adev))
2140 		amdgpu_gmc_init_pdb0(adev);
2141 
2142 	if (adev->gart.bo == NULL) {
2143 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
2144 		return -EINVAL;
2145 	}
2146 
2147 	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
2148 
2149 	if (!adev->in_s0ix) {
2150 		r = adev->gfxhub.funcs->gart_enable(adev);
2151 		if (r)
2152 			return r;
2153 	}
2154 
2155 	r = adev->mmhub.funcs->gart_enable(adev);
2156 	if (r)
2157 		return r;
2158 
2159 	drm_info(adev_to_drm(adev), "PCIE GART of %uM enabled.\n",
2160 		 (unsigned int)(adev->gmc.gart_size >> 20));
2161 	if (adev->gmc.pdb0_bo)
2162 		drm_info(adev_to_drm(adev), "PDB0 located at 0x%016llX\n",
2163 				(unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
2164 	drm_info(adev_to_drm(adev), "PTB located at 0x%016llX\n",
2165 			(unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
2166 
2167 	return 0;
2168 }
2169 
2170 static int gmc_v9_0_hw_init(struct amdgpu_ip_block *ip_block)
2171 {
2172 	struct amdgpu_device *adev = ip_block->adev;
2173 	bool value;
2174 	int i, r;
2175 
2176 	adev->gmc.flush_pasid_uses_kiq = true;
2177 
2178 	/* Vega20+XGMI caches PTEs in TC and TLB. Add a heavy-weight TLB flush
2179 	 * (type 2), which flushes both. Due to a race condition with
2180 	 * concurrent memory accesses using the same TLB cache line, we still
2181 	 * need a second TLB flush after this.
2182 	 */
2183 	adev->gmc.flush_tlb_needs_extra_type_2 =
2184 		amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 0) &&
2185 		adev->gmc.xgmi.num_physical_nodes;
2186 
2187 	/* The sequence of these two function calls matters.*/
2188 	gmc_v9_0_init_golden_registers(adev);
2189 
2190 	if (adev->mode_info.num_crtc) {
2191 		/* Lockout access through VGA aperture*/
2192 		WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
2193 		/* disable VGA render */
2194 		WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
2195 	}
2196 
2197 	if (adev->mmhub.funcs->update_power_gating)
2198 		adev->mmhub.funcs->update_power_gating(adev, true);
2199 
2200 	adev->hdp.funcs->init_registers(adev);
2201 
2202 	/* After HDP is initialized, flush HDP.*/
2203 	amdgpu_device_flush_hdp(adev, NULL);
2204 
2205 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
2206 		value = false;
2207 	else
2208 		value = true;
2209 
2210 	if (!amdgpu_sriov_vf(adev)) {
2211 		if (!adev->in_s0ix)
2212 			adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2213 		adev->mmhub.funcs->set_fault_enable_default(adev, value);
2214 	}
2215 	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
2216 		if (adev->in_s0ix && (i == AMDGPU_GFXHUB(0)))
2217 			continue;
2218 		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
2219 	}
2220 
2221 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
2222 		adev->umc.funcs->init_registers(adev);
2223 
2224 	r = gmc_v9_0_gart_enable(adev);
2225 	if (r)
2226 		return r;
2227 
2228 	if (amdgpu_emu_mode == 1)
2229 		return amdgpu_gmc_vram_checking(adev);
2230 
2231 	return 0;
2232 }
2233 
2234 /**
2235  * gmc_v9_0_gart_disable - gart disable
2236  *
2237  * @adev: amdgpu_device pointer
2238  *
2239  * This disables all VM page table.
2240  */
2241 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
2242 {
2243 	if (!adev->in_s0ix)
2244 		adev->gfxhub.funcs->gart_disable(adev);
2245 	adev->mmhub.funcs->gart_disable(adev);
2246 }
2247 
2248 static int gmc_v9_0_hw_fini(struct amdgpu_ip_block *ip_block)
2249 {
2250 	struct amdgpu_device *adev = ip_block->adev;
2251 
2252 	gmc_v9_0_gart_disable(adev);
2253 
2254 	if (amdgpu_sriov_vf(adev)) {
2255 		/* full access mode, so don't touch any GMC register */
2256 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
2257 		return 0;
2258 	}
2259 
2260 	/*
2261 	 * Pair the operations did in gmc_v9_0_hw_init and thus maintain
2262 	 * a correct cached state for GMC. Otherwise, the "gate" again
2263 	 * operation on S3 resuming will fail due to wrong cached state.
2264 	 */
2265 	if (adev->mmhub.funcs->update_power_gating)
2266 		adev->mmhub.funcs->update_power_gating(adev, false);
2267 
2268 	/*
2269 	 * For minimal init, late_init is not called, hence VM fault/RAS irqs
2270 	 * are not enabled.
2271 	 */
2272 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
2273 		amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
2274 
2275 		if (adev->gmc.ecc_irq.funcs &&
2276 		    amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))
2277 			amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
2278 	}
2279 
2280 	return 0;
2281 }
2282 
2283 static int gmc_v9_0_suspend(struct amdgpu_ip_block *ip_block)
2284 {
2285 	return gmc_v9_0_hw_fini(ip_block);
2286 }
2287 
2288 static int gmc_v9_0_resume(struct amdgpu_ip_block *ip_block)
2289 {
2290 	struct amdgpu_device *adev = ip_block->adev;
2291 	int r;
2292 
2293 	/* If a reset is done for NPS mode switch, read the memory range
2294 	 * information again.
2295 	 */
2296 	if (adev->gmc.reset_flags & AMDGPU_GMC_INIT_RESET_NPS) {
2297 		amdgpu_gmc_init_sw_mem_ranges(adev, adev->gmc.mem_partitions);
2298 		adev->gmc.reset_flags &= ~AMDGPU_GMC_INIT_RESET_NPS;
2299 	}
2300 
2301 	r = gmc_v9_0_hw_init(ip_block);
2302 	if (r)
2303 		return r;
2304 
2305 	amdgpu_vmid_reset_all(ip_block->adev);
2306 
2307 	return 0;
2308 }
2309 
2310 static int gmc_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
2311 {
2312 	/* There is no need to wait for MC idle in GMC v9.*/
2313 	return 0;
2314 }
2315 
2316 static int gmc_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2317 					enum amd_clockgating_state state)
2318 {
2319 	struct amdgpu_device *adev = ip_block->adev;
2320 
2321 	adev->mmhub.funcs->set_clockgating(adev, state);
2322 
2323 	athub_v1_0_set_clockgating(adev, state);
2324 
2325 	return 0;
2326 }
2327 
2328 static void gmc_v9_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
2329 {
2330 	struct amdgpu_device *adev = ip_block->adev;
2331 
2332 	adev->mmhub.funcs->get_clockgating(adev, flags);
2333 
2334 	athub_v1_0_get_clockgating(adev, flags);
2335 }
2336 
2337 static int gmc_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
2338 					enum amd_powergating_state state)
2339 {
2340 	return 0;
2341 }
2342 
2343 const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
2344 	.name = "gmc_v9_0",
2345 	.early_init = gmc_v9_0_early_init,
2346 	.late_init = gmc_v9_0_late_init,
2347 	.sw_init = gmc_v9_0_sw_init,
2348 	.sw_fini = gmc_v9_0_sw_fini,
2349 	.hw_init = gmc_v9_0_hw_init,
2350 	.hw_fini = gmc_v9_0_hw_fini,
2351 	.suspend = gmc_v9_0_suspend,
2352 	.resume = gmc_v9_0_resume,
2353 	.wait_for_idle = gmc_v9_0_wait_for_idle,
2354 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
2355 	.set_powergating_state = gmc_v9_0_set_powergating_state,
2356 	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
2357 };
2358 
2359 const struct amdgpu_ip_block_version gmc_v9_0_ip_block = {
2360 	.type = AMD_IP_BLOCK_TYPE_GMC,
2361 	.major = 9,
2362 	.minor = 0,
2363 	.rev = 0,
2364 	.funcs = &gmc_v9_0_ip_funcs,
2365 };
2366