xref: /linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision ae22a94997b8a03dcb3c922857c203246711f9d4) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include <drm/amdgpu_drm.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_atombios.h"
33 #include "amdgpu_ih.h"
34 #include "amdgpu_uvd.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37 #include "atom.h"
38 #include "amd_pcie.h"
39 
40 #include "cik.h"
41 #include "gmc_v7_0.h"
42 #include "cik_ih.h"
43 #include "dce_v8_0.h"
44 #include "gfx_v7_0.h"
45 #include "cik_sdma.h"
46 #include "uvd_v4_2.h"
47 #include "vce_v2_0.h"
48 #include "cik_dpm.h"
49 
50 #include "uvd/uvd_4_2_d.h"
51 
52 #include "smu/smu_7_0_1_d.h"
53 #include "smu/smu_7_0_1_sh_mask.h"
54 
55 #include "dce/dce_8_0_d.h"
56 #include "dce/dce_8_0_sh_mask.h"
57 
58 #include "bif/bif_4_1_d.h"
59 #include "bif/bif_4_1_sh_mask.h"
60 
61 #include "gca/gfx_7_2_d.h"
62 #include "gca/gfx_7_2_enum.h"
63 #include "gca/gfx_7_2_sh_mask.h"
64 
65 #include "gmc/gmc_7_1_d.h"
66 #include "gmc/gmc_7_1_sh_mask.h"
67 
68 #include "oss/oss_2_0_d.h"
69 #include "oss/oss_2_0_sh_mask.h"
70 
71 #include "amdgpu_dm.h"
72 #include "amdgpu_amdkfd.h"
73 #include "amdgpu_vkms.h"
74 
75 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
76 {
77 	{
78 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
79 		.max_width = 2048,
80 		.max_height = 1152,
81 		.max_pixels_per_frame = 2048 * 1152,
82 		.max_level = 0,
83 	},
84 };
85 
86 static const struct amdgpu_video_codecs cik_video_codecs_encode =
87 {
88 	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
89 	.codec_array = cik_video_codecs_encode_array,
90 };
91 
92 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
93 {
94 	{
95 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
96 		.max_width = 2048,
97 		.max_height = 1152,
98 		.max_pixels_per_frame = 2048 * 1152,
99 		.max_level = 3,
100 	},
101 	{
102 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
103 		.max_width = 2048,
104 		.max_height = 1152,
105 		.max_pixels_per_frame = 2048 * 1152,
106 		.max_level = 5,
107 	},
108 	{
109 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
110 		.max_width = 2048,
111 		.max_height = 1152,
112 		.max_pixels_per_frame = 2048 * 1152,
113 		.max_level = 41,
114 	},
115 	{
116 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
117 		.max_width = 2048,
118 		.max_height = 1152,
119 		.max_pixels_per_frame = 2048 * 1152,
120 		.max_level = 4,
121 	},
122 };
123 
124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
125 {
126 	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
127 	.codec_array = cik_video_codecs_decode_array,
128 };
129 
130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
131 				  const struct amdgpu_video_codecs **codecs)
132 {
133 	switch (adev->asic_type) {
134 	case CHIP_BONAIRE:
135 	case CHIP_HAWAII:
136 	case CHIP_KAVERI:
137 	case CHIP_KABINI:
138 	case CHIP_MULLINS:
139 		if (encode)
140 			*codecs = &cik_video_codecs_encode;
141 		else
142 			*codecs = &cik_video_codecs_decode;
143 		return 0;
144 	default:
145 		return -EINVAL;
146 	}
147 }
148 
149 /*
150  * Indirect registers accessor
151  */
152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
153 {
154 	unsigned long flags;
155 	u32 r;
156 
157 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
158 	WREG32(mmPCIE_INDEX, reg);
159 	(void)RREG32(mmPCIE_INDEX);
160 	r = RREG32(mmPCIE_DATA);
161 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
162 	return r;
163 }
164 
165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
166 {
167 	unsigned long flags;
168 
169 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
170 	WREG32(mmPCIE_INDEX, reg);
171 	(void)RREG32(mmPCIE_INDEX);
172 	WREG32(mmPCIE_DATA, v);
173 	(void)RREG32(mmPCIE_DATA);
174 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
175 }
176 
177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
178 {
179 	unsigned long flags;
180 	u32 r;
181 
182 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
183 	WREG32(mmSMC_IND_INDEX_0, (reg));
184 	r = RREG32(mmSMC_IND_DATA_0);
185 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
186 	return r;
187 }
188 
189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
190 {
191 	unsigned long flags;
192 
193 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
194 	WREG32(mmSMC_IND_INDEX_0, (reg));
195 	WREG32(mmSMC_IND_DATA_0, (v));
196 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
197 }
198 
199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
200 {
201 	unsigned long flags;
202 	u32 r;
203 
204 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
205 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
206 	r = RREG32(mmUVD_CTX_DATA);
207 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
208 	return r;
209 }
210 
211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
212 {
213 	unsigned long flags;
214 
215 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
216 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
217 	WREG32(mmUVD_CTX_DATA, (v));
218 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
219 }
220 
221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
222 {
223 	unsigned long flags;
224 	u32 r;
225 
226 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
227 	WREG32(mmDIDT_IND_INDEX, (reg));
228 	r = RREG32(mmDIDT_IND_DATA);
229 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
230 	return r;
231 }
232 
233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
234 {
235 	unsigned long flags;
236 
237 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
238 	WREG32(mmDIDT_IND_INDEX, (reg));
239 	WREG32(mmDIDT_IND_DATA, (v));
240 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
241 }
242 
243 static const u32 bonaire_golden_spm_registers[] =
244 {
245 	0xc200, 0xe0ffffff, 0xe0000000
246 };
247 
248 static const u32 bonaire_golden_common_registers[] =
249 {
250 	0x31dc, 0xffffffff, 0x00000800,
251 	0x31dd, 0xffffffff, 0x00000800,
252 	0x31e6, 0xffffffff, 0x00007fbf,
253 	0x31e7, 0xffffffff, 0x00007faf
254 };
255 
256 static const u32 bonaire_golden_registers[] =
257 {
258 	0xcd5, 0x00000333, 0x00000333,
259 	0xcd4, 0x000c0fc0, 0x00040200,
260 	0x2684, 0x00010000, 0x00058208,
261 	0xf000, 0xffff1fff, 0x00140000,
262 	0xf080, 0xfdfc0fff, 0x00000100,
263 	0xf08d, 0x40000000, 0x40000200,
264 	0x260c, 0xffffffff, 0x00000000,
265 	0x260d, 0xf00fffff, 0x00000400,
266 	0x260e, 0x0002021c, 0x00020200,
267 	0x31e, 0x00000080, 0x00000000,
268 	0x16ec, 0x000000f0, 0x00000070,
269 	0x16f0, 0xf0311fff, 0x80300000,
270 	0x263e, 0x73773777, 0x12010001,
271 	0xd43, 0x00810000, 0x408af000,
272 	0x1c0c, 0x31000111, 0x00000011,
273 	0xbd2, 0x73773777, 0x12010001,
274 	0x883, 0x00007fb6, 0x0021a1b1,
275 	0x884, 0x00007fb6, 0x002021b1,
276 	0x860, 0x00007fb6, 0x00002191,
277 	0x886, 0x00007fb6, 0x002121b1,
278 	0x887, 0x00007fb6, 0x002021b1,
279 	0x877, 0x00007fb6, 0x00002191,
280 	0x878, 0x00007fb6, 0x00002191,
281 	0xd8a, 0x0000003f, 0x0000000a,
282 	0xd8b, 0x0000003f, 0x0000000a,
283 	0xab9, 0x00073ffe, 0x000022a2,
284 	0x903, 0x000007ff, 0x00000000,
285 	0x2285, 0xf000003f, 0x00000007,
286 	0x22fc, 0x00002001, 0x00000001,
287 	0x22c9, 0xffffffff, 0x00ffffff,
288 	0xc281, 0x0000ff0f, 0x00000000,
289 	0xa293, 0x07ffffff, 0x06000000,
290 	0x136, 0x00000fff, 0x00000100,
291 	0xf9e, 0x00000001, 0x00000002,
292 	0x2440, 0x03000000, 0x0362c688,
293 	0x2300, 0x000000ff, 0x00000001,
294 	0x390, 0x00001fff, 0x00001fff,
295 	0x2418, 0x0000007f, 0x00000020,
296 	0x2542, 0x00010000, 0x00010000,
297 	0x2b05, 0x000003ff, 0x000000f3,
298 	0x2b03, 0xffffffff, 0x00001032
299 };
300 
301 static const u32 bonaire_mgcg_cgcg_init[] =
302 {
303 	0x3108, 0xffffffff, 0xfffffffc,
304 	0xc200, 0xffffffff, 0xe0000000,
305 	0xf0a8, 0xffffffff, 0x00000100,
306 	0xf082, 0xffffffff, 0x00000100,
307 	0xf0b0, 0xffffffff, 0xc0000100,
308 	0xf0b2, 0xffffffff, 0xc0000100,
309 	0xf0b1, 0xffffffff, 0xc0000100,
310 	0x1579, 0xffffffff, 0x00600100,
311 	0xf0a0, 0xffffffff, 0x00000100,
312 	0xf085, 0xffffffff, 0x06000100,
313 	0xf088, 0xffffffff, 0x00000100,
314 	0xf086, 0xffffffff, 0x06000100,
315 	0xf081, 0xffffffff, 0x00000100,
316 	0xf0b8, 0xffffffff, 0x00000100,
317 	0xf089, 0xffffffff, 0x00000100,
318 	0xf080, 0xffffffff, 0x00000100,
319 	0xf08c, 0xffffffff, 0x00000100,
320 	0xf08d, 0xffffffff, 0x00000100,
321 	0xf094, 0xffffffff, 0x00000100,
322 	0xf095, 0xffffffff, 0x00000100,
323 	0xf096, 0xffffffff, 0x00000100,
324 	0xf097, 0xffffffff, 0x00000100,
325 	0xf098, 0xffffffff, 0x00000100,
326 	0xf09f, 0xffffffff, 0x00000100,
327 	0xf09e, 0xffffffff, 0x00000100,
328 	0xf084, 0xffffffff, 0x06000100,
329 	0xf0a4, 0xffffffff, 0x00000100,
330 	0xf09d, 0xffffffff, 0x00000100,
331 	0xf0ad, 0xffffffff, 0x00000100,
332 	0xf0ac, 0xffffffff, 0x00000100,
333 	0xf09c, 0xffffffff, 0x00000100,
334 	0xc200, 0xffffffff, 0xe0000000,
335 	0xf008, 0xffffffff, 0x00010000,
336 	0xf009, 0xffffffff, 0x00030002,
337 	0xf00a, 0xffffffff, 0x00040007,
338 	0xf00b, 0xffffffff, 0x00060005,
339 	0xf00c, 0xffffffff, 0x00090008,
340 	0xf00d, 0xffffffff, 0x00010000,
341 	0xf00e, 0xffffffff, 0x00030002,
342 	0xf00f, 0xffffffff, 0x00040007,
343 	0xf010, 0xffffffff, 0x00060005,
344 	0xf011, 0xffffffff, 0x00090008,
345 	0xf012, 0xffffffff, 0x00010000,
346 	0xf013, 0xffffffff, 0x00030002,
347 	0xf014, 0xffffffff, 0x00040007,
348 	0xf015, 0xffffffff, 0x00060005,
349 	0xf016, 0xffffffff, 0x00090008,
350 	0xf017, 0xffffffff, 0x00010000,
351 	0xf018, 0xffffffff, 0x00030002,
352 	0xf019, 0xffffffff, 0x00040007,
353 	0xf01a, 0xffffffff, 0x00060005,
354 	0xf01b, 0xffffffff, 0x00090008,
355 	0xf01c, 0xffffffff, 0x00010000,
356 	0xf01d, 0xffffffff, 0x00030002,
357 	0xf01e, 0xffffffff, 0x00040007,
358 	0xf01f, 0xffffffff, 0x00060005,
359 	0xf020, 0xffffffff, 0x00090008,
360 	0xf021, 0xffffffff, 0x00010000,
361 	0xf022, 0xffffffff, 0x00030002,
362 	0xf023, 0xffffffff, 0x00040007,
363 	0xf024, 0xffffffff, 0x00060005,
364 	0xf025, 0xffffffff, 0x00090008,
365 	0xf026, 0xffffffff, 0x00010000,
366 	0xf027, 0xffffffff, 0x00030002,
367 	0xf028, 0xffffffff, 0x00040007,
368 	0xf029, 0xffffffff, 0x00060005,
369 	0xf02a, 0xffffffff, 0x00090008,
370 	0xf000, 0xffffffff, 0x96e00200,
371 	0x21c2, 0xffffffff, 0x00900100,
372 	0x3109, 0xffffffff, 0x0020003f,
373 	0xe, 0xffffffff, 0x0140001c,
374 	0xf, 0x000f0000, 0x000f0000,
375 	0x88, 0xffffffff, 0xc060000c,
376 	0x89, 0xc0000fff, 0x00000100,
377 	0x3e4, 0xffffffff, 0x00000100,
378 	0x3e6, 0x00000101, 0x00000000,
379 	0x82a, 0xffffffff, 0x00000104,
380 	0x1579, 0xff000fff, 0x00000100,
381 	0xc33, 0xc0000fff, 0x00000104,
382 	0x3079, 0x00000001, 0x00000001,
383 	0x3403, 0xff000ff0, 0x00000100,
384 	0x3603, 0xff000ff0, 0x00000100
385 };
386 
387 static const u32 spectre_golden_spm_registers[] =
388 {
389 	0xc200, 0xe0ffffff, 0xe0000000
390 };
391 
392 static const u32 spectre_golden_common_registers[] =
393 {
394 	0x31dc, 0xffffffff, 0x00000800,
395 	0x31dd, 0xffffffff, 0x00000800,
396 	0x31e6, 0xffffffff, 0x00007fbf,
397 	0x31e7, 0xffffffff, 0x00007faf
398 };
399 
400 static const u32 spectre_golden_registers[] =
401 {
402 	0xf000, 0xffff1fff, 0x96940200,
403 	0xf003, 0xffff0001, 0xff000000,
404 	0xf080, 0xfffc0fff, 0x00000100,
405 	0x1bb6, 0x00010101, 0x00010000,
406 	0x260d, 0xf00fffff, 0x00000400,
407 	0x260e, 0xfffffffc, 0x00020200,
408 	0x16ec, 0x000000f0, 0x00000070,
409 	0x16f0, 0xf0311fff, 0x80300000,
410 	0x263e, 0x73773777, 0x12010001,
411 	0x26df, 0x00ff0000, 0x00fc0000,
412 	0xbd2, 0x73773777, 0x12010001,
413 	0x2285, 0xf000003f, 0x00000007,
414 	0x22c9, 0xffffffff, 0x00ffffff,
415 	0xa0d4, 0x3f3f3fff, 0x00000082,
416 	0xa0d5, 0x0000003f, 0x00000000,
417 	0xf9e, 0x00000001, 0x00000002,
418 	0x244f, 0xffff03df, 0x00000004,
419 	0x31da, 0x00000008, 0x00000008,
420 	0x2300, 0x000008ff, 0x00000800,
421 	0x2542, 0x00010000, 0x00010000,
422 	0x2b03, 0xffffffff, 0x54763210,
423 	0x853e, 0x01ff01ff, 0x00000002,
424 	0x8526, 0x007ff800, 0x00200000,
425 	0x8057, 0xffffffff, 0x00000f40,
426 	0xc24d, 0xffffffff, 0x00000001
427 };
428 
429 static const u32 spectre_mgcg_cgcg_init[] =
430 {
431 	0x3108, 0xffffffff, 0xfffffffc,
432 	0xc200, 0xffffffff, 0xe0000000,
433 	0xf0a8, 0xffffffff, 0x00000100,
434 	0xf082, 0xffffffff, 0x00000100,
435 	0xf0b0, 0xffffffff, 0x00000100,
436 	0xf0b2, 0xffffffff, 0x00000100,
437 	0xf0b1, 0xffffffff, 0x00000100,
438 	0x1579, 0xffffffff, 0x00600100,
439 	0xf0a0, 0xffffffff, 0x00000100,
440 	0xf085, 0xffffffff, 0x06000100,
441 	0xf088, 0xffffffff, 0x00000100,
442 	0xf086, 0xffffffff, 0x06000100,
443 	0xf081, 0xffffffff, 0x00000100,
444 	0xf0b8, 0xffffffff, 0x00000100,
445 	0xf089, 0xffffffff, 0x00000100,
446 	0xf080, 0xffffffff, 0x00000100,
447 	0xf08c, 0xffffffff, 0x00000100,
448 	0xf08d, 0xffffffff, 0x00000100,
449 	0xf094, 0xffffffff, 0x00000100,
450 	0xf095, 0xffffffff, 0x00000100,
451 	0xf096, 0xffffffff, 0x00000100,
452 	0xf097, 0xffffffff, 0x00000100,
453 	0xf098, 0xffffffff, 0x00000100,
454 	0xf09f, 0xffffffff, 0x00000100,
455 	0xf09e, 0xffffffff, 0x00000100,
456 	0xf084, 0xffffffff, 0x06000100,
457 	0xf0a4, 0xffffffff, 0x00000100,
458 	0xf09d, 0xffffffff, 0x00000100,
459 	0xf0ad, 0xffffffff, 0x00000100,
460 	0xf0ac, 0xffffffff, 0x00000100,
461 	0xf09c, 0xffffffff, 0x00000100,
462 	0xc200, 0xffffffff, 0xe0000000,
463 	0xf008, 0xffffffff, 0x00010000,
464 	0xf009, 0xffffffff, 0x00030002,
465 	0xf00a, 0xffffffff, 0x00040007,
466 	0xf00b, 0xffffffff, 0x00060005,
467 	0xf00c, 0xffffffff, 0x00090008,
468 	0xf00d, 0xffffffff, 0x00010000,
469 	0xf00e, 0xffffffff, 0x00030002,
470 	0xf00f, 0xffffffff, 0x00040007,
471 	0xf010, 0xffffffff, 0x00060005,
472 	0xf011, 0xffffffff, 0x00090008,
473 	0xf012, 0xffffffff, 0x00010000,
474 	0xf013, 0xffffffff, 0x00030002,
475 	0xf014, 0xffffffff, 0x00040007,
476 	0xf015, 0xffffffff, 0x00060005,
477 	0xf016, 0xffffffff, 0x00090008,
478 	0xf017, 0xffffffff, 0x00010000,
479 	0xf018, 0xffffffff, 0x00030002,
480 	0xf019, 0xffffffff, 0x00040007,
481 	0xf01a, 0xffffffff, 0x00060005,
482 	0xf01b, 0xffffffff, 0x00090008,
483 	0xf01c, 0xffffffff, 0x00010000,
484 	0xf01d, 0xffffffff, 0x00030002,
485 	0xf01e, 0xffffffff, 0x00040007,
486 	0xf01f, 0xffffffff, 0x00060005,
487 	0xf020, 0xffffffff, 0x00090008,
488 	0xf021, 0xffffffff, 0x00010000,
489 	0xf022, 0xffffffff, 0x00030002,
490 	0xf023, 0xffffffff, 0x00040007,
491 	0xf024, 0xffffffff, 0x00060005,
492 	0xf025, 0xffffffff, 0x00090008,
493 	0xf026, 0xffffffff, 0x00010000,
494 	0xf027, 0xffffffff, 0x00030002,
495 	0xf028, 0xffffffff, 0x00040007,
496 	0xf029, 0xffffffff, 0x00060005,
497 	0xf02a, 0xffffffff, 0x00090008,
498 	0xf02b, 0xffffffff, 0x00010000,
499 	0xf02c, 0xffffffff, 0x00030002,
500 	0xf02d, 0xffffffff, 0x00040007,
501 	0xf02e, 0xffffffff, 0x00060005,
502 	0xf02f, 0xffffffff, 0x00090008,
503 	0xf000, 0xffffffff, 0x96e00200,
504 	0x21c2, 0xffffffff, 0x00900100,
505 	0x3109, 0xffffffff, 0x0020003f,
506 	0xe, 0xffffffff, 0x0140001c,
507 	0xf, 0x000f0000, 0x000f0000,
508 	0x88, 0xffffffff, 0xc060000c,
509 	0x89, 0xc0000fff, 0x00000100,
510 	0x3e4, 0xffffffff, 0x00000100,
511 	0x3e6, 0x00000101, 0x00000000,
512 	0x82a, 0xffffffff, 0x00000104,
513 	0x1579, 0xff000fff, 0x00000100,
514 	0xc33, 0xc0000fff, 0x00000104,
515 	0x3079, 0x00000001, 0x00000001,
516 	0x3403, 0xff000ff0, 0x00000100,
517 	0x3603, 0xff000ff0, 0x00000100
518 };
519 
520 static const u32 kalindi_golden_spm_registers[] =
521 {
522 	0xc200, 0xe0ffffff, 0xe0000000
523 };
524 
525 static const u32 kalindi_golden_common_registers[] =
526 {
527 	0x31dc, 0xffffffff, 0x00000800,
528 	0x31dd, 0xffffffff, 0x00000800,
529 	0x31e6, 0xffffffff, 0x00007fbf,
530 	0x31e7, 0xffffffff, 0x00007faf
531 };
532 
533 static const u32 kalindi_golden_registers[] =
534 {
535 	0xf000, 0xffffdfff, 0x6e944040,
536 	0x1579, 0xff607fff, 0xfc000100,
537 	0xf088, 0xff000fff, 0x00000100,
538 	0xf089, 0xff000fff, 0x00000100,
539 	0xf080, 0xfffc0fff, 0x00000100,
540 	0x1bb6, 0x00010101, 0x00010000,
541 	0x260c, 0xffffffff, 0x00000000,
542 	0x260d, 0xf00fffff, 0x00000400,
543 	0x16ec, 0x000000f0, 0x00000070,
544 	0x16f0, 0xf0311fff, 0x80300000,
545 	0x263e, 0x73773777, 0x12010001,
546 	0x263f, 0xffffffff, 0x00000010,
547 	0x26df, 0x00ff0000, 0x00fc0000,
548 	0x200c, 0x00001f0f, 0x0000100a,
549 	0xbd2, 0x73773777, 0x12010001,
550 	0x902, 0x000fffff, 0x000c007f,
551 	0x2285, 0xf000003f, 0x00000007,
552 	0x22c9, 0x3fff3fff, 0x00ffcfff,
553 	0xc281, 0x0000ff0f, 0x00000000,
554 	0xa293, 0x07ffffff, 0x06000000,
555 	0x136, 0x00000fff, 0x00000100,
556 	0xf9e, 0x00000001, 0x00000002,
557 	0x31da, 0x00000008, 0x00000008,
558 	0x2300, 0x000000ff, 0x00000003,
559 	0x853e, 0x01ff01ff, 0x00000002,
560 	0x8526, 0x007ff800, 0x00200000,
561 	0x8057, 0xffffffff, 0x00000f40,
562 	0x2231, 0x001f3ae3, 0x00000082,
563 	0x2235, 0x0000001f, 0x00000010,
564 	0xc24d, 0xffffffff, 0x00000000
565 };
566 
567 static const u32 kalindi_mgcg_cgcg_init[] =
568 {
569 	0x3108, 0xffffffff, 0xfffffffc,
570 	0xc200, 0xffffffff, 0xe0000000,
571 	0xf0a8, 0xffffffff, 0x00000100,
572 	0xf082, 0xffffffff, 0x00000100,
573 	0xf0b0, 0xffffffff, 0x00000100,
574 	0xf0b2, 0xffffffff, 0x00000100,
575 	0xf0b1, 0xffffffff, 0x00000100,
576 	0x1579, 0xffffffff, 0x00600100,
577 	0xf0a0, 0xffffffff, 0x00000100,
578 	0xf085, 0xffffffff, 0x06000100,
579 	0xf088, 0xffffffff, 0x00000100,
580 	0xf086, 0xffffffff, 0x06000100,
581 	0xf081, 0xffffffff, 0x00000100,
582 	0xf0b8, 0xffffffff, 0x00000100,
583 	0xf089, 0xffffffff, 0x00000100,
584 	0xf080, 0xffffffff, 0x00000100,
585 	0xf08c, 0xffffffff, 0x00000100,
586 	0xf08d, 0xffffffff, 0x00000100,
587 	0xf094, 0xffffffff, 0x00000100,
588 	0xf095, 0xffffffff, 0x00000100,
589 	0xf096, 0xffffffff, 0x00000100,
590 	0xf097, 0xffffffff, 0x00000100,
591 	0xf098, 0xffffffff, 0x00000100,
592 	0xf09f, 0xffffffff, 0x00000100,
593 	0xf09e, 0xffffffff, 0x00000100,
594 	0xf084, 0xffffffff, 0x06000100,
595 	0xf0a4, 0xffffffff, 0x00000100,
596 	0xf09d, 0xffffffff, 0x00000100,
597 	0xf0ad, 0xffffffff, 0x00000100,
598 	0xf0ac, 0xffffffff, 0x00000100,
599 	0xf09c, 0xffffffff, 0x00000100,
600 	0xc200, 0xffffffff, 0xe0000000,
601 	0xf008, 0xffffffff, 0x00010000,
602 	0xf009, 0xffffffff, 0x00030002,
603 	0xf00a, 0xffffffff, 0x00040007,
604 	0xf00b, 0xffffffff, 0x00060005,
605 	0xf00c, 0xffffffff, 0x00090008,
606 	0xf00d, 0xffffffff, 0x00010000,
607 	0xf00e, 0xffffffff, 0x00030002,
608 	0xf00f, 0xffffffff, 0x00040007,
609 	0xf010, 0xffffffff, 0x00060005,
610 	0xf011, 0xffffffff, 0x00090008,
611 	0xf000, 0xffffffff, 0x96e00200,
612 	0x21c2, 0xffffffff, 0x00900100,
613 	0x3109, 0xffffffff, 0x0020003f,
614 	0xe, 0xffffffff, 0x0140001c,
615 	0xf, 0x000f0000, 0x000f0000,
616 	0x88, 0xffffffff, 0xc060000c,
617 	0x89, 0xc0000fff, 0x00000100,
618 	0x82a, 0xffffffff, 0x00000104,
619 	0x1579, 0xff000fff, 0x00000100,
620 	0xc33, 0xc0000fff, 0x00000104,
621 	0x3079, 0x00000001, 0x00000001,
622 	0x3403, 0xff000ff0, 0x00000100,
623 	0x3603, 0xff000ff0, 0x00000100
624 };
625 
626 static const u32 hawaii_golden_spm_registers[] =
627 {
628 	0xc200, 0xe0ffffff, 0xe0000000
629 };
630 
631 static const u32 hawaii_golden_common_registers[] =
632 {
633 	0xc200, 0xffffffff, 0xe0000000,
634 	0xa0d4, 0xffffffff, 0x3a00161a,
635 	0xa0d5, 0xffffffff, 0x0000002e,
636 	0x2684, 0xffffffff, 0x00018208,
637 	0x263e, 0xffffffff, 0x12011003
638 };
639 
640 static const u32 hawaii_golden_registers[] =
641 {
642 	0xcd5, 0x00000333, 0x00000333,
643 	0x2684, 0x00010000, 0x00058208,
644 	0x260c, 0xffffffff, 0x00000000,
645 	0x260d, 0xf00fffff, 0x00000400,
646 	0x260e, 0x0002021c, 0x00020200,
647 	0x31e, 0x00000080, 0x00000000,
648 	0x16ec, 0x000000f0, 0x00000070,
649 	0x16f0, 0xf0311fff, 0x80300000,
650 	0xd43, 0x00810000, 0x408af000,
651 	0x1c0c, 0x31000111, 0x00000011,
652 	0xbd2, 0x73773777, 0x12010001,
653 	0x848, 0x0000007f, 0x0000001b,
654 	0x877, 0x00007fb6, 0x00002191,
655 	0xd8a, 0x0000003f, 0x0000000a,
656 	0xd8b, 0x0000003f, 0x0000000a,
657 	0xab9, 0x00073ffe, 0x000022a2,
658 	0x903, 0x000007ff, 0x00000000,
659 	0x22fc, 0x00002001, 0x00000001,
660 	0x22c9, 0xffffffff, 0x00ffffff,
661 	0xc281, 0x0000ff0f, 0x00000000,
662 	0xa293, 0x07ffffff, 0x06000000,
663 	0xf9e, 0x00000001, 0x00000002,
664 	0x31da, 0x00000008, 0x00000008,
665 	0x31dc, 0x00000f00, 0x00000800,
666 	0x31dd, 0x00000f00, 0x00000800,
667 	0x31e6, 0x00ffffff, 0x00ff7fbf,
668 	0x31e7, 0x00ffffff, 0x00ff7faf,
669 	0x2300, 0x000000ff, 0x00000800,
670 	0x390, 0x00001fff, 0x00001fff,
671 	0x2418, 0x0000007f, 0x00000020,
672 	0x2542, 0x00010000, 0x00010000,
673 	0x2b80, 0x00100000, 0x000ff07c,
674 	0x2b05, 0x000003ff, 0x0000000f,
675 	0x2b04, 0xffffffff, 0x7564fdec,
676 	0x2b03, 0xffffffff, 0x3120b9a8,
677 	0x2b02, 0x20000000, 0x0f9c0000
678 };
679 
680 static const u32 hawaii_mgcg_cgcg_init[] =
681 {
682 	0x3108, 0xffffffff, 0xfffffffd,
683 	0xc200, 0xffffffff, 0xe0000000,
684 	0xf0a8, 0xffffffff, 0x00000100,
685 	0xf082, 0xffffffff, 0x00000100,
686 	0xf0b0, 0xffffffff, 0x00000100,
687 	0xf0b2, 0xffffffff, 0x00000100,
688 	0xf0b1, 0xffffffff, 0x00000100,
689 	0x1579, 0xffffffff, 0x00200100,
690 	0xf0a0, 0xffffffff, 0x00000100,
691 	0xf085, 0xffffffff, 0x06000100,
692 	0xf088, 0xffffffff, 0x00000100,
693 	0xf086, 0xffffffff, 0x06000100,
694 	0xf081, 0xffffffff, 0x00000100,
695 	0xf0b8, 0xffffffff, 0x00000100,
696 	0xf089, 0xffffffff, 0x00000100,
697 	0xf080, 0xffffffff, 0x00000100,
698 	0xf08c, 0xffffffff, 0x00000100,
699 	0xf08d, 0xffffffff, 0x00000100,
700 	0xf094, 0xffffffff, 0x00000100,
701 	0xf095, 0xffffffff, 0x00000100,
702 	0xf096, 0xffffffff, 0x00000100,
703 	0xf097, 0xffffffff, 0x00000100,
704 	0xf098, 0xffffffff, 0x00000100,
705 	0xf09f, 0xffffffff, 0x00000100,
706 	0xf09e, 0xffffffff, 0x00000100,
707 	0xf084, 0xffffffff, 0x06000100,
708 	0xf0a4, 0xffffffff, 0x00000100,
709 	0xf09d, 0xffffffff, 0x00000100,
710 	0xf0ad, 0xffffffff, 0x00000100,
711 	0xf0ac, 0xffffffff, 0x00000100,
712 	0xf09c, 0xffffffff, 0x00000100,
713 	0xc200, 0xffffffff, 0xe0000000,
714 	0xf008, 0xffffffff, 0x00010000,
715 	0xf009, 0xffffffff, 0x00030002,
716 	0xf00a, 0xffffffff, 0x00040007,
717 	0xf00b, 0xffffffff, 0x00060005,
718 	0xf00c, 0xffffffff, 0x00090008,
719 	0xf00d, 0xffffffff, 0x00010000,
720 	0xf00e, 0xffffffff, 0x00030002,
721 	0xf00f, 0xffffffff, 0x00040007,
722 	0xf010, 0xffffffff, 0x00060005,
723 	0xf011, 0xffffffff, 0x00090008,
724 	0xf012, 0xffffffff, 0x00010000,
725 	0xf013, 0xffffffff, 0x00030002,
726 	0xf014, 0xffffffff, 0x00040007,
727 	0xf015, 0xffffffff, 0x00060005,
728 	0xf016, 0xffffffff, 0x00090008,
729 	0xf017, 0xffffffff, 0x00010000,
730 	0xf018, 0xffffffff, 0x00030002,
731 	0xf019, 0xffffffff, 0x00040007,
732 	0xf01a, 0xffffffff, 0x00060005,
733 	0xf01b, 0xffffffff, 0x00090008,
734 	0xf01c, 0xffffffff, 0x00010000,
735 	0xf01d, 0xffffffff, 0x00030002,
736 	0xf01e, 0xffffffff, 0x00040007,
737 	0xf01f, 0xffffffff, 0x00060005,
738 	0xf020, 0xffffffff, 0x00090008,
739 	0xf021, 0xffffffff, 0x00010000,
740 	0xf022, 0xffffffff, 0x00030002,
741 	0xf023, 0xffffffff, 0x00040007,
742 	0xf024, 0xffffffff, 0x00060005,
743 	0xf025, 0xffffffff, 0x00090008,
744 	0xf026, 0xffffffff, 0x00010000,
745 	0xf027, 0xffffffff, 0x00030002,
746 	0xf028, 0xffffffff, 0x00040007,
747 	0xf029, 0xffffffff, 0x00060005,
748 	0xf02a, 0xffffffff, 0x00090008,
749 	0xf02b, 0xffffffff, 0x00010000,
750 	0xf02c, 0xffffffff, 0x00030002,
751 	0xf02d, 0xffffffff, 0x00040007,
752 	0xf02e, 0xffffffff, 0x00060005,
753 	0xf02f, 0xffffffff, 0x00090008,
754 	0xf030, 0xffffffff, 0x00010000,
755 	0xf031, 0xffffffff, 0x00030002,
756 	0xf032, 0xffffffff, 0x00040007,
757 	0xf033, 0xffffffff, 0x00060005,
758 	0xf034, 0xffffffff, 0x00090008,
759 	0xf035, 0xffffffff, 0x00010000,
760 	0xf036, 0xffffffff, 0x00030002,
761 	0xf037, 0xffffffff, 0x00040007,
762 	0xf038, 0xffffffff, 0x00060005,
763 	0xf039, 0xffffffff, 0x00090008,
764 	0xf03a, 0xffffffff, 0x00010000,
765 	0xf03b, 0xffffffff, 0x00030002,
766 	0xf03c, 0xffffffff, 0x00040007,
767 	0xf03d, 0xffffffff, 0x00060005,
768 	0xf03e, 0xffffffff, 0x00090008,
769 	0x30c6, 0xffffffff, 0x00020200,
770 	0xcd4, 0xffffffff, 0x00000200,
771 	0x570, 0xffffffff, 0x00000400,
772 	0x157a, 0xffffffff, 0x00000000,
773 	0xbd4, 0xffffffff, 0x00000902,
774 	0xf000, 0xffffffff, 0x96940200,
775 	0x21c2, 0xffffffff, 0x00900100,
776 	0x3109, 0xffffffff, 0x0020003f,
777 	0xe, 0xffffffff, 0x0140001c,
778 	0xf, 0x000f0000, 0x000f0000,
779 	0x88, 0xffffffff, 0xc060000c,
780 	0x89, 0xc0000fff, 0x00000100,
781 	0x3e4, 0xffffffff, 0x00000100,
782 	0x3e6, 0x00000101, 0x00000000,
783 	0x82a, 0xffffffff, 0x00000104,
784 	0x1579, 0xff000fff, 0x00000100,
785 	0xc33, 0xc0000fff, 0x00000104,
786 	0x3079, 0x00000001, 0x00000001,
787 	0x3403, 0xff000ff0, 0x00000100,
788 	0x3603, 0xff000ff0, 0x00000100
789 };
790 
791 static const u32 godavari_golden_registers[] =
792 {
793 	0x1579, 0xff607fff, 0xfc000100,
794 	0x1bb6, 0x00010101, 0x00010000,
795 	0x260c, 0xffffffff, 0x00000000,
796 	0x260c0, 0xf00fffff, 0x00000400,
797 	0x184c, 0xffffffff, 0x00010000,
798 	0x16ec, 0x000000f0, 0x00000070,
799 	0x16f0, 0xf0311fff, 0x80300000,
800 	0x263e, 0x73773777, 0x12010001,
801 	0x263f, 0xffffffff, 0x00000010,
802 	0x200c, 0x00001f0f, 0x0000100a,
803 	0xbd2, 0x73773777, 0x12010001,
804 	0x902, 0x000fffff, 0x000c007f,
805 	0x2285, 0xf000003f, 0x00000007,
806 	0x22c9, 0xffffffff, 0x00ff0fff,
807 	0xc281, 0x0000ff0f, 0x00000000,
808 	0xa293, 0x07ffffff, 0x06000000,
809 	0x136, 0x00000fff, 0x00000100,
810 	0x3405, 0x00010000, 0x00810001,
811 	0x3605, 0x00010000, 0x00810001,
812 	0xf9e, 0x00000001, 0x00000002,
813 	0x31da, 0x00000008, 0x00000008,
814 	0x31dc, 0x00000f00, 0x00000800,
815 	0x31dd, 0x00000f00, 0x00000800,
816 	0x31e6, 0x00ffffff, 0x00ff7fbf,
817 	0x31e7, 0x00ffffff, 0x00ff7faf,
818 	0x2300, 0x000000ff, 0x00000001,
819 	0x853e, 0x01ff01ff, 0x00000002,
820 	0x8526, 0x007ff800, 0x00200000,
821 	0x8057, 0xffffffff, 0x00000f40,
822 	0x2231, 0x001f3ae3, 0x00000082,
823 	0x2235, 0x0000001f, 0x00000010,
824 	0xc24d, 0xffffffff, 0x00000000
825 };
826 
827 static void cik_init_golden_registers(struct amdgpu_device *adev)
828 {
829 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
830 	mutex_lock(&adev->grbm_idx_mutex);
831 
832 	switch (adev->asic_type) {
833 	case CHIP_BONAIRE:
834 		amdgpu_device_program_register_sequence(adev,
835 							bonaire_mgcg_cgcg_init,
836 							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
837 		amdgpu_device_program_register_sequence(adev,
838 							bonaire_golden_registers,
839 							ARRAY_SIZE(bonaire_golden_registers));
840 		amdgpu_device_program_register_sequence(adev,
841 							bonaire_golden_common_registers,
842 							ARRAY_SIZE(bonaire_golden_common_registers));
843 		amdgpu_device_program_register_sequence(adev,
844 							bonaire_golden_spm_registers,
845 							ARRAY_SIZE(bonaire_golden_spm_registers));
846 		break;
847 	case CHIP_KABINI:
848 		amdgpu_device_program_register_sequence(adev,
849 							kalindi_mgcg_cgcg_init,
850 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
851 		amdgpu_device_program_register_sequence(adev,
852 							kalindi_golden_registers,
853 							ARRAY_SIZE(kalindi_golden_registers));
854 		amdgpu_device_program_register_sequence(adev,
855 							kalindi_golden_common_registers,
856 							ARRAY_SIZE(kalindi_golden_common_registers));
857 		amdgpu_device_program_register_sequence(adev,
858 							kalindi_golden_spm_registers,
859 							ARRAY_SIZE(kalindi_golden_spm_registers));
860 		break;
861 	case CHIP_MULLINS:
862 		amdgpu_device_program_register_sequence(adev,
863 							kalindi_mgcg_cgcg_init,
864 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
865 		amdgpu_device_program_register_sequence(adev,
866 							godavari_golden_registers,
867 							ARRAY_SIZE(godavari_golden_registers));
868 		amdgpu_device_program_register_sequence(adev,
869 							kalindi_golden_common_registers,
870 							ARRAY_SIZE(kalindi_golden_common_registers));
871 		amdgpu_device_program_register_sequence(adev,
872 							kalindi_golden_spm_registers,
873 							ARRAY_SIZE(kalindi_golden_spm_registers));
874 		break;
875 	case CHIP_KAVERI:
876 		amdgpu_device_program_register_sequence(adev,
877 							spectre_mgcg_cgcg_init,
878 							ARRAY_SIZE(spectre_mgcg_cgcg_init));
879 		amdgpu_device_program_register_sequence(adev,
880 							spectre_golden_registers,
881 							ARRAY_SIZE(spectre_golden_registers));
882 		amdgpu_device_program_register_sequence(adev,
883 							spectre_golden_common_registers,
884 							ARRAY_SIZE(spectre_golden_common_registers));
885 		amdgpu_device_program_register_sequence(adev,
886 							spectre_golden_spm_registers,
887 							ARRAY_SIZE(spectre_golden_spm_registers));
888 		break;
889 	case CHIP_HAWAII:
890 		amdgpu_device_program_register_sequence(adev,
891 							hawaii_mgcg_cgcg_init,
892 							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
893 		amdgpu_device_program_register_sequence(adev,
894 							hawaii_golden_registers,
895 							ARRAY_SIZE(hawaii_golden_registers));
896 		amdgpu_device_program_register_sequence(adev,
897 							hawaii_golden_common_registers,
898 							ARRAY_SIZE(hawaii_golden_common_registers));
899 		amdgpu_device_program_register_sequence(adev,
900 							hawaii_golden_spm_registers,
901 							ARRAY_SIZE(hawaii_golden_spm_registers));
902 		break;
903 	default:
904 		break;
905 	}
906 	mutex_unlock(&adev->grbm_idx_mutex);
907 }
908 
909 /**
910  * cik_get_xclk - get the xclk
911  *
912  * @adev: amdgpu_device pointer
913  *
914  * Returns the reference clock used by the gfx engine
915  * (CIK).
916  */
917 static u32 cik_get_xclk(struct amdgpu_device *adev)
918 {
919 	u32 reference_clock = adev->clock.spll.reference_freq;
920 
921 	if (adev->flags & AMD_IS_APU) {
922 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
923 			return reference_clock / 2;
924 	} else {
925 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
926 			return reference_clock / 4;
927 	}
928 	return reference_clock;
929 }
930 
931 /**
932  * cik_srbm_select - select specific register instances
933  *
934  * @adev: amdgpu_device pointer
935  * @me: selected ME (micro engine)
936  * @pipe: pipe
937  * @queue: queue
938  * @vmid: VMID
939  *
940  * Switches the currently active registers instances.  Some
941  * registers are instanced per VMID, others are instanced per
942  * me/pipe/queue combination.
943  */
944 void cik_srbm_select(struct amdgpu_device *adev,
945 		     u32 me, u32 pipe, u32 queue, u32 vmid)
946 {
947 	u32 srbm_gfx_cntl =
948 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
949 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
950 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
951 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
952 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
953 }
954 
955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
956 {
957 	uint32_t tmp;
958 
959 	tmp = RREG32(mmCONFIG_CNTL);
960 	if (!state)
961 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
962 	else
963 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
964 	WREG32(mmCONFIG_CNTL, tmp);
965 }
966 
967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
968 {
969 	u32 bus_cntl;
970 	u32 d1vga_control = 0;
971 	u32 d2vga_control = 0;
972 	u32 vga_render_control = 0;
973 	u32 rom_cntl;
974 	bool r;
975 
976 	bus_cntl = RREG32(mmBUS_CNTL);
977 	if (adev->mode_info.num_crtc) {
978 		d1vga_control = RREG32(mmD1VGA_CONTROL);
979 		d2vga_control = RREG32(mmD2VGA_CONTROL);
980 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
981 	}
982 	rom_cntl = RREG32_SMC(ixROM_CNTL);
983 
984 	/* enable the rom */
985 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
986 	if (adev->mode_info.num_crtc) {
987 		/* Disable VGA mode */
988 		WREG32(mmD1VGA_CONTROL,
989 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
990 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
991 		WREG32(mmD2VGA_CONTROL,
992 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
993 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
994 		WREG32(mmVGA_RENDER_CONTROL,
995 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
996 	}
997 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
998 
999 	r = amdgpu_read_bios(adev);
1000 
1001 	/* restore regs */
1002 	WREG32(mmBUS_CNTL, bus_cntl);
1003 	if (adev->mode_info.num_crtc) {
1004 		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005 		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007 	}
1008 	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009 	return r;
1010 }
1011 
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013 				   u8 *bios, u32 length_bytes)
1014 {
1015 	u32 *dw_ptr;
1016 	unsigned long flags;
1017 	u32 i, length_dw;
1018 
1019 	if (bios == NULL)
1020 		return false;
1021 	if (length_bytes == 0)
1022 		return false;
1023 	/* APU vbios image is part of sbios image */
1024 	if (adev->flags & AMD_IS_APU)
1025 		return false;
1026 
1027 	dw_ptr = (u32 *)bios;
1028 	length_dw = ALIGN(length_bytes, 4) / 4;
1029 	/* take the smc lock since we are using the smc index */
1030 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031 	/* set rom index to 0 */
1032 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033 	WREG32(mmSMC_IND_DATA_0, 0);
1034 	/* set index to data for continous read */
1035 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036 	for (i = 0; i < length_dw; i++)
1037 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039 
1040 	return true;
1041 }
1042 
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044 	{mmGRBM_STATUS},
1045 	{mmGRBM_STATUS2},
1046 	{mmGRBM_STATUS_SE0},
1047 	{mmGRBM_STATUS_SE1},
1048 	{mmGRBM_STATUS_SE2},
1049 	{mmGRBM_STATUS_SE3},
1050 	{mmSRBM_STATUS},
1051 	{mmSRBM_STATUS2},
1052 	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053 	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054 	{mmCP_STAT},
1055 	{mmCP_STALLED_STAT1},
1056 	{mmCP_STALLED_STAT2},
1057 	{mmCP_STALLED_STAT3},
1058 	{mmCP_CPF_BUSY_STAT},
1059 	{mmCP_CPF_STALLED_STAT1},
1060 	{mmCP_CPF_STATUS},
1061 	{mmCP_CPC_BUSY_STAT},
1062 	{mmCP_CPC_STALLED_STAT1},
1063 	{mmCP_CPC_STATUS},
1064 	{mmGB_ADDR_CONFIG},
1065 	{mmMC_ARB_RAMCFG},
1066 	{mmGB_TILE_MODE0},
1067 	{mmGB_TILE_MODE1},
1068 	{mmGB_TILE_MODE2},
1069 	{mmGB_TILE_MODE3},
1070 	{mmGB_TILE_MODE4},
1071 	{mmGB_TILE_MODE5},
1072 	{mmGB_TILE_MODE6},
1073 	{mmGB_TILE_MODE7},
1074 	{mmGB_TILE_MODE8},
1075 	{mmGB_TILE_MODE9},
1076 	{mmGB_TILE_MODE10},
1077 	{mmGB_TILE_MODE11},
1078 	{mmGB_TILE_MODE12},
1079 	{mmGB_TILE_MODE13},
1080 	{mmGB_TILE_MODE14},
1081 	{mmGB_TILE_MODE15},
1082 	{mmGB_TILE_MODE16},
1083 	{mmGB_TILE_MODE17},
1084 	{mmGB_TILE_MODE18},
1085 	{mmGB_TILE_MODE19},
1086 	{mmGB_TILE_MODE20},
1087 	{mmGB_TILE_MODE21},
1088 	{mmGB_TILE_MODE22},
1089 	{mmGB_TILE_MODE23},
1090 	{mmGB_TILE_MODE24},
1091 	{mmGB_TILE_MODE25},
1092 	{mmGB_TILE_MODE26},
1093 	{mmGB_TILE_MODE27},
1094 	{mmGB_TILE_MODE28},
1095 	{mmGB_TILE_MODE29},
1096 	{mmGB_TILE_MODE30},
1097 	{mmGB_TILE_MODE31},
1098 	{mmGB_MACROTILE_MODE0},
1099 	{mmGB_MACROTILE_MODE1},
1100 	{mmGB_MACROTILE_MODE2},
1101 	{mmGB_MACROTILE_MODE3},
1102 	{mmGB_MACROTILE_MODE4},
1103 	{mmGB_MACROTILE_MODE5},
1104 	{mmGB_MACROTILE_MODE6},
1105 	{mmGB_MACROTILE_MODE7},
1106 	{mmGB_MACROTILE_MODE8},
1107 	{mmGB_MACROTILE_MODE9},
1108 	{mmGB_MACROTILE_MODE10},
1109 	{mmGB_MACROTILE_MODE11},
1110 	{mmGB_MACROTILE_MODE12},
1111 	{mmGB_MACROTILE_MODE13},
1112 	{mmGB_MACROTILE_MODE14},
1113 	{mmGB_MACROTILE_MODE15},
1114 	{mmCC_RB_BACKEND_DISABLE, true},
1115 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116 	{mmGB_BACKEND_MAP, false},
1117 	{mmPA_SC_RASTER_CONFIG, true},
1118 	{mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120 
1121 
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123 				       bool indexed, u32 se_num,
1124 				       u32 sh_num, u32 reg_offset)
1125 {
1126 	if (indexed) {
1127 		uint32_t val;
1128 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130 
1131 		switch (reg_offset) {
1132 		case mmCC_RB_BACKEND_DISABLE:
1133 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134 		case mmGC_USER_RB_BACKEND_DISABLE:
1135 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136 		case mmPA_SC_RASTER_CONFIG:
1137 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138 		case mmPA_SC_RASTER_CONFIG_1:
1139 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140 		}
1141 
1142 		mutex_lock(&adev->grbm_idx_mutex);
1143 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145 
1146 		val = RREG32(reg_offset);
1147 
1148 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150 		mutex_unlock(&adev->grbm_idx_mutex);
1151 		return val;
1152 	} else {
1153 		unsigned idx;
1154 
1155 		switch (reg_offset) {
1156 		case mmGB_ADDR_CONFIG:
1157 			return adev->gfx.config.gb_addr_config;
1158 		case mmMC_ARB_RAMCFG:
1159 			return adev->gfx.config.mc_arb_ramcfg;
1160 		case mmGB_TILE_MODE0:
1161 		case mmGB_TILE_MODE1:
1162 		case mmGB_TILE_MODE2:
1163 		case mmGB_TILE_MODE3:
1164 		case mmGB_TILE_MODE4:
1165 		case mmGB_TILE_MODE5:
1166 		case mmGB_TILE_MODE6:
1167 		case mmGB_TILE_MODE7:
1168 		case mmGB_TILE_MODE8:
1169 		case mmGB_TILE_MODE9:
1170 		case mmGB_TILE_MODE10:
1171 		case mmGB_TILE_MODE11:
1172 		case mmGB_TILE_MODE12:
1173 		case mmGB_TILE_MODE13:
1174 		case mmGB_TILE_MODE14:
1175 		case mmGB_TILE_MODE15:
1176 		case mmGB_TILE_MODE16:
1177 		case mmGB_TILE_MODE17:
1178 		case mmGB_TILE_MODE18:
1179 		case mmGB_TILE_MODE19:
1180 		case mmGB_TILE_MODE20:
1181 		case mmGB_TILE_MODE21:
1182 		case mmGB_TILE_MODE22:
1183 		case mmGB_TILE_MODE23:
1184 		case mmGB_TILE_MODE24:
1185 		case mmGB_TILE_MODE25:
1186 		case mmGB_TILE_MODE26:
1187 		case mmGB_TILE_MODE27:
1188 		case mmGB_TILE_MODE28:
1189 		case mmGB_TILE_MODE29:
1190 		case mmGB_TILE_MODE30:
1191 		case mmGB_TILE_MODE31:
1192 			idx = (reg_offset - mmGB_TILE_MODE0);
1193 			return adev->gfx.config.tile_mode_array[idx];
1194 		case mmGB_MACROTILE_MODE0:
1195 		case mmGB_MACROTILE_MODE1:
1196 		case mmGB_MACROTILE_MODE2:
1197 		case mmGB_MACROTILE_MODE3:
1198 		case mmGB_MACROTILE_MODE4:
1199 		case mmGB_MACROTILE_MODE5:
1200 		case mmGB_MACROTILE_MODE6:
1201 		case mmGB_MACROTILE_MODE7:
1202 		case mmGB_MACROTILE_MODE8:
1203 		case mmGB_MACROTILE_MODE9:
1204 		case mmGB_MACROTILE_MODE10:
1205 		case mmGB_MACROTILE_MODE11:
1206 		case mmGB_MACROTILE_MODE12:
1207 		case mmGB_MACROTILE_MODE13:
1208 		case mmGB_MACROTILE_MODE14:
1209 		case mmGB_MACROTILE_MODE15:
1210 			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211 			return adev->gfx.config.macrotile_mode_array[idx];
1212 		default:
1213 			return RREG32(reg_offset);
1214 		}
1215 	}
1216 }
1217 
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219 			     u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221 	uint32_t i;
1222 
1223 	*value = 0;
1224 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225 		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226 
1227 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228 			continue;
1229 
1230 		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231 						reg_offset);
1232 		return 0;
1233 	}
1234 	return -EINVAL;
1235 }
1236 
1237 struct kv_reset_save_regs {
1238 	u32 gmcon_reng_execute;
1239 	u32 gmcon_misc;
1240 	u32 gmcon_misc3;
1241 };
1242 
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244 				   struct kv_reset_save_regs *save)
1245 {
1246 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249 
1250 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256 
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258 				      struct kv_reset_save_regs *save)
1259 {
1260 	int i;
1261 
1262 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264 
1265 	for (i = 0; i < 5; i++)
1266 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267 
1268 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270 
1271 	for (i = 0; i < 5; i++)
1272 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273 
1274 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276 
1277 	for (i = 0; i < 5; i++)
1278 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279 
1280 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282 
1283 	for (i = 0; i < 5; i++)
1284 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285 
1286 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288 
1289 	for (i = 0; i < 5; i++)
1290 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291 
1292 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294 
1295 	for (i = 0; i < 5; i++)
1296 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297 
1298 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300 
1301 	for (i = 0; i < 5; i++)
1302 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303 
1304 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306 
1307 	for (i = 0; i < 5; i++)
1308 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309 
1310 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312 
1313 	for (i = 0; i < 5; i++)
1314 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315 
1316 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318 
1319 	for (i = 0; i < 5; i++)
1320 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321 
1322 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324 
1325 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329 
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341 	struct kv_reset_save_regs kv_save = { 0 };
1342 	u32 i;
1343 	int r = -EINVAL;
1344 
1345 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346 
1347 	if (adev->flags & AMD_IS_APU)
1348 		kv_save_regs_for_reset(adev, &kv_save);
1349 
1350 	/* disable BM */
1351 	pci_clear_master(adev->pdev);
1352 	/* reset */
1353 	amdgpu_device_pci_config_reset(adev);
1354 
1355 	udelay(100);
1356 
1357 	/* wait for asic to come out of reset */
1358 	for (i = 0; i < adev->usec_timeout; i++) {
1359 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360 			/* enable BM */
1361 			pci_set_master(adev->pdev);
1362 			adev->has_hw_reset = true;
1363 			r = 0;
1364 			break;
1365 		}
1366 		udelay(1);
1367 	}
1368 
1369 	/* does asic init need to be run first??? */
1370 	if (adev->flags & AMD_IS_APU)
1371 		kv_restore_regs_for_reset(adev, &kv_save);
1372 
1373 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374 
1375 	return r;
1376 }
1377 
1378 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380 	switch (adev->asic_type) {
1381 	case CHIP_BONAIRE:
1382 	case CHIP_HAWAII:
1383 		return amdgpu_dpm_is_baco_supported(adev);
1384 	default:
1385 		return false;
1386 	}
1387 }
1388 
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392 	bool baco_reset;
1393 
1394 	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395 	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396 		return amdgpu_reset_method;
1397 
1398 	if (amdgpu_reset_method != -1)
1399 		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400 				  amdgpu_reset_method);
1401 
1402 	switch (adev->asic_type) {
1403 	case CHIP_BONAIRE:
1404 	case CHIP_HAWAII:
1405 		baco_reset = cik_asic_supports_baco(adev);
1406 		break;
1407 	default:
1408 		baco_reset = false;
1409 		break;
1410 	}
1411 
1412 	if (baco_reset)
1413 		return AMD_RESET_METHOD_BACO;
1414 	else
1415 		return AMD_RESET_METHOD_LEGACY;
1416 }
1417 
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429 	int r;
1430 
1431 	/* APUs don't have full asic reset */
1432 	if (adev->flags & AMD_IS_APU)
1433 		return 0;
1434 
1435 	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436 		dev_info(adev->dev, "BACO reset\n");
1437 		r = amdgpu_dpm_baco_reset(adev);
1438 	} else {
1439 		dev_info(adev->dev, "PCI CONFIG reset\n");
1440 		r = cik_asic_pci_config_reset(adev);
1441 	}
1442 
1443 	return r;
1444 }
1445 
1446 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447 {
1448 	return RREG32(mmCONFIG_MEMSIZE);
1449 }
1450 
1451 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452 			      u32 cntl_reg, u32 status_reg)
1453 {
1454 	int r, i;
1455 	struct atom_clock_dividers dividers;
1456 	uint32_t tmp;
1457 
1458 	r = amdgpu_atombios_get_clock_dividers(adev,
1459 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460 					       clock, false, &dividers);
1461 	if (r)
1462 		return r;
1463 
1464 	tmp = RREG32_SMC(cntl_reg);
1465 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467 	tmp |= dividers.post_divider;
1468 	WREG32_SMC(cntl_reg, tmp);
1469 
1470 	for (i = 0; i < 100; i++) {
1471 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472 			break;
1473 		mdelay(10);
1474 	}
1475 	if (i == 100)
1476 		return -ETIMEDOUT;
1477 
1478 	return 0;
1479 }
1480 
1481 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482 {
1483 	int r = 0;
1484 
1485 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486 	if (r)
1487 		return r;
1488 
1489 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490 	return r;
1491 }
1492 
1493 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494 {
1495 	int r, i;
1496 	struct atom_clock_dividers dividers;
1497 	u32 tmp;
1498 
1499 	r = amdgpu_atombios_get_clock_dividers(adev,
1500 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501 					       ecclk, false, &dividers);
1502 	if (r)
1503 		return r;
1504 
1505 	for (i = 0; i < 100; i++) {
1506 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507 			break;
1508 		mdelay(10);
1509 	}
1510 	if (i == 100)
1511 		return -ETIMEDOUT;
1512 
1513 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516 	tmp |= dividers.post_divider;
1517 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518 
1519 	for (i = 0; i < 100; i++) {
1520 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521 			break;
1522 		mdelay(10);
1523 	}
1524 	if (i == 100)
1525 		return -ETIMEDOUT;
1526 
1527 	return 0;
1528 }
1529 
1530 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531 {
1532 	struct pci_dev *root = adev->pdev->bus->self;
1533 	u32 speed_cntl, current_data_rate;
1534 	int i;
1535 	u16 tmp16;
1536 
1537 	if (pci_is_root_bus(adev->pdev->bus))
1538 		return;
1539 
1540 	if (amdgpu_pcie_gen2 == 0)
1541 		return;
1542 
1543 	if (adev->flags & AMD_IS_APU)
1544 		return;
1545 
1546 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548 		return;
1549 
1550 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554 		if (current_data_rate == 2) {
1555 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556 			return;
1557 		}
1558 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560 		if (current_data_rate == 1) {
1561 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562 			return;
1563 		}
1564 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565 	}
1566 
1567 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568 		return;
1569 
1570 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571 		/* re-try equalization if gen3 is not already enabled */
1572 		if (current_data_rate != 2) {
1573 			u16 bridge_cfg, gpu_cfg;
1574 			u16 bridge_cfg2, gpu_cfg2;
1575 			u32 max_lw, current_lw, tmp;
1576 
1577 			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1578 			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1579 
1580 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1581 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1582 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1583 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1584 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1585 
1586 			if (current_lw < max_lw) {
1587 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1588 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1589 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1590 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1591 					tmp |= (max_lw <<
1592 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1593 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1594 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1595 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1596 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1597 				}
1598 			}
1599 
1600 			for (i = 0; i < 10; i++) {
1601 				/* check status */
1602 				pcie_capability_read_word(adev->pdev,
1603 							  PCI_EXP_DEVSTA,
1604 							  &tmp16);
1605 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1606 					break;
1607 
1608 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1609 							  &bridge_cfg);
1610 				pcie_capability_read_word(adev->pdev,
1611 							  PCI_EXP_LNKCTL,
1612 							  &gpu_cfg);
1613 
1614 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1615 							  &bridge_cfg2);
1616 				pcie_capability_read_word(adev->pdev,
1617 							  PCI_EXP_LNKCTL2,
1618 							  &gpu_cfg2);
1619 
1620 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1621 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1622 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1623 
1624 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1625 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1626 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1627 
1628 				msleep(100);
1629 
1630 				/* linkctl */
1631 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1632 								   PCI_EXP_LNKCTL_HAWD,
1633 								   bridge_cfg &
1634 								   PCI_EXP_LNKCTL_HAWD);
1635 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1636 								   PCI_EXP_LNKCTL_HAWD,
1637 								   gpu_cfg &
1638 								   PCI_EXP_LNKCTL_HAWD);
1639 
1640 				/* linkctl2 */
1641 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
1642 								   PCI_EXP_LNKCTL2_ENTER_COMP |
1643 								   PCI_EXP_LNKCTL2_TX_MARGIN,
1644 								   bridge_cfg2 &
1645 								   (PCI_EXP_LNKCTL2_ENTER_COMP |
1646 								    PCI_EXP_LNKCTL2_TX_MARGIN));
1647 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1648 								   PCI_EXP_LNKCTL2_ENTER_COMP |
1649 								   PCI_EXP_LNKCTL2_TX_MARGIN,
1650 								   gpu_cfg2 &
1651 								   (PCI_EXP_LNKCTL2_ENTER_COMP |
1652 								    PCI_EXP_LNKCTL2_TX_MARGIN));
1653 
1654 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1655 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1656 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1657 			}
1658 		}
1659 	}
1660 
1661 	/* set the link speed */
1662 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1663 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1664 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1665 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1666 
1667 	tmp16 = 0;
1668 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1669 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1670 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1671 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1672 	else
1673 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1674 	pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1675 					   PCI_EXP_LNKCTL2_TLS, tmp16);
1676 
1677 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1678 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1679 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1680 
1681 	for (i = 0; i < adev->usec_timeout; i++) {
1682 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1683 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1684 			break;
1685 		udelay(1);
1686 	}
1687 }
1688 
1689 static void cik_program_aspm(struct amdgpu_device *adev)
1690 {
1691 	u32 data, orig;
1692 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1693 	bool disable_clkreq = false;
1694 
1695 	if (!amdgpu_device_should_use_aspm(adev))
1696 		return;
1697 
1698 	if (pci_is_root_bus(adev->pdev->bus))
1699 		return;
1700 
1701 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1702 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1703 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1704 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1705 	if (orig != data)
1706 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1707 
1708 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1709 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1710 	if (orig != data)
1711 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1712 
1713 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1714 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1715 	if (orig != data)
1716 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1717 
1718 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1719 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1720 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1721 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1722 	if (!disable_l0s)
1723 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1724 
1725 	if (!disable_l1) {
1726 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1727 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1728 		if (orig != data)
1729 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1730 
1731 		if (!disable_plloff_in_l1) {
1732 			bool clk_req_support;
1733 
1734 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1735 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1736 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1737 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1738 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1739 			if (orig != data)
1740 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1741 
1742 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1743 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1744 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1745 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1746 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1747 			if (orig != data)
1748 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1749 
1750 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1751 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1752 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1753 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1754 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1755 			if (orig != data)
1756 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1757 
1758 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1759 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1760 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1761 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1762 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1763 			if (orig != data)
1764 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1765 
1766 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1767 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1768 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1769 			if (orig != data)
1770 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1771 
1772 			if (!disable_clkreq) {
1773 				struct pci_dev *root = adev->pdev->bus->self;
1774 				u32 lnkcap;
1775 
1776 				clk_req_support = false;
1777 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1778 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1779 					clk_req_support = true;
1780 			} else {
1781 				clk_req_support = false;
1782 			}
1783 
1784 			if (clk_req_support) {
1785 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1786 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1787 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1788 				if (orig != data)
1789 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1790 
1791 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1792 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1793 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1794 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1795 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1796 				if (orig != data)
1797 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1798 
1799 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1800 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1801 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1802 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1803 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1804 				if (orig != data)
1805 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1806 
1807 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1808 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1809 				if (orig != data)
1810 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1811 
1812 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1813 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1814 				if (orig != data)
1815 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1816 
1817 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1818 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1819 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1820 				if (orig != data)
1821 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1822 			}
1823 		}
1824 	} else {
1825 		if (orig != data)
1826 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1827 	}
1828 
1829 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1830 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1831 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1832 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1833 	if (orig != data)
1834 		WREG32_PCIE(ixPCIE_CNTL2, data);
1835 
1836 	if (!disable_l0s) {
1837 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1838 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1839 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1840 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1841 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1842 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1843 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1844 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1845 				if (orig != data)
1846 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1847 			}
1848 		}
1849 	}
1850 }
1851 
1852 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1853 {
1854 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1855 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1856 }
1857 
1858 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1859 {
1860 	if (!ring || !ring->funcs->emit_wreg) {
1861 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1862 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1863 	} else {
1864 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1865 	}
1866 }
1867 
1868 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1869 			       struct amdgpu_ring *ring)
1870 {
1871 	if (!ring || !ring->funcs->emit_wreg) {
1872 		WREG32(mmHDP_DEBUG0, 1);
1873 		RREG32(mmHDP_DEBUG0);
1874 	} else {
1875 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1876 	}
1877 }
1878 
1879 static bool cik_need_full_reset(struct amdgpu_device *adev)
1880 {
1881 	/* change this when we support soft reset */
1882 	return true;
1883 }
1884 
1885 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1886 			       uint64_t *count1)
1887 {
1888 	uint32_t perfctr = 0;
1889 	uint64_t cnt0_of, cnt1_of;
1890 	int tmp;
1891 
1892 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1893 	 * that may or may not be different from their GPU counterparts
1894 	 */
1895 	if (adev->flags & AMD_IS_APU)
1896 		return;
1897 
1898 	/* Set the 2 events that we wish to watch, defined above */
1899 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1900 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1901 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1902 
1903 	/* Write to enable desired perf counters */
1904 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1905 	/* Zero out and enable the perf counters
1906 	 * Write 0x5:
1907 	 * Bit 0 = Start all counters(1)
1908 	 * Bit 2 = Global counter reset enable(1)
1909 	 */
1910 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1911 
1912 	msleep(1000);
1913 
1914 	/* Load the shadow and disable the perf counters
1915 	 * Write 0x2:
1916 	 * Bit 0 = Stop counters(0)
1917 	 * Bit 1 = Load the shadow counters(1)
1918 	 */
1919 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1920 
1921 	/* Read register values to get any >32bit overflow */
1922 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1923 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1924 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1925 
1926 	/* Get the values and add the overflow */
1927 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1928 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1929 }
1930 
1931 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1932 {
1933 	u32 clock_cntl, pc;
1934 
1935 	if (adev->flags & AMD_IS_APU)
1936 		return false;
1937 
1938 	/* check if the SMC is already running */
1939 	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1940 	pc = RREG32_SMC(ixSMC_PC_C);
1941 	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1942 	    (0x20100 <= pc))
1943 		return true;
1944 
1945 	return false;
1946 }
1947 
1948 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1949 {
1950 	uint64_t nak_r, nak_g;
1951 
1952 	/* Get the number of NAKs received and generated */
1953 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1954 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1955 
1956 	/* Add the total number of NAKs, i.e the number of replays */
1957 	return (nak_r + nak_g);
1958 }
1959 
1960 static void cik_pre_asic_init(struct amdgpu_device *adev)
1961 {
1962 }
1963 
1964 static const struct amdgpu_asic_funcs cik_asic_funcs =
1965 {
1966 	.read_disabled_bios = &cik_read_disabled_bios,
1967 	.read_bios_from_rom = &cik_read_bios_from_rom,
1968 	.read_register = &cik_read_register,
1969 	.reset = &cik_asic_reset,
1970 	.reset_method = &cik_asic_reset_method,
1971 	.set_vga_state = &cik_vga_set_state,
1972 	.get_xclk = &cik_get_xclk,
1973 	.set_uvd_clocks = &cik_set_uvd_clocks,
1974 	.set_vce_clocks = &cik_set_vce_clocks,
1975 	.get_config_memsize = &cik_get_config_memsize,
1976 	.flush_hdp = &cik_flush_hdp,
1977 	.invalidate_hdp = &cik_invalidate_hdp,
1978 	.need_full_reset = &cik_need_full_reset,
1979 	.init_doorbell_index = &legacy_doorbell_index_init,
1980 	.get_pcie_usage = &cik_get_pcie_usage,
1981 	.need_reset_on_init = &cik_need_reset_on_init,
1982 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1983 	.supports_baco = &cik_asic_supports_baco,
1984 	.pre_asic_init = &cik_pre_asic_init,
1985 	.query_video_codecs = &cik_query_video_codecs,
1986 };
1987 
1988 static int cik_common_early_init(void *handle)
1989 {
1990 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1991 
1992 	adev->smc_rreg = &cik_smc_rreg;
1993 	adev->smc_wreg = &cik_smc_wreg;
1994 	adev->pcie_rreg = &cik_pcie_rreg;
1995 	adev->pcie_wreg = &cik_pcie_wreg;
1996 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
1997 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
1998 	adev->didt_rreg = &cik_didt_rreg;
1999 	adev->didt_wreg = &cik_didt_wreg;
2000 
2001 	adev->asic_funcs = &cik_asic_funcs;
2002 
2003 	adev->rev_id = cik_get_rev_id(adev);
2004 	adev->external_rev_id = 0xFF;
2005 	switch (adev->asic_type) {
2006 	case CHIP_BONAIRE:
2007 		adev->cg_flags =
2008 			AMD_CG_SUPPORT_GFX_MGCG |
2009 			AMD_CG_SUPPORT_GFX_MGLS |
2010 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2011 			AMD_CG_SUPPORT_GFX_CGLS |
2012 			AMD_CG_SUPPORT_GFX_CGTS |
2013 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2014 			AMD_CG_SUPPORT_GFX_CP_LS |
2015 			AMD_CG_SUPPORT_MC_LS |
2016 			AMD_CG_SUPPORT_MC_MGCG |
2017 			AMD_CG_SUPPORT_SDMA_MGCG |
2018 			AMD_CG_SUPPORT_SDMA_LS |
2019 			AMD_CG_SUPPORT_BIF_LS |
2020 			AMD_CG_SUPPORT_VCE_MGCG |
2021 			AMD_CG_SUPPORT_UVD_MGCG |
2022 			AMD_CG_SUPPORT_HDP_LS |
2023 			AMD_CG_SUPPORT_HDP_MGCG;
2024 		adev->pg_flags = 0;
2025 		adev->external_rev_id = adev->rev_id + 0x14;
2026 		break;
2027 	case CHIP_HAWAII:
2028 		adev->cg_flags =
2029 			AMD_CG_SUPPORT_GFX_MGCG |
2030 			AMD_CG_SUPPORT_GFX_MGLS |
2031 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2032 			AMD_CG_SUPPORT_GFX_CGLS |
2033 			AMD_CG_SUPPORT_GFX_CGTS |
2034 			AMD_CG_SUPPORT_GFX_CP_LS |
2035 			AMD_CG_SUPPORT_MC_LS |
2036 			AMD_CG_SUPPORT_MC_MGCG |
2037 			AMD_CG_SUPPORT_SDMA_MGCG |
2038 			AMD_CG_SUPPORT_SDMA_LS |
2039 			AMD_CG_SUPPORT_BIF_LS |
2040 			AMD_CG_SUPPORT_VCE_MGCG |
2041 			AMD_CG_SUPPORT_UVD_MGCG |
2042 			AMD_CG_SUPPORT_HDP_LS |
2043 			AMD_CG_SUPPORT_HDP_MGCG;
2044 		adev->pg_flags = 0;
2045 		adev->external_rev_id = 0x28;
2046 		break;
2047 	case CHIP_KAVERI:
2048 		adev->cg_flags =
2049 			AMD_CG_SUPPORT_GFX_MGCG |
2050 			AMD_CG_SUPPORT_GFX_MGLS |
2051 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2052 			AMD_CG_SUPPORT_GFX_CGLS |
2053 			AMD_CG_SUPPORT_GFX_CGTS |
2054 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2055 			AMD_CG_SUPPORT_GFX_CP_LS |
2056 			AMD_CG_SUPPORT_SDMA_MGCG |
2057 			AMD_CG_SUPPORT_SDMA_LS |
2058 			AMD_CG_SUPPORT_BIF_LS |
2059 			AMD_CG_SUPPORT_VCE_MGCG |
2060 			AMD_CG_SUPPORT_UVD_MGCG |
2061 			AMD_CG_SUPPORT_HDP_LS |
2062 			AMD_CG_SUPPORT_HDP_MGCG;
2063 		adev->pg_flags =
2064 			/*AMD_PG_SUPPORT_GFX_PG |
2065 			  AMD_PG_SUPPORT_GFX_SMG |
2066 			  AMD_PG_SUPPORT_GFX_DMG |*/
2067 			AMD_PG_SUPPORT_UVD |
2068 			AMD_PG_SUPPORT_VCE |
2069 			/*  AMD_PG_SUPPORT_CP |
2070 			  AMD_PG_SUPPORT_GDS |
2071 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2072 			  AMD_PG_SUPPORT_ACP |
2073 			  AMD_PG_SUPPORT_SAMU |*/
2074 			0;
2075 		if (adev->pdev->device == 0x1312 ||
2076 			adev->pdev->device == 0x1316 ||
2077 			adev->pdev->device == 0x1317)
2078 			adev->external_rev_id = 0x41;
2079 		else
2080 			adev->external_rev_id = 0x1;
2081 		break;
2082 	case CHIP_KABINI:
2083 	case CHIP_MULLINS:
2084 		adev->cg_flags =
2085 			AMD_CG_SUPPORT_GFX_MGCG |
2086 			AMD_CG_SUPPORT_GFX_MGLS |
2087 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2088 			AMD_CG_SUPPORT_GFX_CGLS |
2089 			AMD_CG_SUPPORT_GFX_CGTS |
2090 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2091 			AMD_CG_SUPPORT_GFX_CP_LS |
2092 			AMD_CG_SUPPORT_SDMA_MGCG |
2093 			AMD_CG_SUPPORT_SDMA_LS |
2094 			AMD_CG_SUPPORT_BIF_LS |
2095 			AMD_CG_SUPPORT_VCE_MGCG |
2096 			AMD_CG_SUPPORT_UVD_MGCG |
2097 			AMD_CG_SUPPORT_HDP_LS |
2098 			AMD_CG_SUPPORT_HDP_MGCG;
2099 		adev->pg_flags =
2100 			/*AMD_PG_SUPPORT_GFX_PG |
2101 			  AMD_PG_SUPPORT_GFX_SMG | */
2102 			AMD_PG_SUPPORT_UVD |
2103 			/*AMD_PG_SUPPORT_VCE |
2104 			  AMD_PG_SUPPORT_CP |
2105 			  AMD_PG_SUPPORT_GDS |
2106 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2107 			  AMD_PG_SUPPORT_SAMU |*/
2108 			0;
2109 		if (adev->asic_type == CHIP_KABINI) {
2110 			if (adev->rev_id == 0)
2111 				adev->external_rev_id = 0x81;
2112 			else if (adev->rev_id == 1)
2113 				adev->external_rev_id = 0x82;
2114 			else if (adev->rev_id == 2)
2115 				adev->external_rev_id = 0x85;
2116 		} else
2117 			adev->external_rev_id = adev->rev_id + 0xa1;
2118 		break;
2119 	default:
2120 		/* FIXME: not supported yet */
2121 		return -EINVAL;
2122 	}
2123 
2124 	return 0;
2125 }
2126 
2127 static int cik_common_sw_init(void *handle)
2128 {
2129 	return 0;
2130 }
2131 
2132 static int cik_common_sw_fini(void *handle)
2133 {
2134 	return 0;
2135 }
2136 
2137 static int cik_common_hw_init(void *handle)
2138 {
2139 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2140 
2141 	/* move the golden regs per IP block */
2142 	cik_init_golden_registers(adev);
2143 	/* enable pcie gen2/3 link */
2144 	cik_pcie_gen3_enable(adev);
2145 	/* enable aspm */
2146 	cik_program_aspm(adev);
2147 
2148 	return 0;
2149 }
2150 
2151 static int cik_common_hw_fini(void *handle)
2152 {
2153 	return 0;
2154 }
2155 
2156 static int cik_common_suspend(void *handle)
2157 {
2158 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2159 
2160 	return cik_common_hw_fini(adev);
2161 }
2162 
2163 static int cik_common_resume(void *handle)
2164 {
2165 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2166 
2167 	return cik_common_hw_init(adev);
2168 }
2169 
2170 static bool cik_common_is_idle(void *handle)
2171 {
2172 	return true;
2173 }
2174 
2175 static int cik_common_wait_for_idle(void *handle)
2176 {
2177 	return 0;
2178 }
2179 
2180 static int cik_common_soft_reset(void *handle)
2181 {
2182 	/* XXX hard reset?? */
2183 	return 0;
2184 }
2185 
2186 static int cik_common_set_clockgating_state(void *handle,
2187 					    enum amd_clockgating_state state)
2188 {
2189 	return 0;
2190 }
2191 
2192 static int cik_common_set_powergating_state(void *handle,
2193 					    enum amd_powergating_state state)
2194 {
2195 	return 0;
2196 }
2197 
2198 static const struct amd_ip_funcs cik_common_ip_funcs = {
2199 	.name = "cik_common",
2200 	.early_init = cik_common_early_init,
2201 	.late_init = NULL,
2202 	.sw_init = cik_common_sw_init,
2203 	.sw_fini = cik_common_sw_fini,
2204 	.hw_init = cik_common_hw_init,
2205 	.hw_fini = cik_common_hw_fini,
2206 	.suspend = cik_common_suspend,
2207 	.resume = cik_common_resume,
2208 	.is_idle = cik_common_is_idle,
2209 	.wait_for_idle = cik_common_wait_for_idle,
2210 	.soft_reset = cik_common_soft_reset,
2211 	.set_clockgating_state = cik_common_set_clockgating_state,
2212 	.set_powergating_state = cik_common_set_powergating_state,
2213 };
2214 
2215 static const struct amdgpu_ip_block_version cik_common_ip_block =
2216 {
2217 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2218 	.major = 1,
2219 	.minor = 0,
2220 	.rev = 0,
2221 	.funcs = &cik_common_ip_funcs,
2222 };
2223 
2224 int cik_set_ip_blocks(struct amdgpu_device *adev)
2225 {
2226 	switch (adev->asic_type) {
2227 	case CHIP_BONAIRE:
2228 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2229 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2230 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2231 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2232 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2233 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2234 		if (adev->enable_virtual_display)
2235 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2236 #if defined(CONFIG_DRM_AMD_DC)
2237 		else if (amdgpu_device_has_dc_support(adev))
2238 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2239 #endif
2240 		else
2241 			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2242 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2243 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2244 		break;
2245 	case CHIP_HAWAII:
2246 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2247 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2248 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2249 		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2250 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2251 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2252 		if (adev->enable_virtual_display)
2253 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2254 #if defined(CONFIG_DRM_AMD_DC)
2255 		else if (amdgpu_device_has_dc_support(adev))
2256 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2257 #endif
2258 		else
2259 			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2260 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2261 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2262 		break;
2263 	case CHIP_KAVERI:
2264 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2265 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2266 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2267 		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2268 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2269 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2270 		if (adev->enable_virtual_display)
2271 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2272 #if defined(CONFIG_DRM_AMD_DC)
2273 		else if (amdgpu_device_has_dc_support(adev))
2274 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2275 #endif
2276 		else
2277 			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2278 
2279 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2280 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2281 		break;
2282 	case CHIP_KABINI:
2283 	case CHIP_MULLINS:
2284 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2285 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2286 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2287 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2288 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2289 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2290 		if (adev->enable_virtual_display)
2291 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2292 #if defined(CONFIG_DRM_AMD_DC)
2293 		else if (amdgpu_device_has_dc_support(adev))
2294 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2295 #endif
2296 		else
2297 			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2298 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2299 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2300 		break;
2301 	default:
2302 		/* FIXME: not supported yet */
2303 		return -EINVAL;
2304 	}
2305 	return 0;
2306 }
2307