xref: /linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision 570f7e331f5febb30f1384817463c7e42b65ca7d) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include <drm/amdgpu_drm.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_atombios.h"
33 #include "amdgpu_ih.h"
34 #include "amdgpu_uvd.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37 #include "atom.h"
38 #include "amd_pcie.h"
39 #include "amdgpu_video_codecs.h"
40 
41 #include "cik.h"
42 #include "gmc_v7_0.h"
43 #include "cik_ih.h"
44 #include "dce_v8_0.h"
45 #include "gfx_v7_0.h"
46 #include "cik_sdma.h"
47 #include "uvd_v4_2.h"
48 #include "vce_v2_0.h"
49 #include "cik_dpm.h"
50 
51 #include "uvd/uvd_4_2_d.h"
52 
53 #include "smu/smu_7_0_1_d.h"
54 #include "smu/smu_7_0_1_sh_mask.h"
55 
56 #include "dce/dce_8_0_d.h"
57 #include "dce/dce_8_0_sh_mask.h"
58 
59 #include "bif/bif_4_1_d.h"
60 #include "bif/bif_4_1_sh_mask.h"
61 
62 #include "gca/gfx_7_2_d.h"
63 #include "gca/gfx_7_2_enum.h"
64 #include "gca/gfx_7_2_sh_mask.h"
65 
66 #include "gmc/gmc_7_1_d.h"
67 #include "gmc/gmc_7_1_sh_mask.h"
68 
69 #include "oss/oss_2_0_d.h"
70 #include "oss/oss_2_0_sh_mask.h"
71 
72 #include "amdgpu_dm.h"
73 #include "amdgpu_amdkfd.h"
74 #include "amdgpu_vkms.h"
75 
76 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
77 {
78 	{
79 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
80 		.max_width = 2048,
81 		.max_height = 1152,
82 		.max_pixels_per_frame = 2048 * 1152,
83 		.max_level = 0,
84 	},
85 };
86 
87 static const struct amdgpu_video_codecs cik_video_codecs_encode =
88 {
89 	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
90 	.codec_array = cik_video_codecs_encode_array,
91 };
92 
93 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
94 {
95 	{
96 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
97 		.max_width = 2048,
98 		.max_height = 1152,
99 		.max_pixels_per_frame = 2048 * 1152,
100 		.max_level = 3,
101 	},
102 	{
103 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
104 		.max_width = 2048,
105 		.max_height = 1152,
106 		.max_pixels_per_frame = 2048 * 1152,
107 		.max_level = 5,
108 	},
109 	{
110 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
111 		.max_width = 2048,
112 		.max_height = 1152,
113 		.max_pixels_per_frame = 2048 * 1152,
114 		.max_level = 41,
115 	},
116 	{
117 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
118 		.max_width = 2048,
119 		.max_height = 1152,
120 		.max_pixels_per_frame = 2048 * 1152,
121 		.max_level = 4,
122 	},
123 };
124 
125 static const struct amdgpu_video_codecs cik_video_codecs_decode =
126 {
127 	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
128 	.codec_array = cik_video_codecs_decode_array,
129 };
130 
131 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
132 				  const struct amdgpu_video_codecs **codecs)
133 {
134 	switch (adev->asic_type) {
135 	case CHIP_BONAIRE:
136 	case CHIP_HAWAII:
137 	case CHIP_KAVERI:
138 	case CHIP_KABINI:
139 	case CHIP_MULLINS:
140 		if (encode)
141 			*codecs = &cik_video_codecs_encode;
142 		else
143 			*codecs = &cik_video_codecs_decode;
144 		return 0;
145 	default:
146 		return -EINVAL;
147 	}
148 }
149 
150 /*
151  * Indirect registers accessor
152  */
153 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
154 {
155 	unsigned long flags;
156 	u32 r;
157 
158 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
159 	WREG32(mmPCIE_INDEX, reg);
160 	(void)RREG32(mmPCIE_INDEX);
161 	r = RREG32(mmPCIE_DATA);
162 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
163 	return r;
164 }
165 
166 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
167 {
168 	unsigned long flags;
169 
170 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
171 	WREG32(mmPCIE_INDEX, reg);
172 	(void)RREG32(mmPCIE_INDEX);
173 	WREG32(mmPCIE_DATA, v);
174 	(void)RREG32(mmPCIE_DATA);
175 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
176 }
177 
178 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
179 {
180 	unsigned long flags;
181 	u32 r;
182 
183 	spin_lock_irqsave(&adev->reg.smc.lock, flags);
184 	WREG32(mmSMC_IND_INDEX_0, (reg));
185 	r = RREG32(mmSMC_IND_DATA_0);
186 	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
187 	return r;
188 }
189 
190 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
191 {
192 	unsigned long flags;
193 
194 	spin_lock_irqsave(&adev->reg.smc.lock, flags);
195 	WREG32(mmSMC_IND_INDEX_0, (reg));
196 	WREG32(mmSMC_IND_DATA_0, (v));
197 	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
198 }
199 
200 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
201 {
202 	unsigned long flags;
203 	u32 r;
204 
205 	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
206 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
207 	r = RREG32(mmUVD_CTX_DATA);
208 	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
209 	return r;
210 }
211 
212 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
213 {
214 	unsigned long flags;
215 
216 	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
217 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
218 	WREG32(mmUVD_CTX_DATA, (v));
219 	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
220 }
221 
222 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
223 {
224 	unsigned long flags;
225 	u32 r;
226 
227 	spin_lock_irqsave(&adev->reg.didt.lock, flags);
228 	WREG32(mmDIDT_IND_INDEX, (reg));
229 	r = RREG32(mmDIDT_IND_DATA);
230 	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
231 	return r;
232 }
233 
234 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
235 {
236 	unsigned long flags;
237 
238 	spin_lock_irqsave(&adev->reg.didt.lock, flags);
239 	WREG32(mmDIDT_IND_INDEX, (reg));
240 	WREG32(mmDIDT_IND_DATA, (v));
241 	spin_unlock_irqrestore(&adev->reg.didt.lock, flags);
242 }
243 
244 static const u32 bonaire_golden_spm_registers[] =
245 {
246 	0xc200, 0xe0ffffff, 0xe0000000
247 };
248 
249 static const u32 bonaire_golden_common_registers[] =
250 {
251 	0x31dc, 0xffffffff, 0x00000800,
252 	0x31dd, 0xffffffff, 0x00000800,
253 	0x31e6, 0xffffffff, 0x00007fbf,
254 	0x31e7, 0xffffffff, 0x00007faf
255 };
256 
257 static const u32 bonaire_golden_registers[] =
258 {
259 	0xcd5, 0x00000333, 0x00000333,
260 	0xcd4, 0x000c0fc0, 0x00040200,
261 	0x2684, 0x00010000, 0x00058208,
262 	0xf000, 0xffff1fff, 0x00140000,
263 	0xf080, 0xfdfc0fff, 0x00000100,
264 	0xf08d, 0x40000000, 0x40000200,
265 	0x260c, 0xffffffff, 0x00000000,
266 	0x260d, 0xf00fffff, 0x00000400,
267 	0x260e, 0x0002021c, 0x00020200,
268 	0x31e, 0x00000080, 0x00000000,
269 	0x16ec, 0x000000f0, 0x00000070,
270 	0x16f0, 0xf0311fff, 0x80300000,
271 	0x263e, 0x73773777, 0x12010001,
272 	0xd43, 0x00810000, 0x408af000,
273 	0x1c0c, 0x31000111, 0x00000011,
274 	0xbd2, 0x73773777, 0x12010001,
275 	0x883, 0x00007fb6, 0x0021a1b1,
276 	0x884, 0x00007fb6, 0x002021b1,
277 	0x860, 0x00007fb6, 0x00002191,
278 	0x886, 0x00007fb6, 0x002121b1,
279 	0x887, 0x00007fb6, 0x002021b1,
280 	0x877, 0x00007fb6, 0x00002191,
281 	0x878, 0x00007fb6, 0x00002191,
282 	0xd8a, 0x0000003f, 0x0000000a,
283 	0xd8b, 0x0000003f, 0x0000000a,
284 	0xab9, 0x00073ffe, 0x000022a2,
285 	0x903, 0x000007ff, 0x00000000,
286 	0x2285, 0xf000003f, 0x00000007,
287 	0x22fc, 0x00002001, 0x00000001,
288 	0x22c9, 0xffffffff, 0x00ffffff,
289 	0xc281, 0x0000ff0f, 0x00000000,
290 	0xa293, 0x07ffffff, 0x06000000,
291 	0x136, 0x00000fff, 0x00000100,
292 	0xf9e, 0x00000001, 0x00000002,
293 	0x2440, 0x03000000, 0x0362c688,
294 	0x2300, 0x000000ff, 0x00000001,
295 	0x390, 0x00001fff, 0x00001fff,
296 	0x2418, 0x0000007f, 0x00000020,
297 	0x2542, 0x00010000, 0x00010000,
298 	0x2b05, 0x000003ff, 0x000000f3,
299 	0x2b03, 0xffffffff, 0x00001032
300 };
301 
302 static const u32 bonaire_mgcg_cgcg_init[] =
303 {
304 	0x3108, 0xffffffff, 0xfffffffc,
305 	0xc200, 0xffffffff, 0xe0000000,
306 	0xf0a8, 0xffffffff, 0x00000100,
307 	0xf082, 0xffffffff, 0x00000100,
308 	0xf0b0, 0xffffffff, 0xc0000100,
309 	0xf0b2, 0xffffffff, 0xc0000100,
310 	0xf0b1, 0xffffffff, 0xc0000100,
311 	0x1579, 0xffffffff, 0x00600100,
312 	0xf0a0, 0xffffffff, 0x00000100,
313 	0xf085, 0xffffffff, 0x06000100,
314 	0xf088, 0xffffffff, 0x00000100,
315 	0xf086, 0xffffffff, 0x06000100,
316 	0xf081, 0xffffffff, 0x00000100,
317 	0xf0b8, 0xffffffff, 0x00000100,
318 	0xf089, 0xffffffff, 0x00000100,
319 	0xf080, 0xffffffff, 0x00000100,
320 	0xf08c, 0xffffffff, 0x00000100,
321 	0xf08d, 0xffffffff, 0x00000100,
322 	0xf094, 0xffffffff, 0x00000100,
323 	0xf095, 0xffffffff, 0x00000100,
324 	0xf096, 0xffffffff, 0x00000100,
325 	0xf097, 0xffffffff, 0x00000100,
326 	0xf098, 0xffffffff, 0x00000100,
327 	0xf09f, 0xffffffff, 0x00000100,
328 	0xf09e, 0xffffffff, 0x00000100,
329 	0xf084, 0xffffffff, 0x06000100,
330 	0xf0a4, 0xffffffff, 0x00000100,
331 	0xf09d, 0xffffffff, 0x00000100,
332 	0xf0ad, 0xffffffff, 0x00000100,
333 	0xf0ac, 0xffffffff, 0x00000100,
334 	0xf09c, 0xffffffff, 0x00000100,
335 	0xc200, 0xffffffff, 0xe0000000,
336 	0xf008, 0xffffffff, 0x00010000,
337 	0xf009, 0xffffffff, 0x00030002,
338 	0xf00a, 0xffffffff, 0x00040007,
339 	0xf00b, 0xffffffff, 0x00060005,
340 	0xf00c, 0xffffffff, 0x00090008,
341 	0xf00d, 0xffffffff, 0x00010000,
342 	0xf00e, 0xffffffff, 0x00030002,
343 	0xf00f, 0xffffffff, 0x00040007,
344 	0xf010, 0xffffffff, 0x00060005,
345 	0xf011, 0xffffffff, 0x00090008,
346 	0xf012, 0xffffffff, 0x00010000,
347 	0xf013, 0xffffffff, 0x00030002,
348 	0xf014, 0xffffffff, 0x00040007,
349 	0xf015, 0xffffffff, 0x00060005,
350 	0xf016, 0xffffffff, 0x00090008,
351 	0xf017, 0xffffffff, 0x00010000,
352 	0xf018, 0xffffffff, 0x00030002,
353 	0xf019, 0xffffffff, 0x00040007,
354 	0xf01a, 0xffffffff, 0x00060005,
355 	0xf01b, 0xffffffff, 0x00090008,
356 	0xf01c, 0xffffffff, 0x00010000,
357 	0xf01d, 0xffffffff, 0x00030002,
358 	0xf01e, 0xffffffff, 0x00040007,
359 	0xf01f, 0xffffffff, 0x00060005,
360 	0xf020, 0xffffffff, 0x00090008,
361 	0xf021, 0xffffffff, 0x00010000,
362 	0xf022, 0xffffffff, 0x00030002,
363 	0xf023, 0xffffffff, 0x00040007,
364 	0xf024, 0xffffffff, 0x00060005,
365 	0xf025, 0xffffffff, 0x00090008,
366 	0xf026, 0xffffffff, 0x00010000,
367 	0xf027, 0xffffffff, 0x00030002,
368 	0xf028, 0xffffffff, 0x00040007,
369 	0xf029, 0xffffffff, 0x00060005,
370 	0xf02a, 0xffffffff, 0x00090008,
371 	0xf000, 0xffffffff, 0x96e00200,
372 	0x21c2, 0xffffffff, 0x00900100,
373 	0x3109, 0xffffffff, 0x0020003f,
374 	0xe, 0xffffffff, 0x0140001c,
375 	0xf, 0x000f0000, 0x000f0000,
376 	0x88, 0xffffffff, 0xc060000c,
377 	0x89, 0xc0000fff, 0x00000100,
378 	0x3e4, 0xffffffff, 0x00000100,
379 	0x3e6, 0x00000101, 0x00000000,
380 	0x82a, 0xffffffff, 0x00000104,
381 	0x1579, 0xff000fff, 0x00000100,
382 	0xc33, 0xc0000fff, 0x00000104,
383 	0x3079, 0x00000001, 0x00000001,
384 	0x3403, 0xff000ff0, 0x00000100,
385 	0x3603, 0xff000ff0, 0x00000100
386 };
387 
388 static const u32 spectre_golden_spm_registers[] =
389 {
390 	0xc200, 0xe0ffffff, 0xe0000000
391 };
392 
393 static const u32 spectre_golden_common_registers[] =
394 {
395 	0x31dc, 0xffffffff, 0x00000800,
396 	0x31dd, 0xffffffff, 0x00000800,
397 	0x31e6, 0xffffffff, 0x00007fbf,
398 	0x31e7, 0xffffffff, 0x00007faf
399 };
400 
401 static const u32 spectre_golden_registers[] =
402 {
403 	0xf000, 0xffff1fff, 0x96940200,
404 	0xf003, 0xffff0001, 0xff000000,
405 	0xf080, 0xfffc0fff, 0x00000100,
406 	0x1bb6, 0x00010101, 0x00010000,
407 	0x260d, 0xf00fffff, 0x00000400,
408 	0x260e, 0xfffffffc, 0x00020200,
409 	0x16ec, 0x000000f0, 0x00000070,
410 	0x16f0, 0xf0311fff, 0x80300000,
411 	0x263e, 0x73773777, 0x12010001,
412 	0x26df, 0x00ff0000, 0x00fc0000,
413 	0xbd2, 0x73773777, 0x12010001,
414 	0x2285, 0xf000003f, 0x00000007,
415 	0x22c9, 0xffffffff, 0x00ffffff,
416 	0xa0d4, 0x3f3f3fff, 0x00000082,
417 	0xa0d5, 0x0000003f, 0x00000000,
418 	0xf9e, 0x00000001, 0x00000002,
419 	0x244f, 0xffff03df, 0x00000004,
420 	0x31da, 0x00000008, 0x00000008,
421 	0x2300, 0x000008ff, 0x00000800,
422 	0x2542, 0x00010000, 0x00010000,
423 	0x2b03, 0xffffffff, 0x54763210,
424 	0x853e, 0x01ff01ff, 0x00000002,
425 	0x8526, 0x007ff800, 0x00200000,
426 	0x8057, 0xffffffff, 0x00000f40,
427 	0xc24d, 0xffffffff, 0x00000001
428 };
429 
430 static const u32 spectre_mgcg_cgcg_init[] =
431 {
432 	0x3108, 0xffffffff, 0xfffffffc,
433 	0xc200, 0xffffffff, 0xe0000000,
434 	0xf0a8, 0xffffffff, 0x00000100,
435 	0xf082, 0xffffffff, 0x00000100,
436 	0xf0b0, 0xffffffff, 0x00000100,
437 	0xf0b2, 0xffffffff, 0x00000100,
438 	0xf0b1, 0xffffffff, 0x00000100,
439 	0x1579, 0xffffffff, 0x00600100,
440 	0xf0a0, 0xffffffff, 0x00000100,
441 	0xf085, 0xffffffff, 0x06000100,
442 	0xf088, 0xffffffff, 0x00000100,
443 	0xf086, 0xffffffff, 0x06000100,
444 	0xf081, 0xffffffff, 0x00000100,
445 	0xf0b8, 0xffffffff, 0x00000100,
446 	0xf089, 0xffffffff, 0x00000100,
447 	0xf080, 0xffffffff, 0x00000100,
448 	0xf08c, 0xffffffff, 0x00000100,
449 	0xf08d, 0xffffffff, 0x00000100,
450 	0xf094, 0xffffffff, 0x00000100,
451 	0xf095, 0xffffffff, 0x00000100,
452 	0xf096, 0xffffffff, 0x00000100,
453 	0xf097, 0xffffffff, 0x00000100,
454 	0xf098, 0xffffffff, 0x00000100,
455 	0xf09f, 0xffffffff, 0x00000100,
456 	0xf09e, 0xffffffff, 0x00000100,
457 	0xf084, 0xffffffff, 0x06000100,
458 	0xf0a4, 0xffffffff, 0x00000100,
459 	0xf09d, 0xffffffff, 0x00000100,
460 	0xf0ad, 0xffffffff, 0x00000100,
461 	0xf0ac, 0xffffffff, 0x00000100,
462 	0xf09c, 0xffffffff, 0x00000100,
463 	0xc200, 0xffffffff, 0xe0000000,
464 	0xf008, 0xffffffff, 0x00010000,
465 	0xf009, 0xffffffff, 0x00030002,
466 	0xf00a, 0xffffffff, 0x00040007,
467 	0xf00b, 0xffffffff, 0x00060005,
468 	0xf00c, 0xffffffff, 0x00090008,
469 	0xf00d, 0xffffffff, 0x00010000,
470 	0xf00e, 0xffffffff, 0x00030002,
471 	0xf00f, 0xffffffff, 0x00040007,
472 	0xf010, 0xffffffff, 0x00060005,
473 	0xf011, 0xffffffff, 0x00090008,
474 	0xf012, 0xffffffff, 0x00010000,
475 	0xf013, 0xffffffff, 0x00030002,
476 	0xf014, 0xffffffff, 0x00040007,
477 	0xf015, 0xffffffff, 0x00060005,
478 	0xf016, 0xffffffff, 0x00090008,
479 	0xf017, 0xffffffff, 0x00010000,
480 	0xf018, 0xffffffff, 0x00030002,
481 	0xf019, 0xffffffff, 0x00040007,
482 	0xf01a, 0xffffffff, 0x00060005,
483 	0xf01b, 0xffffffff, 0x00090008,
484 	0xf01c, 0xffffffff, 0x00010000,
485 	0xf01d, 0xffffffff, 0x00030002,
486 	0xf01e, 0xffffffff, 0x00040007,
487 	0xf01f, 0xffffffff, 0x00060005,
488 	0xf020, 0xffffffff, 0x00090008,
489 	0xf021, 0xffffffff, 0x00010000,
490 	0xf022, 0xffffffff, 0x00030002,
491 	0xf023, 0xffffffff, 0x00040007,
492 	0xf024, 0xffffffff, 0x00060005,
493 	0xf025, 0xffffffff, 0x00090008,
494 	0xf026, 0xffffffff, 0x00010000,
495 	0xf027, 0xffffffff, 0x00030002,
496 	0xf028, 0xffffffff, 0x00040007,
497 	0xf029, 0xffffffff, 0x00060005,
498 	0xf02a, 0xffffffff, 0x00090008,
499 	0xf02b, 0xffffffff, 0x00010000,
500 	0xf02c, 0xffffffff, 0x00030002,
501 	0xf02d, 0xffffffff, 0x00040007,
502 	0xf02e, 0xffffffff, 0x00060005,
503 	0xf02f, 0xffffffff, 0x00090008,
504 	0xf000, 0xffffffff, 0x96e00200,
505 	0x21c2, 0xffffffff, 0x00900100,
506 	0x3109, 0xffffffff, 0x0020003f,
507 	0xe, 0xffffffff, 0x0140001c,
508 	0xf, 0x000f0000, 0x000f0000,
509 	0x88, 0xffffffff, 0xc060000c,
510 	0x89, 0xc0000fff, 0x00000100,
511 	0x3e4, 0xffffffff, 0x00000100,
512 	0x3e6, 0x00000101, 0x00000000,
513 	0x82a, 0xffffffff, 0x00000104,
514 	0x1579, 0xff000fff, 0x00000100,
515 	0xc33, 0xc0000fff, 0x00000104,
516 	0x3079, 0x00000001, 0x00000001,
517 	0x3403, 0xff000ff0, 0x00000100,
518 	0x3603, 0xff000ff0, 0x00000100
519 };
520 
521 static const u32 kalindi_golden_spm_registers[] =
522 {
523 	0xc200, 0xe0ffffff, 0xe0000000
524 };
525 
526 static const u32 kalindi_golden_common_registers[] =
527 {
528 	0x31dc, 0xffffffff, 0x00000800,
529 	0x31dd, 0xffffffff, 0x00000800,
530 	0x31e6, 0xffffffff, 0x00007fbf,
531 	0x31e7, 0xffffffff, 0x00007faf
532 };
533 
534 static const u32 kalindi_golden_registers[] =
535 {
536 	0xf000, 0xffffdfff, 0x6e944040,
537 	0x1579, 0xff607fff, 0xfc000100,
538 	0xf088, 0xff000fff, 0x00000100,
539 	0xf089, 0xff000fff, 0x00000100,
540 	0xf080, 0xfffc0fff, 0x00000100,
541 	0x1bb6, 0x00010101, 0x00010000,
542 	0x260c, 0xffffffff, 0x00000000,
543 	0x260d, 0xf00fffff, 0x00000400,
544 	0x16ec, 0x000000f0, 0x00000070,
545 	0x16f0, 0xf0311fff, 0x80300000,
546 	0x263e, 0x73773777, 0x12010001,
547 	0x263f, 0xffffffff, 0x00000010,
548 	0x26df, 0x00ff0000, 0x00fc0000,
549 	0x200c, 0x00001f0f, 0x0000100a,
550 	0xbd2, 0x73773777, 0x12010001,
551 	0x902, 0x000fffff, 0x000c007f,
552 	0x2285, 0xf000003f, 0x00000007,
553 	0x22c9, 0x3fff3fff, 0x00ffcfff,
554 	0xc281, 0x0000ff0f, 0x00000000,
555 	0xa293, 0x07ffffff, 0x06000000,
556 	0x136, 0x00000fff, 0x00000100,
557 	0xf9e, 0x00000001, 0x00000002,
558 	0x31da, 0x00000008, 0x00000008,
559 	0x2300, 0x000000ff, 0x00000003,
560 	0x853e, 0x01ff01ff, 0x00000002,
561 	0x8526, 0x007ff800, 0x00200000,
562 	0x8057, 0xffffffff, 0x00000f40,
563 	0x2231, 0x001f3ae3, 0x00000082,
564 	0x2235, 0x0000001f, 0x00000010,
565 	0xc24d, 0xffffffff, 0x00000000
566 };
567 
568 static const u32 kalindi_mgcg_cgcg_init[] =
569 {
570 	0x3108, 0xffffffff, 0xfffffffc,
571 	0xc200, 0xffffffff, 0xe0000000,
572 	0xf0a8, 0xffffffff, 0x00000100,
573 	0xf082, 0xffffffff, 0x00000100,
574 	0xf0b0, 0xffffffff, 0x00000100,
575 	0xf0b2, 0xffffffff, 0x00000100,
576 	0xf0b1, 0xffffffff, 0x00000100,
577 	0x1579, 0xffffffff, 0x00600100,
578 	0xf0a0, 0xffffffff, 0x00000100,
579 	0xf085, 0xffffffff, 0x06000100,
580 	0xf088, 0xffffffff, 0x00000100,
581 	0xf086, 0xffffffff, 0x06000100,
582 	0xf081, 0xffffffff, 0x00000100,
583 	0xf0b8, 0xffffffff, 0x00000100,
584 	0xf089, 0xffffffff, 0x00000100,
585 	0xf080, 0xffffffff, 0x00000100,
586 	0xf08c, 0xffffffff, 0x00000100,
587 	0xf08d, 0xffffffff, 0x00000100,
588 	0xf094, 0xffffffff, 0x00000100,
589 	0xf095, 0xffffffff, 0x00000100,
590 	0xf096, 0xffffffff, 0x00000100,
591 	0xf097, 0xffffffff, 0x00000100,
592 	0xf098, 0xffffffff, 0x00000100,
593 	0xf09f, 0xffffffff, 0x00000100,
594 	0xf09e, 0xffffffff, 0x00000100,
595 	0xf084, 0xffffffff, 0x06000100,
596 	0xf0a4, 0xffffffff, 0x00000100,
597 	0xf09d, 0xffffffff, 0x00000100,
598 	0xf0ad, 0xffffffff, 0x00000100,
599 	0xf0ac, 0xffffffff, 0x00000100,
600 	0xf09c, 0xffffffff, 0x00000100,
601 	0xc200, 0xffffffff, 0xe0000000,
602 	0xf008, 0xffffffff, 0x00010000,
603 	0xf009, 0xffffffff, 0x00030002,
604 	0xf00a, 0xffffffff, 0x00040007,
605 	0xf00b, 0xffffffff, 0x00060005,
606 	0xf00c, 0xffffffff, 0x00090008,
607 	0xf00d, 0xffffffff, 0x00010000,
608 	0xf00e, 0xffffffff, 0x00030002,
609 	0xf00f, 0xffffffff, 0x00040007,
610 	0xf010, 0xffffffff, 0x00060005,
611 	0xf011, 0xffffffff, 0x00090008,
612 	0xf000, 0xffffffff, 0x96e00200,
613 	0x21c2, 0xffffffff, 0x00900100,
614 	0x3109, 0xffffffff, 0x0020003f,
615 	0xe, 0xffffffff, 0x0140001c,
616 	0xf, 0x000f0000, 0x000f0000,
617 	0x88, 0xffffffff, 0xc060000c,
618 	0x89, 0xc0000fff, 0x00000100,
619 	0x82a, 0xffffffff, 0x00000104,
620 	0x1579, 0xff000fff, 0x00000100,
621 	0xc33, 0xc0000fff, 0x00000104,
622 	0x3079, 0x00000001, 0x00000001,
623 	0x3403, 0xff000ff0, 0x00000100,
624 	0x3603, 0xff000ff0, 0x00000100
625 };
626 
627 static const u32 hawaii_golden_spm_registers[] =
628 {
629 	0xc200, 0xe0ffffff, 0xe0000000
630 };
631 
632 static const u32 hawaii_golden_common_registers[] =
633 {
634 	0xc200, 0xffffffff, 0xe0000000,
635 	0xa0d4, 0xffffffff, 0x3a00161a,
636 	0xa0d5, 0xffffffff, 0x0000002e,
637 	0x2684, 0xffffffff, 0x00018208,
638 	0x263e, 0xffffffff, 0x12011003
639 };
640 
641 static const u32 hawaii_golden_registers[] =
642 {
643 	0xcd5, 0x00000333, 0x00000333,
644 	0x2684, 0x00010000, 0x00058208,
645 	0x260c, 0xffffffff, 0x00000000,
646 	0x260d, 0xf00fffff, 0x00000400,
647 	0x260e, 0x0002021c, 0x00020200,
648 	0x31e, 0x00000080, 0x00000000,
649 	0x16ec, 0x000000f0, 0x00000070,
650 	0x16f0, 0xf0311fff, 0x80300000,
651 	0xd43, 0x00810000, 0x408af000,
652 	0x1c0c, 0x31000111, 0x00000011,
653 	0xbd2, 0x73773777, 0x12010001,
654 	0x848, 0x0000007f, 0x0000001b,
655 	0x877, 0x00007fb6, 0x00002191,
656 	0xd8a, 0x0000003f, 0x0000000a,
657 	0xd8b, 0x0000003f, 0x0000000a,
658 	0xab9, 0x00073ffe, 0x000022a2,
659 	0x903, 0x000007ff, 0x00000000,
660 	0x22fc, 0x00002001, 0x00000001,
661 	0x22c9, 0xffffffff, 0x00ffffff,
662 	0xc281, 0x0000ff0f, 0x00000000,
663 	0xa293, 0x07ffffff, 0x06000000,
664 	0xf9e, 0x00000001, 0x00000002,
665 	0x31da, 0x00000008, 0x00000008,
666 	0x31dc, 0x00000f00, 0x00000800,
667 	0x31dd, 0x00000f00, 0x00000800,
668 	0x31e6, 0x00ffffff, 0x00ff7fbf,
669 	0x31e7, 0x00ffffff, 0x00ff7faf,
670 	0x2300, 0x000000ff, 0x00000800,
671 	0x390, 0x00001fff, 0x00001fff,
672 	0x2418, 0x0000007f, 0x00000020,
673 	0x2542, 0x00010000, 0x00010000,
674 	0x2b80, 0x00100000, 0x000ff07c,
675 	0x2b05, 0x000003ff, 0x0000000f,
676 	0x2b04, 0xffffffff, 0x7564fdec,
677 	0x2b03, 0xffffffff, 0x3120b9a8,
678 	0x2b02, 0x20000000, 0x0f9c0000
679 };
680 
681 static const u32 hawaii_mgcg_cgcg_init[] =
682 {
683 	0x3108, 0xffffffff, 0xfffffffd,
684 	0xc200, 0xffffffff, 0xe0000000,
685 	0xf0a8, 0xffffffff, 0x00000100,
686 	0xf082, 0xffffffff, 0x00000100,
687 	0xf0b0, 0xffffffff, 0x00000100,
688 	0xf0b2, 0xffffffff, 0x00000100,
689 	0xf0b1, 0xffffffff, 0x00000100,
690 	0x1579, 0xffffffff, 0x00200100,
691 	0xf0a0, 0xffffffff, 0x00000100,
692 	0xf085, 0xffffffff, 0x06000100,
693 	0xf088, 0xffffffff, 0x00000100,
694 	0xf086, 0xffffffff, 0x06000100,
695 	0xf081, 0xffffffff, 0x00000100,
696 	0xf0b8, 0xffffffff, 0x00000100,
697 	0xf089, 0xffffffff, 0x00000100,
698 	0xf080, 0xffffffff, 0x00000100,
699 	0xf08c, 0xffffffff, 0x00000100,
700 	0xf08d, 0xffffffff, 0x00000100,
701 	0xf094, 0xffffffff, 0x00000100,
702 	0xf095, 0xffffffff, 0x00000100,
703 	0xf096, 0xffffffff, 0x00000100,
704 	0xf097, 0xffffffff, 0x00000100,
705 	0xf098, 0xffffffff, 0x00000100,
706 	0xf09f, 0xffffffff, 0x00000100,
707 	0xf09e, 0xffffffff, 0x00000100,
708 	0xf084, 0xffffffff, 0x06000100,
709 	0xf0a4, 0xffffffff, 0x00000100,
710 	0xf09d, 0xffffffff, 0x00000100,
711 	0xf0ad, 0xffffffff, 0x00000100,
712 	0xf0ac, 0xffffffff, 0x00000100,
713 	0xf09c, 0xffffffff, 0x00000100,
714 	0xc200, 0xffffffff, 0xe0000000,
715 	0xf008, 0xffffffff, 0x00010000,
716 	0xf009, 0xffffffff, 0x00030002,
717 	0xf00a, 0xffffffff, 0x00040007,
718 	0xf00b, 0xffffffff, 0x00060005,
719 	0xf00c, 0xffffffff, 0x00090008,
720 	0xf00d, 0xffffffff, 0x00010000,
721 	0xf00e, 0xffffffff, 0x00030002,
722 	0xf00f, 0xffffffff, 0x00040007,
723 	0xf010, 0xffffffff, 0x00060005,
724 	0xf011, 0xffffffff, 0x00090008,
725 	0xf012, 0xffffffff, 0x00010000,
726 	0xf013, 0xffffffff, 0x00030002,
727 	0xf014, 0xffffffff, 0x00040007,
728 	0xf015, 0xffffffff, 0x00060005,
729 	0xf016, 0xffffffff, 0x00090008,
730 	0xf017, 0xffffffff, 0x00010000,
731 	0xf018, 0xffffffff, 0x00030002,
732 	0xf019, 0xffffffff, 0x00040007,
733 	0xf01a, 0xffffffff, 0x00060005,
734 	0xf01b, 0xffffffff, 0x00090008,
735 	0xf01c, 0xffffffff, 0x00010000,
736 	0xf01d, 0xffffffff, 0x00030002,
737 	0xf01e, 0xffffffff, 0x00040007,
738 	0xf01f, 0xffffffff, 0x00060005,
739 	0xf020, 0xffffffff, 0x00090008,
740 	0xf021, 0xffffffff, 0x00010000,
741 	0xf022, 0xffffffff, 0x00030002,
742 	0xf023, 0xffffffff, 0x00040007,
743 	0xf024, 0xffffffff, 0x00060005,
744 	0xf025, 0xffffffff, 0x00090008,
745 	0xf026, 0xffffffff, 0x00010000,
746 	0xf027, 0xffffffff, 0x00030002,
747 	0xf028, 0xffffffff, 0x00040007,
748 	0xf029, 0xffffffff, 0x00060005,
749 	0xf02a, 0xffffffff, 0x00090008,
750 	0xf02b, 0xffffffff, 0x00010000,
751 	0xf02c, 0xffffffff, 0x00030002,
752 	0xf02d, 0xffffffff, 0x00040007,
753 	0xf02e, 0xffffffff, 0x00060005,
754 	0xf02f, 0xffffffff, 0x00090008,
755 	0xf030, 0xffffffff, 0x00010000,
756 	0xf031, 0xffffffff, 0x00030002,
757 	0xf032, 0xffffffff, 0x00040007,
758 	0xf033, 0xffffffff, 0x00060005,
759 	0xf034, 0xffffffff, 0x00090008,
760 	0xf035, 0xffffffff, 0x00010000,
761 	0xf036, 0xffffffff, 0x00030002,
762 	0xf037, 0xffffffff, 0x00040007,
763 	0xf038, 0xffffffff, 0x00060005,
764 	0xf039, 0xffffffff, 0x00090008,
765 	0xf03a, 0xffffffff, 0x00010000,
766 	0xf03b, 0xffffffff, 0x00030002,
767 	0xf03c, 0xffffffff, 0x00040007,
768 	0xf03d, 0xffffffff, 0x00060005,
769 	0xf03e, 0xffffffff, 0x00090008,
770 	0x30c6, 0xffffffff, 0x00020200,
771 	0xcd4, 0xffffffff, 0x00000200,
772 	0x570, 0xffffffff, 0x00000400,
773 	0x157a, 0xffffffff, 0x00000000,
774 	0xbd4, 0xffffffff, 0x00000902,
775 	0xf000, 0xffffffff, 0x96940200,
776 	0x21c2, 0xffffffff, 0x00900100,
777 	0x3109, 0xffffffff, 0x0020003f,
778 	0xe, 0xffffffff, 0x0140001c,
779 	0xf, 0x000f0000, 0x000f0000,
780 	0x88, 0xffffffff, 0xc060000c,
781 	0x89, 0xc0000fff, 0x00000100,
782 	0x3e4, 0xffffffff, 0x00000100,
783 	0x3e6, 0x00000101, 0x00000000,
784 	0x82a, 0xffffffff, 0x00000104,
785 	0x1579, 0xff000fff, 0x00000100,
786 	0xc33, 0xc0000fff, 0x00000104,
787 	0x3079, 0x00000001, 0x00000001,
788 	0x3403, 0xff000ff0, 0x00000100,
789 	0x3603, 0xff000ff0, 0x00000100
790 };
791 
792 static const u32 godavari_golden_registers[] =
793 {
794 	0x1579, 0xff607fff, 0xfc000100,
795 	0x1bb6, 0x00010101, 0x00010000,
796 	0x260c, 0xffffffff, 0x00000000,
797 	0x260c0, 0xf00fffff, 0x00000400,
798 	0x184c, 0xffffffff, 0x00010000,
799 	0x16ec, 0x000000f0, 0x00000070,
800 	0x16f0, 0xf0311fff, 0x80300000,
801 	0x263e, 0x73773777, 0x12010001,
802 	0x263f, 0xffffffff, 0x00000010,
803 	0x200c, 0x00001f0f, 0x0000100a,
804 	0xbd2, 0x73773777, 0x12010001,
805 	0x902, 0x000fffff, 0x000c007f,
806 	0x2285, 0xf000003f, 0x00000007,
807 	0x22c9, 0xffffffff, 0x00ff0fff,
808 	0xc281, 0x0000ff0f, 0x00000000,
809 	0xa293, 0x07ffffff, 0x06000000,
810 	0x136, 0x00000fff, 0x00000100,
811 	0x3405, 0x00010000, 0x00810001,
812 	0x3605, 0x00010000, 0x00810001,
813 	0xf9e, 0x00000001, 0x00000002,
814 	0x31da, 0x00000008, 0x00000008,
815 	0x31dc, 0x00000f00, 0x00000800,
816 	0x31dd, 0x00000f00, 0x00000800,
817 	0x31e6, 0x00ffffff, 0x00ff7fbf,
818 	0x31e7, 0x00ffffff, 0x00ff7faf,
819 	0x2300, 0x000000ff, 0x00000001,
820 	0x853e, 0x01ff01ff, 0x00000002,
821 	0x8526, 0x007ff800, 0x00200000,
822 	0x8057, 0xffffffff, 0x00000f40,
823 	0x2231, 0x001f3ae3, 0x00000082,
824 	0x2235, 0x0000001f, 0x00000010,
825 	0xc24d, 0xffffffff, 0x00000000
826 };
827 
828 static void cik_init_golden_registers(struct amdgpu_device *adev)
829 {
830 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
831 	mutex_lock(&adev->grbm_idx_mutex);
832 
833 	switch (adev->asic_type) {
834 	case CHIP_BONAIRE:
835 		amdgpu_device_program_register_sequence(adev,
836 							bonaire_mgcg_cgcg_init,
837 							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
838 		amdgpu_device_program_register_sequence(adev,
839 							bonaire_golden_registers,
840 							ARRAY_SIZE(bonaire_golden_registers));
841 		amdgpu_device_program_register_sequence(adev,
842 							bonaire_golden_common_registers,
843 							ARRAY_SIZE(bonaire_golden_common_registers));
844 		amdgpu_device_program_register_sequence(adev,
845 							bonaire_golden_spm_registers,
846 							ARRAY_SIZE(bonaire_golden_spm_registers));
847 		break;
848 	case CHIP_KABINI:
849 		amdgpu_device_program_register_sequence(adev,
850 							kalindi_mgcg_cgcg_init,
851 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
852 		amdgpu_device_program_register_sequence(adev,
853 							kalindi_golden_registers,
854 							ARRAY_SIZE(kalindi_golden_registers));
855 		amdgpu_device_program_register_sequence(adev,
856 							kalindi_golden_common_registers,
857 							ARRAY_SIZE(kalindi_golden_common_registers));
858 		amdgpu_device_program_register_sequence(adev,
859 							kalindi_golden_spm_registers,
860 							ARRAY_SIZE(kalindi_golden_spm_registers));
861 		break;
862 	case CHIP_MULLINS:
863 		amdgpu_device_program_register_sequence(adev,
864 							kalindi_mgcg_cgcg_init,
865 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
866 		amdgpu_device_program_register_sequence(adev,
867 							godavari_golden_registers,
868 							ARRAY_SIZE(godavari_golden_registers));
869 		amdgpu_device_program_register_sequence(adev,
870 							kalindi_golden_common_registers,
871 							ARRAY_SIZE(kalindi_golden_common_registers));
872 		amdgpu_device_program_register_sequence(adev,
873 							kalindi_golden_spm_registers,
874 							ARRAY_SIZE(kalindi_golden_spm_registers));
875 		break;
876 	case CHIP_KAVERI:
877 		amdgpu_device_program_register_sequence(adev,
878 							spectre_mgcg_cgcg_init,
879 							ARRAY_SIZE(spectre_mgcg_cgcg_init));
880 		amdgpu_device_program_register_sequence(adev,
881 							spectre_golden_registers,
882 							ARRAY_SIZE(spectre_golden_registers));
883 		amdgpu_device_program_register_sequence(adev,
884 							spectre_golden_common_registers,
885 							ARRAY_SIZE(spectre_golden_common_registers));
886 		amdgpu_device_program_register_sequence(adev,
887 							spectre_golden_spm_registers,
888 							ARRAY_SIZE(spectre_golden_spm_registers));
889 		break;
890 	case CHIP_HAWAII:
891 		amdgpu_device_program_register_sequence(adev,
892 							hawaii_mgcg_cgcg_init,
893 							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
894 		amdgpu_device_program_register_sequence(adev,
895 							hawaii_golden_registers,
896 							ARRAY_SIZE(hawaii_golden_registers));
897 		amdgpu_device_program_register_sequence(adev,
898 							hawaii_golden_common_registers,
899 							ARRAY_SIZE(hawaii_golden_common_registers));
900 		amdgpu_device_program_register_sequence(adev,
901 							hawaii_golden_spm_registers,
902 							ARRAY_SIZE(hawaii_golden_spm_registers));
903 		break;
904 	default:
905 		break;
906 	}
907 	mutex_unlock(&adev->grbm_idx_mutex);
908 }
909 
910 /**
911  * cik_get_xclk - get the xclk
912  *
913  * @adev: amdgpu_device pointer
914  *
915  * Returns the reference clock used by the gfx engine
916  * (CIK).
917  */
918 static u32 cik_get_xclk(struct amdgpu_device *adev)
919 {
920 	u32 reference_clock = adev->clock.spll.reference_freq;
921 
922 	if (adev->flags & AMD_IS_APU) {
923 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
924 			return reference_clock / 2;
925 	} else {
926 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
927 			return reference_clock / 4;
928 	}
929 	return reference_clock;
930 }
931 
932 /**
933  * cik_srbm_select - select specific register instances
934  *
935  * @adev: amdgpu_device pointer
936  * @me: selected ME (micro engine)
937  * @pipe: pipe
938  * @queue: queue
939  * @vmid: VMID
940  *
941  * Switches the currently active registers instances.  Some
942  * registers are instanced per VMID, others are instanced per
943  * me/pipe/queue combination.
944  */
945 void cik_srbm_select(struct amdgpu_device *adev,
946 		     u32 me, u32 pipe, u32 queue, u32 vmid)
947 {
948 	u32 srbm_gfx_cntl =
949 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
950 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
951 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
952 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
953 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
954 }
955 
956 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
957 {
958 	uint32_t tmp;
959 
960 	tmp = RREG32(mmCONFIG_CNTL);
961 	if (!state)
962 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
963 	else
964 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
965 	WREG32(mmCONFIG_CNTL, tmp);
966 }
967 
968 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
969 {
970 	u32 bus_cntl;
971 	u32 d1vga_control = 0;
972 	u32 d2vga_control = 0;
973 	u32 vga_render_control = 0;
974 	u32 rom_cntl;
975 	bool r;
976 
977 	bus_cntl = RREG32(mmBUS_CNTL);
978 	if (adev->mode_info.num_crtc) {
979 		d1vga_control = RREG32(mmD1VGA_CONTROL);
980 		d2vga_control = RREG32(mmD2VGA_CONTROL);
981 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
982 	}
983 	rom_cntl = RREG32_SMC(ixROM_CNTL);
984 
985 	/* enable the rom */
986 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
987 	if (adev->mode_info.num_crtc) {
988 		/* Disable VGA mode */
989 		WREG32(mmD1VGA_CONTROL,
990 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
991 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
992 		WREG32(mmD2VGA_CONTROL,
993 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
994 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
995 		WREG32(mmVGA_RENDER_CONTROL,
996 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
997 	}
998 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
999 
1000 	r = amdgpu_read_bios(adev);
1001 
1002 	/* restore regs */
1003 	WREG32(mmBUS_CNTL, bus_cntl);
1004 	if (adev->mode_info.num_crtc) {
1005 		WREG32(mmD1VGA_CONTROL, d1vga_control);
1006 		WREG32(mmD2VGA_CONTROL, d2vga_control);
1007 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1008 	}
1009 	WREG32_SMC(ixROM_CNTL, rom_cntl);
1010 	return r;
1011 }
1012 
1013 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1014 				   u8 *bios, u32 length_bytes)
1015 {
1016 	u32 *dw_ptr;
1017 	unsigned long flags;
1018 	u32 i, length_dw;
1019 
1020 	if (bios == NULL)
1021 		return false;
1022 	if (length_bytes == 0)
1023 		return false;
1024 	/* APU vbios image is part of sbios image */
1025 	if (adev->flags & AMD_IS_APU)
1026 		return false;
1027 
1028 	dw_ptr = (u32 *)bios;
1029 	length_dw = ALIGN(length_bytes, 4) / 4;
1030 	/* take the smc lock since we are using the smc index */
1031 	spin_lock_irqsave(&adev->reg.smc.lock, flags);
1032 	/* set rom index to 0 */
1033 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1034 	WREG32(mmSMC_IND_DATA_0, 0);
1035 	/* set index to data for continous read */
1036 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1037 	for (i = 0; i < length_dw; i++)
1038 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1039 	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
1040 
1041 	return true;
1042 }
1043 
1044 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1045 	{mmGRBM_STATUS},
1046 	{mmGRBM_STATUS2},
1047 	{mmGRBM_STATUS_SE0},
1048 	{mmGRBM_STATUS_SE1},
1049 	{mmGRBM_STATUS_SE2},
1050 	{mmGRBM_STATUS_SE3},
1051 	{mmSRBM_STATUS},
1052 	{mmSRBM_STATUS2},
1053 	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1054 	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1055 	{mmCP_STAT},
1056 	{mmCP_STALLED_STAT1},
1057 	{mmCP_STALLED_STAT2},
1058 	{mmCP_STALLED_STAT3},
1059 	{mmCP_CPF_BUSY_STAT},
1060 	{mmCP_CPF_STALLED_STAT1},
1061 	{mmCP_CPF_STATUS},
1062 	{mmCP_CPC_BUSY_STAT},
1063 	{mmCP_CPC_STALLED_STAT1},
1064 	{mmCP_CPC_STATUS},
1065 	{mmGB_ADDR_CONFIG},
1066 	{mmMC_ARB_RAMCFG},
1067 	{mmGB_TILE_MODE0},
1068 	{mmGB_TILE_MODE1},
1069 	{mmGB_TILE_MODE2},
1070 	{mmGB_TILE_MODE3},
1071 	{mmGB_TILE_MODE4},
1072 	{mmGB_TILE_MODE5},
1073 	{mmGB_TILE_MODE6},
1074 	{mmGB_TILE_MODE7},
1075 	{mmGB_TILE_MODE8},
1076 	{mmGB_TILE_MODE9},
1077 	{mmGB_TILE_MODE10},
1078 	{mmGB_TILE_MODE11},
1079 	{mmGB_TILE_MODE12},
1080 	{mmGB_TILE_MODE13},
1081 	{mmGB_TILE_MODE14},
1082 	{mmGB_TILE_MODE15},
1083 	{mmGB_TILE_MODE16},
1084 	{mmGB_TILE_MODE17},
1085 	{mmGB_TILE_MODE18},
1086 	{mmGB_TILE_MODE19},
1087 	{mmGB_TILE_MODE20},
1088 	{mmGB_TILE_MODE21},
1089 	{mmGB_TILE_MODE22},
1090 	{mmGB_TILE_MODE23},
1091 	{mmGB_TILE_MODE24},
1092 	{mmGB_TILE_MODE25},
1093 	{mmGB_TILE_MODE26},
1094 	{mmGB_TILE_MODE27},
1095 	{mmGB_TILE_MODE28},
1096 	{mmGB_TILE_MODE29},
1097 	{mmGB_TILE_MODE30},
1098 	{mmGB_TILE_MODE31},
1099 	{mmGB_MACROTILE_MODE0},
1100 	{mmGB_MACROTILE_MODE1},
1101 	{mmGB_MACROTILE_MODE2},
1102 	{mmGB_MACROTILE_MODE3},
1103 	{mmGB_MACROTILE_MODE4},
1104 	{mmGB_MACROTILE_MODE5},
1105 	{mmGB_MACROTILE_MODE6},
1106 	{mmGB_MACROTILE_MODE7},
1107 	{mmGB_MACROTILE_MODE8},
1108 	{mmGB_MACROTILE_MODE9},
1109 	{mmGB_MACROTILE_MODE10},
1110 	{mmGB_MACROTILE_MODE11},
1111 	{mmGB_MACROTILE_MODE12},
1112 	{mmGB_MACROTILE_MODE13},
1113 	{mmGB_MACROTILE_MODE14},
1114 	{mmGB_MACROTILE_MODE15},
1115 	{mmCC_RB_BACKEND_DISABLE, true},
1116 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1117 	{mmGB_BACKEND_MAP, false},
1118 	{mmPA_SC_RASTER_CONFIG, true},
1119 	{mmPA_SC_RASTER_CONFIG_1, true},
1120 };
1121 
1122 
1123 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1124 				       bool indexed, u32 se_num,
1125 				       u32 sh_num, u32 reg_offset)
1126 {
1127 	if (indexed) {
1128 		uint32_t val;
1129 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1130 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1131 
1132 		switch (reg_offset) {
1133 		case mmCC_RB_BACKEND_DISABLE:
1134 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1135 		case mmGC_USER_RB_BACKEND_DISABLE:
1136 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1137 		case mmPA_SC_RASTER_CONFIG:
1138 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1139 		case mmPA_SC_RASTER_CONFIG_1:
1140 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1141 		}
1142 
1143 		mutex_lock(&adev->grbm_idx_mutex);
1144 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1145 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1146 
1147 		val = RREG32(reg_offset);
1148 
1149 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1150 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1151 		mutex_unlock(&adev->grbm_idx_mutex);
1152 		return val;
1153 	} else {
1154 		unsigned idx;
1155 
1156 		switch (reg_offset) {
1157 		case mmGB_ADDR_CONFIG:
1158 			return adev->gfx.config.gb_addr_config;
1159 		case mmMC_ARB_RAMCFG:
1160 			return adev->gfx.config.mc_arb_ramcfg;
1161 		case mmGB_TILE_MODE0:
1162 		case mmGB_TILE_MODE1:
1163 		case mmGB_TILE_MODE2:
1164 		case mmGB_TILE_MODE3:
1165 		case mmGB_TILE_MODE4:
1166 		case mmGB_TILE_MODE5:
1167 		case mmGB_TILE_MODE6:
1168 		case mmGB_TILE_MODE7:
1169 		case mmGB_TILE_MODE8:
1170 		case mmGB_TILE_MODE9:
1171 		case mmGB_TILE_MODE10:
1172 		case mmGB_TILE_MODE11:
1173 		case mmGB_TILE_MODE12:
1174 		case mmGB_TILE_MODE13:
1175 		case mmGB_TILE_MODE14:
1176 		case mmGB_TILE_MODE15:
1177 		case mmGB_TILE_MODE16:
1178 		case mmGB_TILE_MODE17:
1179 		case mmGB_TILE_MODE18:
1180 		case mmGB_TILE_MODE19:
1181 		case mmGB_TILE_MODE20:
1182 		case mmGB_TILE_MODE21:
1183 		case mmGB_TILE_MODE22:
1184 		case mmGB_TILE_MODE23:
1185 		case mmGB_TILE_MODE24:
1186 		case mmGB_TILE_MODE25:
1187 		case mmGB_TILE_MODE26:
1188 		case mmGB_TILE_MODE27:
1189 		case mmGB_TILE_MODE28:
1190 		case mmGB_TILE_MODE29:
1191 		case mmGB_TILE_MODE30:
1192 		case mmGB_TILE_MODE31:
1193 			idx = (reg_offset - mmGB_TILE_MODE0);
1194 			return adev->gfx.config.tile_mode_array[idx];
1195 		case mmGB_MACROTILE_MODE0:
1196 		case mmGB_MACROTILE_MODE1:
1197 		case mmGB_MACROTILE_MODE2:
1198 		case mmGB_MACROTILE_MODE3:
1199 		case mmGB_MACROTILE_MODE4:
1200 		case mmGB_MACROTILE_MODE5:
1201 		case mmGB_MACROTILE_MODE6:
1202 		case mmGB_MACROTILE_MODE7:
1203 		case mmGB_MACROTILE_MODE8:
1204 		case mmGB_MACROTILE_MODE9:
1205 		case mmGB_MACROTILE_MODE10:
1206 		case mmGB_MACROTILE_MODE11:
1207 		case mmGB_MACROTILE_MODE12:
1208 		case mmGB_MACROTILE_MODE13:
1209 		case mmGB_MACROTILE_MODE14:
1210 		case mmGB_MACROTILE_MODE15:
1211 			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1212 			return adev->gfx.config.macrotile_mode_array[idx];
1213 		default:
1214 			return RREG32(reg_offset);
1215 		}
1216 	}
1217 }
1218 
1219 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1220 			     u32 sh_num, u32 reg_offset, u32 *value)
1221 {
1222 	uint32_t i;
1223 
1224 	*value = 0;
1225 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1226 		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1227 
1228 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1229 			continue;
1230 
1231 		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1232 						reg_offset);
1233 		return 0;
1234 	}
1235 	return -EINVAL;
1236 }
1237 
1238 struct kv_reset_save_regs {
1239 	u32 gmcon_reng_execute;
1240 	u32 gmcon_misc;
1241 	u32 gmcon_misc3;
1242 };
1243 
1244 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1245 				   struct kv_reset_save_regs *save)
1246 {
1247 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1248 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1249 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1250 
1251 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1252 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1253 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1254 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1255 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1256 }
1257 
1258 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1259 				      struct kv_reset_save_regs *save)
1260 {
1261 	int i;
1262 
1263 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1264 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1265 
1266 	for (i = 0; i < 5; i++)
1267 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1268 
1269 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1270 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1271 
1272 	for (i = 0; i < 5; i++)
1273 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1274 
1275 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1276 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1277 
1278 	for (i = 0; i < 5; i++)
1279 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1280 
1281 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1282 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1283 
1284 	for (i = 0; i < 5; i++)
1285 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1286 
1287 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1288 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1289 
1290 	for (i = 0; i < 5; i++)
1291 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1292 
1293 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1294 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1295 
1296 	for (i = 0; i < 5; i++)
1297 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1298 
1299 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1300 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1301 
1302 	for (i = 0; i < 5; i++)
1303 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1304 
1305 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1306 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1307 
1308 	for (i = 0; i < 5; i++)
1309 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1310 
1311 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1312 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1313 
1314 	for (i = 0; i < 5; i++)
1315 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1316 
1317 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1318 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1319 
1320 	for (i = 0; i < 5; i++)
1321 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1322 
1323 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1324 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1325 
1326 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1327 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1328 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1329 }
1330 
1331 /**
1332  * cik_asic_pci_config_reset - soft reset GPU
1333  *
1334  * @adev: amdgpu_device pointer
1335  *
1336  * Use PCI Config method to reset the GPU.
1337  *
1338  * Returns 0 for success.
1339  */
1340 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1341 {
1342 	struct kv_reset_save_regs kv_save = { 0 };
1343 	u32 i;
1344 	int r = -EINVAL;
1345 
1346 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1347 
1348 	if (adev->flags & AMD_IS_APU)
1349 		kv_save_regs_for_reset(adev, &kv_save);
1350 
1351 	/* disable BM */
1352 	pci_clear_master(adev->pdev);
1353 	/* reset */
1354 	amdgpu_device_pci_config_reset(adev);
1355 
1356 	udelay(100);
1357 
1358 	/* wait for asic to come out of reset */
1359 	for (i = 0; i < adev->usec_timeout; i++) {
1360 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1361 			/* enable BM */
1362 			pci_set_master(adev->pdev);
1363 			adev->has_hw_reset = true;
1364 			r = 0;
1365 			break;
1366 		}
1367 		udelay(1);
1368 	}
1369 
1370 	/* does asic init need to be run first??? */
1371 	if (adev->flags & AMD_IS_APU)
1372 		kv_restore_regs_for_reset(adev, &kv_save);
1373 
1374 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1375 
1376 	return r;
1377 }
1378 
1379 static int cik_asic_supports_baco(struct amdgpu_device *adev)
1380 {
1381 	switch (adev->asic_type) {
1382 	case CHIP_BONAIRE:
1383 	case CHIP_HAWAII:
1384 		return amdgpu_dpm_is_baco_supported(adev);
1385 	default:
1386 		return 0;
1387 	}
1388 }
1389 
1390 static enum amd_reset_method
1391 cik_asic_reset_method(struct amdgpu_device *adev)
1392 {
1393 	bool baco_reset;
1394 
1395 	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1396 	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1397 		return amdgpu_reset_method;
1398 
1399 	if (amdgpu_reset_method != -1)
1400 		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1401 				  amdgpu_reset_method);
1402 
1403 	switch (adev->asic_type) {
1404 	case CHIP_BONAIRE:
1405 	case CHIP_HAWAII:
1406 		baco_reset = cik_asic_supports_baco(adev);
1407 		break;
1408 	default:
1409 		baco_reset = false;
1410 		break;
1411 	}
1412 
1413 	if (baco_reset)
1414 		return AMD_RESET_METHOD_BACO;
1415 	else
1416 		return AMD_RESET_METHOD_LEGACY;
1417 }
1418 
1419 /**
1420  * cik_asic_reset - soft reset GPU
1421  *
1422  * @adev: amdgpu_device pointer
1423  *
1424  * Look up which blocks are hung and attempt
1425  * to reset them.
1426  * Returns 0 for success.
1427  */
1428 static int cik_asic_reset(struct amdgpu_device *adev)
1429 {
1430 	int r;
1431 
1432 	/* APUs don't have full asic reset */
1433 	if (adev->flags & AMD_IS_APU)
1434 		return 0;
1435 
1436 	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1437 		dev_info(adev->dev, "BACO reset\n");
1438 		r = amdgpu_dpm_baco_reset(adev);
1439 	} else {
1440 		dev_info(adev->dev, "PCI CONFIG reset\n");
1441 		r = cik_asic_pci_config_reset(adev);
1442 	}
1443 
1444 	return r;
1445 }
1446 
1447 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1448 {
1449 	return RREG32(mmCONFIG_MEMSIZE);
1450 }
1451 
1452 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1453 			      u32 cntl_reg, u32 status_reg)
1454 {
1455 	int r, i;
1456 	struct atom_clock_dividers dividers;
1457 	uint32_t tmp;
1458 
1459 	r = amdgpu_atombios_get_clock_dividers(adev,
1460 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1461 					       clock, false, &dividers);
1462 	if (r)
1463 		return r;
1464 
1465 	tmp = RREG32_SMC(cntl_reg);
1466 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1467 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1468 	tmp |= dividers.post_divider;
1469 	WREG32_SMC(cntl_reg, tmp);
1470 
1471 	for (i = 0; i < 100; i++) {
1472 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1473 			break;
1474 		mdelay(10);
1475 	}
1476 	if (i == 100)
1477 		return -ETIMEDOUT;
1478 
1479 	return 0;
1480 }
1481 
1482 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1483 {
1484 	int r = 0;
1485 
1486 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1487 	if (r)
1488 		return r;
1489 
1490 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1491 	return r;
1492 }
1493 
1494 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1495 {
1496 	int r, i;
1497 	struct atom_clock_dividers dividers;
1498 	u32 tmp;
1499 
1500 	r = amdgpu_atombios_get_clock_dividers(adev,
1501 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1502 					       ecclk, false, &dividers);
1503 	if (r)
1504 		return r;
1505 
1506 	for (i = 0; i < 100; i++) {
1507 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1508 			break;
1509 		mdelay(10);
1510 	}
1511 	if (i == 100)
1512 		return -ETIMEDOUT;
1513 
1514 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1515 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1516 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1517 	tmp |= dividers.post_divider;
1518 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1519 
1520 	for (i = 0; i < 100; i++) {
1521 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1522 			break;
1523 		mdelay(10);
1524 	}
1525 	if (i == 100)
1526 		return -ETIMEDOUT;
1527 
1528 	return 0;
1529 }
1530 
1531 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1532 {
1533 	struct pci_dev *root = adev->pdev->bus->self;
1534 	u32 speed_cntl, current_data_rate;
1535 	int i;
1536 	u16 tmp16;
1537 
1538 	if (pci_is_root_bus(adev->pdev->bus))
1539 		return;
1540 
1541 	if (amdgpu_pcie_gen2 == 0)
1542 		return;
1543 
1544 	if (adev->flags & AMD_IS_APU)
1545 		return;
1546 
1547 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1548 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1549 		return;
1550 
1551 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1552 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1553 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1554 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1555 		if (current_data_rate == 2) {
1556 			drm_info(adev_to_drm(adev), "PCIE gen 3 link speeds already enabled\n");
1557 			return;
1558 		}
1559 		drm_info(adev_to_drm(adev), "enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1560 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1561 		if (current_data_rate == 1) {
1562 			drm_info(adev_to_drm(adev), "PCIE gen 2 link speeds already enabled\n");
1563 			return;
1564 		}
1565 		drm_info(adev_to_drm(adev), "enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1566 	}
1567 
1568 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1569 		return;
1570 
1571 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1572 		/* re-try equalization if gen3 is not already enabled */
1573 		if (current_data_rate != 2) {
1574 			u16 bridge_cfg, gpu_cfg;
1575 			u16 bridge_cfg2, gpu_cfg2;
1576 			u32 max_lw, current_lw, tmp;
1577 
1578 			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1579 			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1580 
1581 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1582 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1583 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1584 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1585 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1586 
1587 			if (current_lw < max_lw) {
1588 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1589 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1590 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1591 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1592 					tmp |= (max_lw <<
1593 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1594 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1595 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1596 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1597 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1598 				}
1599 			}
1600 
1601 			for (i = 0; i < 10; i++) {
1602 				/* check status */
1603 				pcie_capability_read_word(adev->pdev,
1604 							  PCI_EXP_DEVSTA,
1605 							  &tmp16);
1606 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1607 					break;
1608 
1609 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1610 							  &bridge_cfg);
1611 				pcie_capability_read_word(adev->pdev,
1612 							  PCI_EXP_LNKCTL,
1613 							  &gpu_cfg);
1614 
1615 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1616 							  &bridge_cfg2);
1617 				pcie_capability_read_word(adev->pdev,
1618 							  PCI_EXP_LNKCTL2,
1619 							  &gpu_cfg2);
1620 
1621 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1622 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1623 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1624 
1625 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1626 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1627 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1628 
1629 				msleep(100);
1630 
1631 				/* linkctl */
1632 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1633 								   PCI_EXP_LNKCTL_HAWD,
1634 								   bridge_cfg &
1635 								   PCI_EXP_LNKCTL_HAWD);
1636 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1637 								   PCI_EXP_LNKCTL_HAWD,
1638 								   gpu_cfg &
1639 								   PCI_EXP_LNKCTL_HAWD);
1640 
1641 				/* linkctl2 */
1642 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
1643 								   PCI_EXP_LNKCTL2_ENTER_COMP |
1644 								   PCI_EXP_LNKCTL2_TX_MARGIN,
1645 								   bridge_cfg2 &
1646 								   (PCI_EXP_LNKCTL2_ENTER_COMP |
1647 								    PCI_EXP_LNKCTL2_TX_MARGIN));
1648 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1649 								   PCI_EXP_LNKCTL2_ENTER_COMP |
1650 								   PCI_EXP_LNKCTL2_TX_MARGIN,
1651 								   gpu_cfg2 &
1652 								   (PCI_EXP_LNKCTL2_ENTER_COMP |
1653 								    PCI_EXP_LNKCTL2_TX_MARGIN));
1654 
1655 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1656 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1657 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1658 			}
1659 		}
1660 	}
1661 
1662 	/* set the link speed */
1663 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1664 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1665 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1666 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1667 
1668 	tmp16 = 0;
1669 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1670 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1671 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1672 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1673 	else
1674 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1675 	pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
1676 					   PCI_EXP_LNKCTL2_TLS, tmp16);
1677 
1678 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1679 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1680 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1681 
1682 	for (i = 0; i < adev->usec_timeout; i++) {
1683 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1684 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1685 			break;
1686 		udelay(1);
1687 	}
1688 }
1689 
1690 static void cik_program_aspm(struct amdgpu_device *adev)
1691 {
1692 	u32 data, orig;
1693 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1694 	bool disable_clkreq = false;
1695 
1696 	if (!amdgpu_device_should_use_aspm(adev))
1697 		return;
1698 
1699 	if (pci_is_root_bus(adev->pdev->bus))
1700 		return;
1701 
1702 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1703 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1704 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1705 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1706 	if (orig != data)
1707 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1708 
1709 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1710 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1711 	if (orig != data)
1712 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1713 
1714 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1715 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1716 	if (orig != data)
1717 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1718 
1719 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1720 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1721 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1722 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1723 	if (!disable_l0s)
1724 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1725 
1726 	if (!disable_l1) {
1727 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1728 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1729 		if (orig != data)
1730 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1731 
1732 		if (!disable_plloff_in_l1) {
1733 			bool clk_req_support;
1734 
1735 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1736 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1737 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1738 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1739 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1740 			if (orig != data)
1741 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1742 
1743 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1744 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1745 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1746 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1747 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1748 			if (orig != data)
1749 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1750 
1751 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1752 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1753 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1754 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1755 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1756 			if (orig != data)
1757 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1758 
1759 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1760 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1761 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1762 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1763 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1764 			if (orig != data)
1765 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1766 
1767 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1768 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1769 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1770 			if (orig != data)
1771 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1772 
1773 			if (!disable_clkreq) {
1774 				struct pci_dev *root = adev->pdev->bus->self;
1775 				u32 lnkcap;
1776 
1777 				clk_req_support = false;
1778 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1779 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1780 					clk_req_support = true;
1781 			} else {
1782 				clk_req_support = false;
1783 			}
1784 
1785 			if (clk_req_support) {
1786 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1787 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1788 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1789 				if (orig != data)
1790 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1791 
1792 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1793 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1794 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1795 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1796 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1797 				if (orig != data)
1798 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1799 
1800 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1801 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1802 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1803 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1804 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1805 				if (orig != data)
1806 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1807 
1808 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1809 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1810 				if (orig != data)
1811 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1812 
1813 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1814 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1815 				if (orig != data)
1816 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1817 
1818 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1819 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1820 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1821 				if (orig != data)
1822 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1823 			}
1824 		}
1825 	} else {
1826 		if (orig != data)
1827 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1828 	}
1829 
1830 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1831 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1832 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1833 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1834 	if (orig != data)
1835 		WREG32_PCIE(ixPCIE_CNTL2, data);
1836 
1837 	if (!disable_l0s) {
1838 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1839 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1840 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1841 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1842 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1843 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1844 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1845 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1846 				if (orig != data)
1847 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1848 			}
1849 		}
1850 	}
1851 }
1852 
1853 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1854 {
1855 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1856 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1857 }
1858 
1859 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1860 {
1861 	if (!ring || !ring->funcs->emit_wreg) {
1862 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1863 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1864 	} else {
1865 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1866 	}
1867 }
1868 
1869 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1870 			       struct amdgpu_ring *ring)
1871 {
1872 	if (!ring || !ring->funcs->emit_wreg) {
1873 		WREG32(mmHDP_DEBUG0, 1);
1874 		RREG32(mmHDP_DEBUG0);
1875 	} else {
1876 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1877 	}
1878 }
1879 
1880 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1881 			       uint64_t *count1)
1882 {
1883 	uint32_t perfctr = 0;
1884 	uint64_t cnt0_of, cnt1_of;
1885 	int tmp;
1886 
1887 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1888 	 * that may or may not be different from their GPU counterparts
1889 	 */
1890 	if (adev->flags & AMD_IS_APU)
1891 		return;
1892 
1893 	/* Set the 2 events that we wish to watch, defined above */
1894 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1895 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1896 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1897 
1898 	/* Write to enable desired perf counters */
1899 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1900 	/* Zero out and enable the perf counters
1901 	 * Write 0x5:
1902 	 * Bit 0 = Start all counters(1)
1903 	 * Bit 2 = Global counter reset enable(1)
1904 	 */
1905 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1906 
1907 	msleep(1000);
1908 
1909 	/* Load the shadow and disable the perf counters
1910 	 * Write 0x2:
1911 	 * Bit 0 = Stop counters(0)
1912 	 * Bit 1 = Load the shadow counters(1)
1913 	 */
1914 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1915 
1916 	/* Read register values to get any >32bit overflow */
1917 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1918 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1919 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1920 
1921 	/* Get the values and add the overflow */
1922 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1923 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1924 }
1925 
1926 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1927 {
1928 	u32 clock_cntl, pc;
1929 
1930 	if (adev->flags & AMD_IS_APU)
1931 		return false;
1932 
1933 	/* check if the SMC is already running */
1934 	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1935 	pc = RREG32_SMC(ixSMC_PC_C);
1936 	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1937 	    (0x20100 <= pc))
1938 		return true;
1939 
1940 	return false;
1941 }
1942 
1943 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1944 {
1945 	uint64_t nak_r, nak_g;
1946 
1947 	/* Get the number of NAKs received and generated */
1948 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1949 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1950 
1951 	/* Add the total number of NAKs, i.e the number of replays */
1952 	return (nak_r + nak_g);
1953 }
1954 
1955 static const struct amdgpu_asic_funcs cik_asic_funcs =
1956 {
1957 	.read_disabled_bios = &cik_read_disabled_bios,
1958 	.read_bios_from_rom = &cik_read_bios_from_rom,
1959 	.read_register = &cik_read_register,
1960 	.reset = &cik_asic_reset,
1961 	.reset_method = &cik_asic_reset_method,
1962 	.set_vga_state = &cik_vga_set_state,
1963 	.get_xclk = &cik_get_xclk,
1964 	.set_uvd_clocks = &cik_set_uvd_clocks,
1965 	.set_vce_clocks = &cik_set_vce_clocks,
1966 	.get_config_memsize = &cik_get_config_memsize,
1967 	.flush_hdp = &cik_flush_hdp,
1968 	.invalidate_hdp = &cik_invalidate_hdp,
1969 	.init_doorbell_index = &legacy_doorbell_index_init,
1970 	.get_pcie_usage = &cik_get_pcie_usage,
1971 	.need_reset_on_init = &cik_need_reset_on_init,
1972 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1973 	.supports_baco = &cik_asic_supports_baco,
1974 	.query_video_codecs = &cik_query_video_codecs,
1975 };
1976 
1977 static int cik_common_early_init(struct amdgpu_ip_block *ip_block)
1978 {
1979 	struct amdgpu_device *adev = ip_block->adev;
1980 
1981 	adev->reg.smc.rreg = cik_smc_rreg;
1982 	adev->reg.smc.wreg = cik_smc_wreg;
1983 	adev->reg.pcie.rreg = &cik_pcie_rreg;
1984 	adev->reg.pcie.wreg = &cik_pcie_wreg;
1985 	adev->reg.uvd_ctx.rreg = &cik_uvd_ctx_rreg;
1986 	adev->reg.uvd_ctx.wreg = &cik_uvd_ctx_wreg;
1987 	adev->reg.didt.rreg = &cik_didt_rreg;
1988 	adev->reg.didt.wreg = &cik_didt_wreg;
1989 
1990 	adev->asic_funcs = &cik_asic_funcs;
1991 
1992 	adev->rev_id = cik_get_rev_id(adev);
1993 	adev->external_rev_id = 0xFF;
1994 	switch (adev->asic_type) {
1995 	case CHIP_BONAIRE:
1996 		adev->cg_flags =
1997 			AMD_CG_SUPPORT_GFX_MGCG |
1998 			AMD_CG_SUPPORT_GFX_MGLS |
1999 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2000 			AMD_CG_SUPPORT_GFX_CGLS |
2001 			AMD_CG_SUPPORT_GFX_CGTS |
2002 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2003 			AMD_CG_SUPPORT_GFX_CP_LS |
2004 			AMD_CG_SUPPORT_MC_LS |
2005 			AMD_CG_SUPPORT_MC_MGCG |
2006 			AMD_CG_SUPPORT_SDMA_MGCG |
2007 			AMD_CG_SUPPORT_SDMA_LS |
2008 			AMD_CG_SUPPORT_BIF_LS |
2009 			AMD_CG_SUPPORT_VCE_MGCG |
2010 			AMD_CG_SUPPORT_UVD_MGCG |
2011 			AMD_CG_SUPPORT_HDP_LS |
2012 			AMD_CG_SUPPORT_HDP_MGCG;
2013 		adev->pg_flags = 0;
2014 		adev->external_rev_id = adev->rev_id + 0x14;
2015 		break;
2016 	case CHIP_HAWAII:
2017 		adev->cg_flags =
2018 			AMD_CG_SUPPORT_GFX_MGCG |
2019 			AMD_CG_SUPPORT_GFX_MGLS |
2020 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2021 			AMD_CG_SUPPORT_GFX_CGLS |
2022 			AMD_CG_SUPPORT_GFX_CGTS |
2023 			AMD_CG_SUPPORT_GFX_CP_LS |
2024 			AMD_CG_SUPPORT_MC_LS |
2025 			AMD_CG_SUPPORT_MC_MGCG |
2026 			AMD_CG_SUPPORT_SDMA_MGCG |
2027 			AMD_CG_SUPPORT_SDMA_LS |
2028 			AMD_CG_SUPPORT_BIF_LS |
2029 			AMD_CG_SUPPORT_VCE_MGCG |
2030 			AMD_CG_SUPPORT_UVD_MGCG |
2031 			AMD_CG_SUPPORT_HDP_LS |
2032 			AMD_CG_SUPPORT_HDP_MGCG;
2033 		adev->pg_flags = 0;
2034 		adev->external_rev_id = 0x28;
2035 		break;
2036 	case CHIP_KAVERI:
2037 		adev->cg_flags =
2038 			AMD_CG_SUPPORT_GFX_MGCG |
2039 			AMD_CG_SUPPORT_GFX_MGLS |
2040 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2041 			AMD_CG_SUPPORT_GFX_CGLS |
2042 			AMD_CG_SUPPORT_GFX_CGTS |
2043 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2044 			AMD_CG_SUPPORT_GFX_CP_LS |
2045 			AMD_CG_SUPPORT_SDMA_MGCG |
2046 			AMD_CG_SUPPORT_SDMA_LS |
2047 			AMD_CG_SUPPORT_BIF_LS |
2048 			AMD_CG_SUPPORT_VCE_MGCG |
2049 			AMD_CG_SUPPORT_UVD_MGCG |
2050 			AMD_CG_SUPPORT_HDP_LS |
2051 			AMD_CG_SUPPORT_HDP_MGCG;
2052 		adev->pg_flags =
2053 			/*AMD_PG_SUPPORT_GFX_PG |
2054 			  AMD_PG_SUPPORT_GFX_SMG |
2055 			  AMD_PG_SUPPORT_GFX_DMG |*/
2056 			AMD_PG_SUPPORT_UVD |
2057 			AMD_PG_SUPPORT_VCE |
2058 			/*  AMD_PG_SUPPORT_CP |
2059 			  AMD_PG_SUPPORT_GDS |
2060 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2061 			  AMD_PG_SUPPORT_ACP |
2062 			  AMD_PG_SUPPORT_SAMU |*/
2063 			0;
2064 		if (adev->pdev->device == 0x1312 ||
2065 			adev->pdev->device == 0x1316 ||
2066 			adev->pdev->device == 0x1317)
2067 			adev->external_rev_id = 0x41;
2068 		else
2069 			adev->external_rev_id = 0x1;
2070 		break;
2071 	case CHIP_KABINI:
2072 	case CHIP_MULLINS:
2073 		adev->cg_flags =
2074 			AMD_CG_SUPPORT_GFX_MGCG |
2075 			AMD_CG_SUPPORT_GFX_MGLS |
2076 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2077 			AMD_CG_SUPPORT_GFX_CGLS |
2078 			AMD_CG_SUPPORT_GFX_CGTS |
2079 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2080 			AMD_CG_SUPPORT_GFX_CP_LS |
2081 			AMD_CG_SUPPORT_SDMA_MGCG |
2082 			AMD_CG_SUPPORT_SDMA_LS |
2083 			AMD_CG_SUPPORT_BIF_LS |
2084 			AMD_CG_SUPPORT_VCE_MGCG |
2085 			AMD_CG_SUPPORT_UVD_MGCG |
2086 			AMD_CG_SUPPORT_HDP_LS |
2087 			AMD_CG_SUPPORT_HDP_MGCG;
2088 		adev->pg_flags =
2089 			/*AMD_PG_SUPPORT_GFX_PG |
2090 			  AMD_PG_SUPPORT_GFX_SMG | */
2091 			AMD_PG_SUPPORT_UVD |
2092 			/*AMD_PG_SUPPORT_VCE |
2093 			  AMD_PG_SUPPORT_CP |
2094 			  AMD_PG_SUPPORT_GDS |
2095 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2096 			  AMD_PG_SUPPORT_SAMU |*/
2097 			0;
2098 		if (adev->asic_type == CHIP_KABINI) {
2099 			if (adev->rev_id == 0)
2100 				adev->external_rev_id = 0x81;
2101 			else if (adev->rev_id == 1)
2102 				adev->external_rev_id = 0x82;
2103 			else if (adev->rev_id == 2)
2104 				adev->external_rev_id = 0x85;
2105 		} else
2106 			adev->external_rev_id = adev->rev_id + 0xa1;
2107 		break;
2108 	default:
2109 		/* FIXME: not supported yet */
2110 		return -EINVAL;
2111 	}
2112 
2113 	return 0;
2114 }
2115 
2116 static int cik_common_hw_init(struct amdgpu_ip_block *ip_block)
2117 {
2118 	struct amdgpu_device *adev = ip_block->adev;
2119 
2120 	/* move the golden regs per IP block */
2121 	cik_init_golden_registers(adev);
2122 	/* enable pcie gen2/3 link */
2123 	cik_pcie_gen3_enable(adev);
2124 	/* enable aspm */
2125 	cik_program_aspm(adev);
2126 
2127 	return 0;
2128 }
2129 
2130 static int cik_common_hw_fini(struct amdgpu_ip_block *ip_block)
2131 {
2132 	return 0;
2133 }
2134 
2135 static int cik_common_resume(struct amdgpu_ip_block *ip_block)
2136 {
2137 	return cik_common_hw_init(ip_block);
2138 }
2139 
2140 static bool cik_common_is_idle(struct amdgpu_ip_block *ip_block)
2141 {
2142 	return true;
2143 }
2144 
2145 
2146 
2147 static int cik_common_soft_reset(struct amdgpu_ip_block *ip_block)
2148 {
2149 	/* XXX hard reset?? */
2150 	return 0;
2151 }
2152 
2153 static int cik_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2154 					    enum amd_clockgating_state state)
2155 {
2156 	return 0;
2157 }
2158 
2159 static int cik_common_set_powergating_state(struct amdgpu_ip_block *ip_block,
2160 					    enum amd_powergating_state state)
2161 {
2162 	return 0;
2163 }
2164 
2165 static const struct amd_ip_funcs cik_common_ip_funcs = {
2166 	.name = "cik_common",
2167 	.early_init = cik_common_early_init,
2168 	.hw_init = cik_common_hw_init,
2169 	.hw_fini = cik_common_hw_fini,
2170 	.resume = cik_common_resume,
2171 	.is_idle = cik_common_is_idle,
2172 	.soft_reset = cik_common_soft_reset,
2173 	.set_clockgating_state = cik_common_set_clockgating_state,
2174 	.set_powergating_state = cik_common_set_powergating_state,
2175 };
2176 
2177 static const struct amdgpu_ip_block_version cik_common_ip_block =
2178 {
2179 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2180 	.major = 1,
2181 	.minor = 0,
2182 	.rev = 0,
2183 	.funcs = &cik_common_ip_funcs,
2184 };
2185 
2186 int cik_set_ip_blocks(struct amdgpu_device *adev)
2187 {
2188 	switch (adev->asic_type) {
2189 	case CHIP_BONAIRE:
2190 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2191 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2192 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2193 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2194 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2195 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2196 		if (adev->enable_virtual_display)
2197 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2198 #if defined(CONFIG_DRM_AMD_DC)
2199 		else if (amdgpu_device_has_dc_support(adev))
2200 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2201 #endif
2202 		else
2203 			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2204 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2205 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2206 		break;
2207 	case CHIP_HAWAII:
2208 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2209 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2210 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2211 		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2212 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2213 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2214 		if (adev->enable_virtual_display)
2215 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2216 #if defined(CONFIG_DRM_AMD_DC)
2217 		else if (amdgpu_device_has_dc_support(adev))
2218 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2219 #endif
2220 		else
2221 			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2222 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2223 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2224 		break;
2225 	case CHIP_KAVERI:
2226 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2227 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2228 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2229 		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2230 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2231 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2232 		if (adev->enable_virtual_display)
2233 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2234 #if defined(CONFIG_DRM_AMD_DC)
2235 		else if (amdgpu_device_has_dc_support(adev))
2236 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2237 #endif
2238 		else
2239 			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2240 
2241 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2242 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2243 		break;
2244 	case CHIP_KABINI:
2245 	case CHIP_MULLINS:
2246 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2247 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2248 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2249 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2250 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2251 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2252 		if (adev->enable_virtual_display)
2253 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2254 #if defined(CONFIG_DRM_AMD_DC)
2255 		else if (amdgpu_device_has_dc_support(adev))
2256 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2257 #endif
2258 		else
2259 			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2260 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2261 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2262 		break;
2263 	default:
2264 		/* FIXME: not supported yet */
2265 		return -EINVAL;
2266 	}
2267 	return 0;
2268 }
2269