xref: /linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision 5027ec19f1049a07df5b0a37b1f462514cf2724b) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include <drm/amdgpu_drm.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_atombios.h"
33 #include "amdgpu_ih.h"
34 #include "amdgpu_uvd.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37 #include "atom.h"
38 #include "amd_pcie.h"
39 
40 #include "cik.h"
41 #include "gmc_v7_0.h"
42 #include "cik_ih.h"
43 #include "dce_v8_0.h"
44 #include "gfx_v7_0.h"
45 #include "cik_sdma.h"
46 #include "uvd_v4_2.h"
47 #include "vce_v2_0.h"
48 #include "cik_dpm.h"
49 
50 #include "uvd/uvd_4_2_d.h"
51 
52 #include "smu/smu_7_0_1_d.h"
53 #include "smu/smu_7_0_1_sh_mask.h"
54 
55 #include "dce/dce_8_0_d.h"
56 #include "dce/dce_8_0_sh_mask.h"
57 
58 #include "bif/bif_4_1_d.h"
59 #include "bif/bif_4_1_sh_mask.h"
60 
61 #include "gca/gfx_7_2_d.h"
62 #include "gca/gfx_7_2_enum.h"
63 #include "gca/gfx_7_2_sh_mask.h"
64 
65 #include "gmc/gmc_7_1_d.h"
66 #include "gmc/gmc_7_1_sh_mask.h"
67 
68 #include "oss/oss_2_0_d.h"
69 #include "oss/oss_2_0_sh_mask.h"
70 
71 #include "amdgpu_dm.h"
72 #include "amdgpu_amdkfd.h"
73 #include "amdgpu_vkms.h"
74 
75 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
76 {
77 	{
78 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
79 		.max_width = 2048,
80 		.max_height = 1152,
81 		.max_pixels_per_frame = 2048 * 1152,
82 		.max_level = 0,
83 	},
84 };
85 
86 static const struct amdgpu_video_codecs cik_video_codecs_encode =
87 {
88 	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
89 	.codec_array = cik_video_codecs_encode_array,
90 };
91 
92 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
93 {
94 	{
95 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
96 		.max_width = 2048,
97 		.max_height = 1152,
98 		.max_pixels_per_frame = 2048 * 1152,
99 		.max_level = 3,
100 	},
101 	{
102 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
103 		.max_width = 2048,
104 		.max_height = 1152,
105 		.max_pixels_per_frame = 2048 * 1152,
106 		.max_level = 5,
107 	},
108 	{
109 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
110 		.max_width = 2048,
111 		.max_height = 1152,
112 		.max_pixels_per_frame = 2048 * 1152,
113 		.max_level = 41,
114 	},
115 	{
116 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
117 		.max_width = 2048,
118 		.max_height = 1152,
119 		.max_pixels_per_frame = 2048 * 1152,
120 		.max_level = 4,
121 	},
122 };
123 
124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
125 {
126 	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
127 	.codec_array = cik_video_codecs_decode_array,
128 };
129 
130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
131 				  const struct amdgpu_video_codecs **codecs)
132 {
133 	switch (adev->asic_type) {
134 	case CHIP_BONAIRE:
135 	case CHIP_HAWAII:
136 	case CHIP_KAVERI:
137 	case CHIP_KABINI:
138 	case CHIP_MULLINS:
139 		if (encode)
140 			*codecs = &cik_video_codecs_encode;
141 		else
142 			*codecs = &cik_video_codecs_decode;
143 		return 0;
144 	default:
145 		return -EINVAL;
146 	}
147 }
148 
149 /*
150  * Indirect registers accessor
151  */
152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
153 {
154 	unsigned long flags;
155 	u32 r;
156 
157 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
158 	WREG32(mmPCIE_INDEX, reg);
159 	(void)RREG32(mmPCIE_INDEX);
160 	r = RREG32(mmPCIE_DATA);
161 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
162 	return r;
163 }
164 
165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
166 {
167 	unsigned long flags;
168 
169 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
170 	WREG32(mmPCIE_INDEX, reg);
171 	(void)RREG32(mmPCIE_INDEX);
172 	WREG32(mmPCIE_DATA, v);
173 	(void)RREG32(mmPCIE_DATA);
174 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
175 }
176 
177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
178 {
179 	unsigned long flags;
180 	u32 r;
181 
182 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
183 	WREG32(mmSMC_IND_INDEX_0, (reg));
184 	r = RREG32(mmSMC_IND_DATA_0);
185 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
186 	return r;
187 }
188 
189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
190 {
191 	unsigned long flags;
192 
193 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
194 	WREG32(mmSMC_IND_INDEX_0, (reg));
195 	WREG32(mmSMC_IND_DATA_0, (v));
196 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
197 }
198 
199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
200 {
201 	unsigned long flags;
202 	u32 r;
203 
204 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
205 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
206 	r = RREG32(mmUVD_CTX_DATA);
207 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
208 	return r;
209 }
210 
211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
212 {
213 	unsigned long flags;
214 
215 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
216 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
217 	WREG32(mmUVD_CTX_DATA, (v));
218 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
219 }
220 
221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
222 {
223 	unsigned long flags;
224 	u32 r;
225 
226 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
227 	WREG32(mmDIDT_IND_INDEX, (reg));
228 	r = RREG32(mmDIDT_IND_DATA);
229 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
230 	return r;
231 }
232 
233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
234 {
235 	unsigned long flags;
236 
237 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
238 	WREG32(mmDIDT_IND_INDEX, (reg));
239 	WREG32(mmDIDT_IND_DATA, (v));
240 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
241 }
242 
243 static const u32 bonaire_golden_spm_registers[] =
244 {
245 	0xc200, 0xe0ffffff, 0xe0000000
246 };
247 
248 static const u32 bonaire_golden_common_registers[] =
249 {
250 	0x31dc, 0xffffffff, 0x00000800,
251 	0x31dd, 0xffffffff, 0x00000800,
252 	0x31e6, 0xffffffff, 0x00007fbf,
253 	0x31e7, 0xffffffff, 0x00007faf
254 };
255 
256 static const u32 bonaire_golden_registers[] =
257 {
258 	0xcd5, 0x00000333, 0x00000333,
259 	0xcd4, 0x000c0fc0, 0x00040200,
260 	0x2684, 0x00010000, 0x00058208,
261 	0xf000, 0xffff1fff, 0x00140000,
262 	0xf080, 0xfdfc0fff, 0x00000100,
263 	0xf08d, 0x40000000, 0x40000200,
264 	0x260c, 0xffffffff, 0x00000000,
265 	0x260d, 0xf00fffff, 0x00000400,
266 	0x260e, 0x0002021c, 0x00020200,
267 	0x31e, 0x00000080, 0x00000000,
268 	0x16ec, 0x000000f0, 0x00000070,
269 	0x16f0, 0xf0311fff, 0x80300000,
270 	0x263e, 0x73773777, 0x12010001,
271 	0xd43, 0x00810000, 0x408af000,
272 	0x1c0c, 0x31000111, 0x00000011,
273 	0xbd2, 0x73773777, 0x12010001,
274 	0x883, 0x00007fb6, 0x0021a1b1,
275 	0x884, 0x00007fb6, 0x002021b1,
276 	0x860, 0x00007fb6, 0x00002191,
277 	0x886, 0x00007fb6, 0x002121b1,
278 	0x887, 0x00007fb6, 0x002021b1,
279 	0x877, 0x00007fb6, 0x00002191,
280 	0x878, 0x00007fb6, 0x00002191,
281 	0xd8a, 0x0000003f, 0x0000000a,
282 	0xd8b, 0x0000003f, 0x0000000a,
283 	0xab9, 0x00073ffe, 0x000022a2,
284 	0x903, 0x000007ff, 0x00000000,
285 	0x2285, 0xf000003f, 0x00000007,
286 	0x22fc, 0x00002001, 0x00000001,
287 	0x22c9, 0xffffffff, 0x00ffffff,
288 	0xc281, 0x0000ff0f, 0x00000000,
289 	0xa293, 0x07ffffff, 0x06000000,
290 	0x136, 0x00000fff, 0x00000100,
291 	0xf9e, 0x00000001, 0x00000002,
292 	0x2440, 0x03000000, 0x0362c688,
293 	0x2300, 0x000000ff, 0x00000001,
294 	0x390, 0x00001fff, 0x00001fff,
295 	0x2418, 0x0000007f, 0x00000020,
296 	0x2542, 0x00010000, 0x00010000,
297 	0x2b05, 0x000003ff, 0x000000f3,
298 	0x2b03, 0xffffffff, 0x00001032
299 };
300 
301 static const u32 bonaire_mgcg_cgcg_init[] =
302 {
303 	0x3108, 0xffffffff, 0xfffffffc,
304 	0xc200, 0xffffffff, 0xe0000000,
305 	0xf0a8, 0xffffffff, 0x00000100,
306 	0xf082, 0xffffffff, 0x00000100,
307 	0xf0b0, 0xffffffff, 0xc0000100,
308 	0xf0b2, 0xffffffff, 0xc0000100,
309 	0xf0b1, 0xffffffff, 0xc0000100,
310 	0x1579, 0xffffffff, 0x00600100,
311 	0xf0a0, 0xffffffff, 0x00000100,
312 	0xf085, 0xffffffff, 0x06000100,
313 	0xf088, 0xffffffff, 0x00000100,
314 	0xf086, 0xffffffff, 0x06000100,
315 	0xf081, 0xffffffff, 0x00000100,
316 	0xf0b8, 0xffffffff, 0x00000100,
317 	0xf089, 0xffffffff, 0x00000100,
318 	0xf080, 0xffffffff, 0x00000100,
319 	0xf08c, 0xffffffff, 0x00000100,
320 	0xf08d, 0xffffffff, 0x00000100,
321 	0xf094, 0xffffffff, 0x00000100,
322 	0xf095, 0xffffffff, 0x00000100,
323 	0xf096, 0xffffffff, 0x00000100,
324 	0xf097, 0xffffffff, 0x00000100,
325 	0xf098, 0xffffffff, 0x00000100,
326 	0xf09f, 0xffffffff, 0x00000100,
327 	0xf09e, 0xffffffff, 0x00000100,
328 	0xf084, 0xffffffff, 0x06000100,
329 	0xf0a4, 0xffffffff, 0x00000100,
330 	0xf09d, 0xffffffff, 0x00000100,
331 	0xf0ad, 0xffffffff, 0x00000100,
332 	0xf0ac, 0xffffffff, 0x00000100,
333 	0xf09c, 0xffffffff, 0x00000100,
334 	0xc200, 0xffffffff, 0xe0000000,
335 	0xf008, 0xffffffff, 0x00010000,
336 	0xf009, 0xffffffff, 0x00030002,
337 	0xf00a, 0xffffffff, 0x00040007,
338 	0xf00b, 0xffffffff, 0x00060005,
339 	0xf00c, 0xffffffff, 0x00090008,
340 	0xf00d, 0xffffffff, 0x00010000,
341 	0xf00e, 0xffffffff, 0x00030002,
342 	0xf00f, 0xffffffff, 0x00040007,
343 	0xf010, 0xffffffff, 0x00060005,
344 	0xf011, 0xffffffff, 0x00090008,
345 	0xf012, 0xffffffff, 0x00010000,
346 	0xf013, 0xffffffff, 0x00030002,
347 	0xf014, 0xffffffff, 0x00040007,
348 	0xf015, 0xffffffff, 0x00060005,
349 	0xf016, 0xffffffff, 0x00090008,
350 	0xf017, 0xffffffff, 0x00010000,
351 	0xf018, 0xffffffff, 0x00030002,
352 	0xf019, 0xffffffff, 0x00040007,
353 	0xf01a, 0xffffffff, 0x00060005,
354 	0xf01b, 0xffffffff, 0x00090008,
355 	0xf01c, 0xffffffff, 0x00010000,
356 	0xf01d, 0xffffffff, 0x00030002,
357 	0xf01e, 0xffffffff, 0x00040007,
358 	0xf01f, 0xffffffff, 0x00060005,
359 	0xf020, 0xffffffff, 0x00090008,
360 	0xf021, 0xffffffff, 0x00010000,
361 	0xf022, 0xffffffff, 0x00030002,
362 	0xf023, 0xffffffff, 0x00040007,
363 	0xf024, 0xffffffff, 0x00060005,
364 	0xf025, 0xffffffff, 0x00090008,
365 	0xf026, 0xffffffff, 0x00010000,
366 	0xf027, 0xffffffff, 0x00030002,
367 	0xf028, 0xffffffff, 0x00040007,
368 	0xf029, 0xffffffff, 0x00060005,
369 	0xf02a, 0xffffffff, 0x00090008,
370 	0xf000, 0xffffffff, 0x96e00200,
371 	0x21c2, 0xffffffff, 0x00900100,
372 	0x3109, 0xffffffff, 0x0020003f,
373 	0xe, 0xffffffff, 0x0140001c,
374 	0xf, 0x000f0000, 0x000f0000,
375 	0x88, 0xffffffff, 0xc060000c,
376 	0x89, 0xc0000fff, 0x00000100,
377 	0x3e4, 0xffffffff, 0x00000100,
378 	0x3e6, 0x00000101, 0x00000000,
379 	0x82a, 0xffffffff, 0x00000104,
380 	0x1579, 0xff000fff, 0x00000100,
381 	0xc33, 0xc0000fff, 0x00000104,
382 	0x3079, 0x00000001, 0x00000001,
383 	0x3403, 0xff000ff0, 0x00000100,
384 	0x3603, 0xff000ff0, 0x00000100
385 };
386 
387 static const u32 spectre_golden_spm_registers[] =
388 {
389 	0xc200, 0xe0ffffff, 0xe0000000
390 };
391 
392 static const u32 spectre_golden_common_registers[] =
393 {
394 	0x31dc, 0xffffffff, 0x00000800,
395 	0x31dd, 0xffffffff, 0x00000800,
396 	0x31e6, 0xffffffff, 0x00007fbf,
397 	0x31e7, 0xffffffff, 0x00007faf
398 };
399 
400 static const u32 spectre_golden_registers[] =
401 {
402 	0xf000, 0xffff1fff, 0x96940200,
403 	0xf003, 0xffff0001, 0xff000000,
404 	0xf080, 0xfffc0fff, 0x00000100,
405 	0x1bb6, 0x00010101, 0x00010000,
406 	0x260d, 0xf00fffff, 0x00000400,
407 	0x260e, 0xfffffffc, 0x00020200,
408 	0x16ec, 0x000000f0, 0x00000070,
409 	0x16f0, 0xf0311fff, 0x80300000,
410 	0x263e, 0x73773777, 0x12010001,
411 	0x26df, 0x00ff0000, 0x00fc0000,
412 	0xbd2, 0x73773777, 0x12010001,
413 	0x2285, 0xf000003f, 0x00000007,
414 	0x22c9, 0xffffffff, 0x00ffffff,
415 	0xa0d4, 0x3f3f3fff, 0x00000082,
416 	0xa0d5, 0x0000003f, 0x00000000,
417 	0xf9e, 0x00000001, 0x00000002,
418 	0x244f, 0xffff03df, 0x00000004,
419 	0x31da, 0x00000008, 0x00000008,
420 	0x2300, 0x000008ff, 0x00000800,
421 	0x2542, 0x00010000, 0x00010000,
422 	0x2b03, 0xffffffff, 0x54763210,
423 	0x853e, 0x01ff01ff, 0x00000002,
424 	0x8526, 0x007ff800, 0x00200000,
425 	0x8057, 0xffffffff, 0x00000f40,
426 	0xc24d, 0xffffffff, 0x00000001
427 };
428 
429 static const u32 spectre_mgcg_cgcg_init[] =
430 {
431 	0x3108, 0xffffffff, 0xfffffffc,
432 	0xc200, 0xffffffff, 0xe0000000,
433 	0xf0a8, 0xffffffff, 0x00000100,
434 	0xf082, 0xffffffff, 0x00000100,
435 	0xf0b0, 0xffffffff, 0x00000100,
436 	0xf0b2, 0xffffffff, 0x00000100,
437 	0xf0b1, 0xffffffff, 0x00000100,
438 	0x1579, 0xffffffff, 0x00600100,
439 	0xf0a0, 0xffffffff, 0x00000100,
440 	0xf085, 0xffffffff, 0x06000100,
441 	0xf088, 0xffffffff, 0x00000100,
442 	0xf086, 0xffffffff, 0x06000100,
443 	0xf081, 0xffffffff, 0x00000100,
444 	0xf0b8, 0xffffffff, 0x00000100,
445 	0xf089, 0xffffffff, 0x00000100,
446 	0xf080, 0xffffffff, 0x00000100,
447 	0xf08c, 0xffffffff, 0x00000100,
448 	0xf08d, 0xffffffff, 0x00000100,
449 	0xf094, 0xffffffff, 0x00000100,
450 	0xf095, 0xffffffff, 0x00000100,
451 	0xf096, 0xffffffff, 0x00000100,
452 	0xf097, 0xffffffff, 0x00000100,
453 	0xf098, 0xffffffff, 0x00000100,
454 	0xf09f, 0xffffffff, 0x00000100,
455 	0xf09e, 0xffffffff, 0x00000100,
456 	0xf084, 0xffffffff, 0x06000100,
457 	0xf0a4, 0xffffffff, 0x00000100,
458 	0xf09d, 0xffffffff, 0x00000100,
459 	0xf0ad, 0xffffffff, 0x00000100,
460 	0xf0ac, 0xffffffff, 0x00000100,
461 	0xf09c, 0xffffffff, 0x00000100,
462 	0xc200, 0xffffffff, 0xe0000000,
463 	0xf008, 0xffffffff, 0x00010000,
464 	0xf009, 0xffffffff, 0x00030002,
465 	0xf00a, 0xffffffff, 0x00040007,
466 	0xf00b, 0xffffffff, 0x00060005,
467 	0xf00c, 0xffffffff, 0x00090008,
468 	0xf00d, 0xffffffff, 0x00010000,
469 	0xf00e, 0xffffffff, 0x00030002,
470 	0xf00f, 0xffffffff, 0x00040007,
471 	0xf010, 0xffffffff, 0x00060005,
472 	0xf011, 0xffffffff, 0x00090008,
473 	0xf012, 0xffffffff, 0x00010000,
474 	0xf013, 0xffffffff, 0x00030002,
475 	0xf014, 0xffffffff, 0x00040007,
476 	0xf015, 0xffffffff, 0x00060005,
477 	0xf016, 0xffffffff, 0x00090008,
478 	0xf017, 0xffffffff, 0x00010000,
479 	0xf018, 0xffffffff, 0x00030002,
480 	0xf019, 0xffffffff, 0x00040007,
481 	0xf01a, 0xffffffff, 0x00060005,
482 	0xf01b, 0xffffffff, 0x00090008,
483 	0xf01c, 0xffffffff, 0x00010000,
484 	0xf01d, 0xffffffff, 0x00030002,
485 	0xf01e, 0xffffffff, 0x00040007,
486 	0xf01f, 0xffffffff, 0x00060005,
487 	0xf020, 0xffffffff, 0x00090008,
488 	0xf021, 0xffffffff, 0x00010000,
489 	0xf022, 0xffffffff, 0x00030002,
490 	0xf023, 0xffffffff, 0x00040007,
491 	0xf024, 0xffffffff, 0x00060005,
492 	0xf025, 0xffffffff, 0x00090008,
493 	0xf026, 0xffffffff, 0x00010000,
494 	0xf027, 0xffffffff, 0x00030002,
495 	0xf028, 0xffffffff, 0x00040007,
496 	0xf029, 0xffffffff, 0x00060005,
497 	0xf02a, 0xffffffff, 0x00090008,
498 	0xf02b, 0xffffffff, 0x00010000,
499 	0xf02c, 0xffffffff, 0x00030002,
500 	0xf02d, 0xffffffff, 0x00040007,
501 	0xf02e, 0xffffffff, 0x00060005,
502 	0xf02f, 0xffffffff, 0x00090008,
503 	0xf000, 0xffffffff, 0x96e00200,
504 	0x21c2, 0xffffffff, 0x00900100,
505 	0x3109, 0xffffffff, 0x0020003f,
506 	0xe, 0xffffffff, 0x0140001c,
507 	0xf, 0x000f0000, 0x000f0000,
508 	0x88, 0xffffffff, 0xc060000c,
509 	0x89, 0xc0000fff, 0x00000100,
510 	0x3e4, 0xffffffff, 0x00000100,
511 	0x3e6, 0x00000101, 0x00000000,
512 	0x82a, 0xffffffff, 0x00000104,
513 	0x1579, 0xff000fff, 0x00000100,
514 	0xc33, 0xc0000fff, 0x00000104,
515 	0x3079, 0x00000001, 0x00000001,
516 	0x3403, 0xff000ff0, 0x00000100,
517 	0x3603, 0xff000ff0, 0x00000100
518 };
519 
520 static const u32 kalindi_golden_spm_registers[] =
521 {
522 	0xc200, 0xe0ffffff, 0xe0000000
523 };
524 
525 static const u32 kalindi_golden_common_registers[] =
526 {
527 	0x31dc, 0xffffffff, 0x00000800,
528 	0x31dd, 0xffffffff, 0x00000800,
529 	0x31e6, 0xffffffff, 0x00007fbf,
530 	0x31e7, 0xffffffff, 0x00007faf
531 };
532 
533 static const u32 kalindi_golden_registers[] =
534 {
535 	0xf000, 0xffffdfff, 0x6e944040,
536 	0x1579, 0xff607fff, 0xfc000100,
537 	0xf088, 0xff000fff, 0x00000100,
538 	0xf089, 0xff000fff, 0x00000100,
539 	0xf080, 0xfffc0fff, 0x00000100,
540 	0x1bb6, 0x00010101, 0x00010000,
541 	0x260c, 0xffffffff, 0x00000000,
542 	0x260d, 0xf00fffff, 0x00000400,
543 	0x16ec, 0x000000f0, 0x00000070,
544 	0x16f0, 0xf0311fff, 0x80300000,
545 	0x263e, 0x73773777, 0x12010001,
546 	0x263f, 0xffffffff, 0x00000010,
547 	0x26df, 0x00ff0000, 0x00fc0000,
548 	0x200c, 0x00001f0f, 0x0000100a,
549 	0xbd2, 0x73773777, 0x12010001,
550 	0x902, 0x000fffff, 0x000c007f,
551 	0x2285, 0xf000003f, 0x00000007,
552 	0x22c9, 0x3fff3fff, 0x00ffcfff,
553 	0xc281, 0x0000ff0f, 0x00000000,
554 	0xa293, 0x07ffffff, 0x06000000,
555 	0x136, 0x00000fff, 0x00000100,
556 	0xf9e, 0x00000001, 0x00000002,
557 	0x31da, 0x00000008, 0x00000008,
558 	0x2300, 0x000000ff, 0x00000003,
559 	0x853e, 0x01ff01ff, 0x00000002,
560 	0x8526, 0x007ff800, 0x00200000,
561 	0x8057, 0xffffffff, 0x00000f40,
562 	0x2231, 0x001f3ae3, 0x00000082,
563 	0x2235, 0x0000001f, 0x00000010,
564 	0xc24d, 0xffffffff, 0x00000000
565 };
566 
567 static const u32 kalindi_mgcg_cgcg_init[] =
568 {
569 	0x3108, 0xffffffff, 0xfffffffc,
570 	0xc200, 0xffffffff, 0xe0000000,
571 	0xf0a8, 0xffffffff, 0x00000100,
572 	0xf082, 0xffffffff, 0x00000100,
573 	0xf0b0, 0xffffffff, 0x00000100,
574 	0xf0b2, 0xffffffff, 0x00000100,
575 	0xf0b1, 0xffffffff, 0x00000100,
576 	0x1579, 0xffffffff, 0x00600100,
577 	0xf0a0, 0xffffffff, 0x00000100,
578 	0xf085, 0xffffffff, 0x06000100,
579 	0xf088, 0xffffffff, 0x00000100,
580 	0xf086, 0xffffffff, 0x06000100,
581 	0xf081, 0xffffffff, 0x00000100,
582 	0xf0b8, 0xffffffff, 0x00000100,
583 	0xf089, 0xffffffff, 0x00000100,
584 	0xf080, 0xffffffff, 0x00000100,
585 	0xf08c, 0xffffffff, 0x00000100,
586 	0xf08d, 0xffffffff, 0x00000100,
587 	0xf094, 0xffffffff, 0x00000100,
588 	0xf095, 0xffffffff, 0x00000100,
589 	0xf096, 0xffffffff, 0x00000100,
590 	0xf097, 0xffffffff, 0x00000100,
591 	0xf098, 0xffffffff, 0x00000100,
592 	0xf09f, 0xffffffff, 0x00000100,
593 	0xf09e, 0xffffffff, 0x00000100,
594 	0xf084, 0xffffffff, 0x06000100,
595 	0xf0a4, 0xffffffff, 0x00000100,
596 	0xf09d, 0xffffffff, 0x00000100,
597 	0xf0ad, 0xffffffff, 0x00000100,
598 	0xf0ac, 0xffffffff, 0x00000100,
599 	0xf09c, 0xffffffff, 0x00000100,
600 	0xc200, 0xffffffff, 0xe0000000,
601 	0xf008, 0xffffffff, 0x00010000,
602 	0xf009, 0xffffffff, 0x00030002,
603 	0xf00a, 0xffffffff, 0x00040007,
604 	0xf00b, 0xffffffff, 0x00060005,
605 	0xf00c, 0xffffffff, 0x00090008,
606 	0xf00d, 0xffffffff, 0x00010000,
607 	0xf00e, 0xffffffff, 0x00030002,
608 	0xf00f, 0xffffffff, 0x00040007,
609 	0xf010, 0xffffffff, 0x00060005,
610 	0xf011, 0xffffffff, 0x00090008,
611 	0xf000, 0xffffffff, 0x96e00200,
612 	0x21c2, 0xffffffff, 0x00900100,
613 	0x3109, 0xffffffff, 0x0020003f,
614 	0xe, 0xffffffff, 0x0140001c,
615 	0xf, 0x000f0000, 0x000f0000,
616 	0x88, 0xffffffff, 0xc060000c,
617 	0x89, 0xc0000fff, 0x00000100,
618 	0x82a, 0xffffffff, 0x00000104,
619 	0x1579, 0xff000fff, 0x00000100,
620 	0xc33, 0xc0000fff, 0x00000104,
621 	0x3079, 0x00000001, 0x00000001,
622 	0x3403, 0xff000ff0, 0x00000100,
623 	0x3603, 0xff000ff0, 0x00000100
624 };
625 
626 static const u32 hawaii_golden_spm_registers[] =
627 {
628 	0xc200, 0xe0ffffff, 0xe0000000
629 };
630 
631 static const u32 hawaii_golden_common_registers[] =
632 {
633 	0xc200, 0xffffffff, 0xe0000000,
634 	0xa0d4, 0xffffffff, 0x3a00161a,
635 	0xa0d5, 0xffffffff, 0x0000002e,
636 	0x2684, 0xffffffff, 0x00018208,
637 	0x263e, 0xffffffff, 0x12011003
638 };
639 
640 static const u32 hawaii_golden_registers[] =
641 {
642 	0xcd5, 0x00000333, 0x00000333,
643 	0x2684, 0x00010000, 0x00058208,
644 	0x260c, 0xffffffff, 0x00000000,
645 	0x260d, 0xf00fffff, 0x00000400,
646 	0x260e, 0x0002021c, 0x00020200,
647 	0x31e, 0x00000080, 0x00000000,
648 	0x16ec, 0x000000f0, 0x00000070,
649 	0x16f0, 0xf0311fff, 0x80300000,
650 	0xd43, 0x00810000, 0x408af000,
651 	0x1c0c, 0x31000111, 0x00000011,
652 	0xbd2, 0x73773777, 0x12010001,
653 	0x848, 0x0000007f, 0x0000001b,
654 	0x877, 0x00007fb6, 0x00002191,
655 	0xd8a, 0x0000003f, 0x0000000a,
656 	0xd8b, 0x0000003f, 0x0000000a,
657 	0xab9, 0x00073ffe, 0x000022a2,
658 	0x903, 0x000007ff, 0x00000000,
659 	0x22fc, 0x00002001, 0x00000001,
660 	0x22c9, 0xffffffff, 0x00ffffff,
661 	0xc281, 0x0000ff0f, 0x00000000,
662 	0xa293, 0x07ffffff, 0x06000000,
663 	0xf9e, 0x00000001, 0x00000002,
664 	0x31da, 0x00000008, 0x00000008,
665 	0x31dc, 0x00000f00, 0x00000800,
666 	0x31dd, 0x00000f00, 0x00000800,
667 	0x31e6, 0x00ffffff, 0x00ff7fbf,
668 	0x31e7, 0x00ffffff, 0x00ff7faf,
669 	0x2300, 0x000000ff, 0x00000800,
670 	0x390, 0x00001fff, 0x00001fff,
671 	0x2418, 0x0000007f, 0x00000020,
672 	0x2542, 0x00010000, 0x00010000,
673 	0x2b80, 0x00100000, 0x000ff07c,
674 	0x2b05, 0x000003ff, 0x0000000f,
675 	0x2b04, 0xffffffff, 0x7564fdec,
676 	0x2b03, 0xffffffff, 0x3120b9a8,
677 	0x2b02, 0x20000000, 0x0f9c0000
678 };
679 
680 static const u32 hawaii_mgcg_cgcg_init[] =
681 {
682 	0x3108, 0xffffffff, 0xfffffffd,
683 	0xc200, 0xffffffff, 0xe0000000,
684 	0xf0a8, 0xffffffff, 0x00000100,
685 	0xf082, 0xffffffff, 0x00000100,
686 	0xf0b0, 0xffffffff, 0x00000100,
687 	0xf0b2, 0xffffffff, 0x00000100,
688 	0xf0b1, 0xffffffff, 0x00000100,
689 	0x1579, 0xffffffff, 0x00200100,
690 	0xf0a0, 0xffffffff, 0x00000100,
691 	0xf085, 0xffffffff, 0x06000100,
692 	0xf088, 0xffffffff, 0x00000100,
693 	0xf086, 0xffffffff, 0x06000100,
694 	0xf081, 0xffffffff, 0x00000100,
695 	0xf0b8, 0xffffffff, 0x00000100,
696 	0xf089, 0xffffffff, 0x00000100,
697 	0xf080, 0xffffffff, 0x00000100,
698 	0xf08c, 0xffffffff, 0x00000100,
699 	0xf08d, 0xffffffff, 0x00000100,
700 	0xf094, 0xffffffff, 0x00000100,
701 	0xf095, 0xffffffff, 0x00000100,
702 	0xf096, 0xffffffff, 0x00000100,
703 	0xf097, 0xffffffff, 0x00000100,
704 	0xf098, 0xffffffff, 0x00000100,
705 	0xf09f, 0xffffffff, 0x00000100,
706 	0xf09e, 0xffffffff, 0x00000100,
707 	0xf084, 0xffffffff, 0x06000100,
708 	0xf0a4, 0xffffffff, 0x00000100,
709 	0xf09d, 0xffffffff, 0x00000100,
710 	0xf0ad, 0xffffffff, 0x00000100,
711 	0xf0ac, 0xffffffff, 0x00000100,
712 	0xf09c, 0xffffffff, 0x00000100,
713 	0xc200, 0xffffffff, 0xe0000000,
714 	0xf008, 0xffffffff, 0x00010000,
715 	0xf009, 0xffffffff, 0x00030002,
716 	0xf00a, 0xffffffff, 0x00040007,
717 	0xf00b, 0xffffffff, 0x00060005,
718 	0xf00c, 0xffffffff, 0x00090008,
719 	0xf00d, 0xffffffff, 0x00010000,
720 	0xf00e, 0xffffffff, 0x00030002,
721 	0xf00f, 0xffffffff, 0x00040007,
722 	0xf010, 0xffffffff, 0x00060005,
723 	0xf011, 0xffffffff, 0x00090008,
724 	0xf012, 0xffffffff, 0x00010000,
725 	0xf013, 0xffffffff, 0x00030002,
726 	0xf014, 0xffffffff, 0x00040007,
727 	0xf015, 0xffffffff, 0x00060005,
728 	0xf016, 0xffffffff, 0x00090008,
729 	0xf017, 0xffffffff, 0x00010000,
730 	0xf018, 0xffffffff, 0x00030002,
731 	0xf019, 0xffffffff, 0x00040007,
732 	0xf01a, 0xffffffff, 0x00060005,
733 	0xf01b, 0xffffffff, 0x00090008,
734 	0xf01c, 0xffffffff, 0x00010000,
735 	0xf01d, 0xffffffff, 0x00030002,
736 	0xf01e, 0xffffffff, 0x00040007,
737 	0xf01f, 0xffffffff, 0x00060005,
738 	0xf020, 0xffffffff, 0x00090008,
739 	0xf021, 0xffffffff, 0x00010000,
740 	0xf022, 0xffffffff, 0x00030002,
741 	0xf023, 0xffffffff, 0x00040007,
742 	0xf024, 0xffffffff, 0x00060005,
743 	0xf025, 0xffffffff, 0x00090008,
744 	0xf026, 0xffffffff, 0x00010000,
745 	0xf027, 0xffffffff, 0x00030002,
746 	0xf028, 0xffffffff, 0x00040007,
747 	0xf029, 0xffffffff, 0x00060005,
748 	0xf02a, 0xffffffff, 0x00090008,
749 	0xf02b, 0xffffffff, 0x00010000,
750 	0xf02c, 0xffffffff, 0x00030002,
751 	0xf02d, 0xffffffff, 0x00040007,
752 	0xf02e, 0xffffffff, 0x00060005,
753 	0xf02f, 0xffffffff, 0x00090008,
754 	0xf030, 0xffffffff, 0x00010000,
755 	0xf031, 0xffffffff, 0x00030002,
756 	0xf032, 0xffffffff, 0x00040007,
757 	0xf033, 0xffffffff, 0x00060005,
758 	0xf034, 0xffffffff, 0x00090008,
759 	0xf035, 0xffffffff, 0x00010000,
760 	0xf036, 0xffffffff, 0x00030002,
761 	0xf037, 0xffffffff, 0x00040007,
762 	0xf038, 0xffffffff, 0x00060005,
763 	0xf039, 0xffffffff, 0x00090008,
764 	0xf03a, 0xffffffff, 0x00010000,
765 	0xf03b, 0xffffffff, 0x00030002,
766 	0xf03c, 0xffffffff, 0x00040007,
767 	0xf03d, 0xffffffff, 0x00060005,
768 	0xf03e, 0xffffffff, 0x00090008,
769 	0x30c6, 0xffffffff, 0x00020200,
770 	0xcd4, 0xffffffff, 0x00000200,
771 	0x570, 0xffffffff, 0x00000400,
772 	0x157a, 0xffffffff, 0x00000000,
773 	0xbd4, 0xffffffff, 0x00000902,
774 	0xf000, 0xffffffff, 0x96940200,
775 	0x21c2, 0xffffffff, 0x00900100,
776 	0x3109, 0xffffffff, 0x0020003f,
777 	0xe, 0xffffffff, 0x0140001c,
778 	0xf, 0x000f0000, 0x000f0000,
779 	0x88, 0xffffffff, 0xc060000c,
780 	0x89, 0xc0000fff, 0x00000100,
781 	0x3e4, 0xffffffff, 0x00000100,
782 	0x3e6, 0x00000101, 0x00000000,
783 	0x82a, 0xffffffff, 0x00000104,
784 	0x1579, 0xff000fff, 0x00000100,
785 	0xc33, 0xc0000fff, 0x00000104,
786 	0x3079, 0x00000001, 0x00000001,
787 	0x3403, 0xff000ff0, 0x00000100,
788 	0x3603, 0xff000ff0, 0x00000100
789 };
790 
791 static const u32 godavari_golden_registers[] =
792 {
793 	0x1579, 0xff607fff, 0xfc000100,
794 	0x1bb6, 0x00010101, 0x00010000,
795 	0x260c, 0xffffffff, 0x00000000,
796 	0x260c0, 0xf00fffff, 0x00000400,
797 	0x184c, 0xffffffff, 0x00010000,
798 	0x16ec, 0x000000f0, 0x00000070,
799 	0x16f0, 0xf0311fff, 0x80300000,
800 	0x263e, 0x73773777, 0x12010001,
801 	0x263f, 0xffffffff, 0x00000010,
802 	0x200c, 0x00001f0f, 0x0000100a,
803 	0xbd2, 0x73773777, 0x12010001,
804 	0x902, 0x000fffff, 0x000c007f,
805 	0x2285, 0xf000003f, 0x00000007,
806 	0x22c9, 0xffffffff, 0x00ff0fff,
807 	0xc281, 0x0000ff0f, 0x00000000,
808 	0xa293, 0x07ffffff, 0x06000000,
809 	0x136, 0x00000fff, 0x00000100,
810 	0x3405, 0x00010000, 0x00810001,
811 	0x3605, 0x00010000, 0x00810001,
812 	0xf9e, 0x00000001, 0x00000002,
813 	0x31da, 0x00000008, 0x00000008,
814 	0x31dc, 0x00000f00, 0x00000800,
815 	0x31dd, 0x00000f00, 0x00000800,
816 	0x31e6, 0x00ffffff, 0x00ff7fbf,
817 	0x31e7, 0x00ffffff, 0x00ff7faf,
818 	0x2300, 0x000000ff, 0x00000001,
819 	0x853e, 0x01ff01ff, 0x00000002,
820 	0x8526, 0x007ff800, 0x00200000,
821 	0x8057, 0xffffffff, 0x00000f40,
822 	0x2231, 0x001f3ae3, 0x00000082,
823 	0x2235, 0x0000001f, 0x00000010,
824 	0xc24d, 0xffffffff, 0x00000000
825 };
826 
827 static void cik_init_golden_registers(struct amdgpu_device *adev)
828 {
829 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
830 	mutex_lock(&adev->grbm_idx_mutex);
831 
832 	switch (adev->asic_type) {
833 	case CHIP_BONAIRE:
834 		amdgpu_device_program_register_sequence(adev,
835 							bonaire_mgcg_cgcg_init,
836 							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
837 		amdgpu_device_program_register_sequence(adev,
838 							bonaire_golden_registers,
839 							ARRAY_SIZE(bonaire_golden_registers));
840 		amdgpu_device_program_register_sequence(adev,
841 							bonaire_golden_common_registers,
842 							ARRAY_SIZE(bonaire_golden_common_registers));
843 		amdgpu_device_program_register_sequence(adev,
844 							bonaire_golden_spm_registers,
845 							ARRAY_SIZE(bonaire_golden_spm_registers));
846 		break;
847 	case CHIP_KABINI:
848 		amdgpu_device_program_register_sequence(adev,
849 							kalindi_mgcg_cgcg_init,
850 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
851 		amdgpu_device_program_register_sequence(adev,
852 							kalindi_golden_registers,
853 							ARRAY_SIZE(kalindi_golden_registers));
854 		amdgpu_device_program_register_sequence(adev,
855 							kalindi_golden_common_registers,
856 							ARRAY_SIZE(kalindi_golden_common_registers));
857 		amdgpu_device_program_register_sequence(adev,
858 							kalindi_golden_spm_registers,
859 							ARRAY_SIZE(kalindi_golden_spm_registers));
860 		break;
861 	case CHIP_MULLINS:
862 		amdgpu_device_program_register_sequence(adev,
863 							kalindi_mgcg_cgcg_init,
864 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
865 		amdgpu_device_program_register_sequence(adev,
866 							godavari_golden_registers,
867 							ARRAY_SIZE(godavari_golden_registers));
868 		amdgpu_device_program_register_sequence(adev,
869 							kalindi_golden_common_registers,
870 							ARRAY_SIZE(kalindi_golden_common_registers));
871 		amdgpu_device_program_register_sequence(adev,
872 							kalindi_golden_spm_registers,
873 							ARRAY_SIZE(kalindi_golden_spm_registers));
874 		break;
875 	case CHIP_KAVERI:
876 		amdgpu_device_program_register_sequence(adev,
877 							spectre_mgcg_cgcg_init,
878 							ARRAY_SIZE(spectre_mgcg_cgcg_init));
879 		amdgpu_device_program_register_sequence(adev,
880 							spectre_golden_registers,
881 							ARRAY_SIZE(spectre_golden_registers));
882 		amdgpu_device_program_register_sequence(adev,
883 							spectre_golden_common_registers,
884 							ARRAY_SIZE(spectre_golden_common_registers));
885 		amdgpu_device_program_register_sequence(adev,
886 							spectre_golden_spm_registers,
887 							ARRAY_SIZE(spectre_golden_spm_registers));
888 		break;
889 	case CHIP_HAWAII:
890 		amdgpu_device_program_register_sequence(adev,
891 							hawaii_mgcg_cgcg_init,
892 							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
893 		amdgpu_device_program_register_sequence(adev,
894 							hawaii_golden_registers,
895 							ARRAY_SIZE(hawaii_golden_registers));
896 		amdgpu_device_program_register_sequence(adev,
897 							hawaii_golden_common_registers,
898 							ARRAY_SIZE(hawaii_golden_common_registers));
899 		amdgpu_device_program_register_sequence(adev,
900 							hawaii_golden_spm_registers,
901 							ARRAY_SIZE(hawaii_golden_spm_registers));
902 		break;
903 	default:
904 		break;
905 	}
906 	mutex_unlock(&adev->grbm_idx_mutex);
907 }
908 
909 /**
910  * cik_get_xclk - get the xclk
911  *
912  * @adev: amdgpu_device pointer
913  *
914  * Returns the reference clock used by the gfx engine
915  * (CIK).
916  */
917 static u32 cik_get_xclk(struct amdgpu_device *adev)
918 {
919 	u32 reference_clock = adev->clock.spll.reference_freq;
920 
921 	if (adev->flags & AMD_IS_APU) {
922 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
923 			return reference_clock / 2;
924 	} else {
925 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
926 			return reference_clock / 4;
927 	}
928 	return reference_clock;
929 }
930 
931 /**
932  * cik_srbm_select - select specific register instances
933  *
934  * @adev: amdgpu_device pointer
935  * @me: selected ME (micro engine)
936  * @pipe: pipe
937  * @queue: queue
938  * @vmid: VMID
939  *
940  * Switches the currently active registers instances.  Some
941  * registers are instanced per VMID, others are instanced per
942  * me/pipe/queue combination.
943  */
944 void cik_srbm_select(struct amdgpu_device *adev,
945 		     u32 me, u32 pipe, u32 queue, u32 vmid)
946 {
947 	u32 srbm_gfx_cntl =
948 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
949 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
950 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
951 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
952 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
953 }
954 
955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
956 {
957 	uint32_t tmp;
958 
959 	tmp = RREG32(mmCONFIG_CNTL);
960 	if (!state)
961 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
962 	else
963 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
964 	WREG32(mmCONFIG_CNTL, tmp);
965 }
966 
967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
968 {
969 	u32 bus_cntl;
970 	u32 d1vga_control = 0;
971 	u32 d2vga_control = 0;
972 	u32 vga_render_control = 0;
973 	u32 rom_cntl;
974 	bool r;
975 
976 	bus_cntl = RREG32(mmBUS_CNTL);
977 	if (adev->mode_info.num_crtc) {
978 		d1vga_control = RREG32(mmD1VGA_CONTROL);
979 		d2vga_control = RREG32(mmD2VGA_CONTROL);
980 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
981 	}
982 	rom_cntl = RREG32_SMC(ixROM_CNTL);
983 
984 	/* enable the rom */
985 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
986 	if (adev->mode_info.num_crtc) {
987 		/* Disable VGA mode */
988 		WREG32(mmD1VGA_CONTROL,
989 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
990 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
991 		WREG32(mmD2VGA_CONTROL,
992 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
993 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
994 		WREG32(mmVGA_RENDER_CONTROL,
995 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
996 	}
997 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
998 
999 	r = amdgpu_read_bios(adev);
1000 
1001 	/* restore regs */
1002 	WREG32(mmBUS_CNTL, bus_cntl);
1003 	if (adev->mode_info.num_crtc) {
1004 		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005 		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007 	}
1008 	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009 	return r;
1010 }
1011 
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013 				   u8 *bios, u32 length_bytes)
1014 {
1015 	u32 *dw_ptr;
1016 	unsigned long flags;
1017 	u32 i, length_dw;
1018 
1019 	if (bios == NULL)
1020 		return false;
1021 	if (length_bytes == 0)
1022 		return false;
1023 	/* APU vbios image is part of sbios image */
1024 	if (adev->flags & AMD_IS_APU)
1025 		return false;
1026 
1027 	dw_ptr = (u32 *)bios;
1028 	length_dw = ALIGN(length_bytes, 4) / 4;
1029 	/* take the smc lock since we are using the smc index */
1030 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031 	/* set rom index to 0 */
1032 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033 	WREG32(mmSMC_IND_DATA_0, 0);
1034 	/* set index to data for continous read */
1035 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036 	for (i = 0; i < length_dw; i++)
1037 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039 
1040 	return true;
1041 }
1042 
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044 	{mmGRBM_STATUS},
1045 	{mmGRBM_STATUS2},
1046 	{mmGRBM_STATUS_SE0},
1047 	{mmGRBM_STATUS_SE1},
1048 	{mmGRBM_STATUS_SE2},
1049 	{mmGRBM_STATUS_SE3},
1050 	{mmSRBM_STATUS},
1051 	{mmSRBM_STATUS2},
1052 	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053 	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054 	{mmCP_STAT},
1055 	{mmCP_STALLED_STAT1},
1056 	{mmCP_STALLED_STAT2},
1057 	{mmCP_STALLED_STAT3},
1058 	{mmCP_CPF_BUSY_STAT},
1059 	{mmCP_CPF_STALLED_STAT1},
1060 	{mmCP_CPF_STATUS},
1061 	{mmCP_CPC_BUSY_STAT},
1062 	{mmCP_CPC_STALLED_STAT1},
1063 	{mmCP_CPC_STATUS},
1064 	{mmGB_ADDR_CONFIG},
1065 	{mmMC_ARB_RAMCFG},
1066 	{mmGB_TILE_MODE0},
1067 	{mmGB_TILE_MODE1},
1068 	{mmGB_TILE_MODE2},
1069 	{mmGB_TILE_MODE3},
1070 	{mmGB_TILE_MODE4},
1071 	{mmGB_TILE_MODE5},
1072 	{mmGB_TILE_MODE6},
1073 	{mmGB_TILE_MODE7},
1074 	{mmGB_TILE_MODE8},
1075 	{mmGB_TILE_MODE9},
1076 	{mmGB_TILE_MODE10},
1077 	{mmGB_TILE_MODE11},
1078 	{mmGB_TILE_MODE12},
1079 	{mmGB_TILE_MODE13},
1080 	{mmGB_TILE_MODE14},
1081 	{mmGB_TILE_MODE15},
1082 	{mmGB_TILE_MODE16},
1083 	{mmGB_TILE_MODE17},
1084 	{mmGB_TILE_MODE18},
1085 	{mmGB_TILE_MODE19},
1086 	{mmGB_TILE_MODE20},
1087 	{mmGB_TILE_MODE21},
1088 	{mmGB_TILE_MODE22},
1089 	{mmGB_TILE_MODE23},
1090 	{mmGB_TILE_MODE24},
1091 	{mmGB_TILE_MODE25},
1092 	{mmGB_TILE_MODE26},
1093 	{mmGB_TILE_MODE27},
1094 	{mmGB_TILE_MODE28},
1095 	{mmGB_TILE_MODE29},
1096 	{mmGB_TILE_MODE30},
1097 	{mmGB_TILE_MODE31},
1098 	{mmGB_MACROTILE_MODE0},
1099 	{mmGB_MACROTILE_MODE1},
1100 	{mmGB_MACROTILE_MODE2},
1101 	{mmGB_MACROTILE_MODE3},
1102 	{mmGB_MACROTILE_MODE4},
1103 	{mmGB_MACROTILE_MODE5},
1104 	{mmGB_MACROTILE_MODE6},
1105 	{mmGB_MACROTILE_MODE7},
1106 	{mmGB_MACROTILE_MODE8},
1107 	{mmGB_MACROTILE_MODE9},
1108 	{mmGB_MACROTILE_MODE10},
1109 	{mmGB_MACROTILE_MODE11},
1110 	{mmGB_MACROTILE_MODE12},
1111 	{mmGB_MACROTILE_MODE13},
1112 	{mmGB_MACROTILE_MODE14},
1113 	{mmGB_MACROTILE_MODE15},
1114 	{mmCC_RB_BACKEND_DISABLE, true},
1115 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116 	{mmGB_BACKEND_MAP, false},
1117 	{mmPA_SC_RASTER_CONFIG, true},
1118 	{mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120 
1121 
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123 				       bool indexed, u32 se_num,
1124 				       u32 sh_num, u32 reg_offset)
1125 {
1126 	if (indexed) {
1127 		uint32_t val;
1128 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130 
1131 		switch (reg_offset) {
1132 		case mmCC_RB_BACKEND_DISABLE:
1133 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134 		case mmGC_USER_RB_BACKEND_DISABLE:
1135 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136 		case mmPA_SC_RASTER_CONFIG:
1137 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138 		case mmPA_SC_RASTER_CONFIG_1:
1139 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140 		}
1141 
1142 		mutex_lock(&adev->grbm_idx_mutex);
1143 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145 
1146 		val = RREG32(reg_offset);
1147 
1148 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150 		mutex_unlock(&adev->grbm_idx_mutex);
1151 		return val;
1152 	} else {
1153 		unsigned idx;
1154 
1155 		switch (reg_offset) {
1156 		case mmGB_ADDR_CONFIG:
1157 			return adev->gfx.config.gb_addr_config;
1158 		case mmMC_ARB_RAMCFG:
1159 			return adev->gfx.config.mc_arb_ramcfg;
1160 		case mmGB_TILE_MODE0:
1161 		case mmGB_TILE_MODE1:
1162 		case mmGB_TILE_MODE2:
1163 		case mmGB_TILE_MODE3:
1164 		case mmGB_TILE_MODE4:
1165 		case mmGB_TILE_MODE5:
1166 		case mmGB_TILE_MODE6:
1167 		case mmGB_TILE_MODE7:
1168 		case mmGB_TILE_MODE8:
1169 		case mmGB_TILE_MODE9:
1170 		case mmGB_TILE_MODE10:
1171 		case mmGB_TILE_MODE11:
1172 		case mmGB_TILE_MODE12:
1173 		case mmGB_TILE_MODE13:
1174 		case mmGB_TILE_MODE14:
1175 		case mmGB_TILE_MODE15:
1176 		case mmGB_TILE_MODE16:
1177 		case mmGB_TILE_MODE17:
1178 		case mmGB_TILE_MODE18:
1179 		case mmGB_TILE_MODE19:
1180 		case mmGB_TILE_MODE20:
1181 		case mmGB_TILE_MODE21:
1182 		case mmGB_TILE_MODE22:
1183 		case mmGB_TILE_MODE23:
1184 		case mmGB_TILE_MODE24:
1185 		case mmGB_TILE_MODE25:
1186 		case mmGB_TILE_MODE26:
1187 		case mmGB_TILE_MODE27:
1188 		case mmGB_TILE_MODE28:
1189 		case mmGB_TILE_MODE29:
1190 		case mmGB_TILE_MODE30:
1191 		case mmGB_TILE_MODE31:
1192 			idx = (reg_offset - mmGB_TILE_MODE0);
1193 			return adev->gfx.config.tile_mode_array[idx];
1194 		case mmGB_MACROTILE_MODE0:
1195 		case mmGB_MACROTILE_MODE1:
1196 		case mmGB_MACROTILE_MODE2:
1197 		case mmGB_MACROTILE_MODE3:
1198 		case mmGB_MACROTILE_MODE4:
1199 		case mmGB_MACROTILE_MODE5:
1200 		case mmGB_MACROTILE_MODE6:
1201 		case mmGB_MACROTILE_MODE7:
1202 		case mmGB_MACROTILE_MODE8:
1203 		case mmGB_MACROTILE_MODE9:
1204 		case mmGB_MACROTILE_MODE10:
1205 		case mmGB_MACROTILE_MODE11:
1206 		case mmGB_MACROTILE_MODE12:
1207 		case mmGB_MACROTILE_MODE13:
1208 		case mmGB_MACROTILE_MODE14:
1209 		case mmGB_MACROTILE_MODE15:
1210 			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211 			return adev->gfx.config.macrotile_mode_array[idx];
1212 		default:
1213 			return RREG32(reg_offset);
1214 		}
1215 	}
1216 }
1217 
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219 			     u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221 	uint32_t i;
1222 
1223 	*value = 0;
1224 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225 		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226 
1227 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228 			continue;
1229 
1230 		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231 						reg_offset);
1232 		return 0;
1233 	}
1234 	return -EINVAL;
1235 }
1236 
1237 struct kv_reset_save_regs {
1238 	u32 gmcon_reng_execute;
1239 	u32 gmcon_misc;
1240 	u32 gmcon_misc3;
1241 };
1242 
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244 				   struct kv_reset_save_regs *save)
1245 {
1246 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249 
1250 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256 
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258 				      struct kv_reset_save_regs *save)
1259 {
1260 	int i;
1261 
1262 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264 
1265 	for (i = 0; i < 5; i++)
1266 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267 
1268 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270 
1271 	for (i = 0; i < 5; i++)
1272 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273 
1274 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276 
1277 	for (i = 0; i < 5; i++)
1278 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279 
1280 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282 
1283 	for (i = 0; i < 5; i++)
1284 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285 
1286 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288 
1289 	for (i = 0; i < 5; i++)
1290 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291 
1292 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294 
1295 	for (i = 0; i < 5; i++)
1296 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297 
1298 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300 
1301 	for (i = 0; i < 5; i++)
1302 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303 
1304 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306 
1307 	for (i = 0; i < 5; i++)
1308 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309 
1310 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312 
1313 	for (i = 0; i < 5; i++)
1314 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315 
1316 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318 
1319 	for (i = 0; i < 5; i++)
1320 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321 
1322 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324 
1325 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329 
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341 	struct kv_reset_save_regs kv_save = { 0 };
1342 	u32 i;
1343 	int r = -EINVAL;
1344 
1345 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346 
1347 	if (adev->flags & AMD_IS_APU)
1348 		kv_save_regs_for_reset(adev, &kv_save);
1349 
1350 	/* disable BM */
1351 	pci_clear_master(adev->pdev);
1352 	/* reset */
1353 	amdgpu_device_pci_config_reset(adev);
1354 
1355 	udelay(100);
1356 
1357 	/* wait for asic to come out of reset */
1358 	for (i = 0; i < adev->usec_timeout; i++) {
1359 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360 			/* enable BM */
1361 			pci_set_master(adev->pdev);
1362 			adev->has_hw_reset = true;
1363 			r = 0;
1364 			break;
1365 		}
1366 		udelay(1);
1367 	}
1368 
1369 	/* does asic init need to be run first??? */
1370 	if (adev->flags & AMD_IS_APU)
1371 		kv_restore_regs_for_reset(adev, &kv_save);
1372 
1373 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374 
1375 	return r;
1376 }
1377 
1378 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380 	switch (adev->asic_type) {
1381 	case CHIP_BONAIRE:
1382 	case CHIP_HAWAII:
1383 		return amdgpu_dpm_is_baco_supported(adev);
1384 	default:
1385 		return false;
1386 	}
1387 }
1388 
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392 	bool baco_reset;
1393 
1394 	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395 	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396 		return amdgpu_reset_method;
1397 
1398 	if (amdgpu_reset_method != -1)
1399 		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400 				  amdgpu_reset_method);
1401 
1402 	switch (adev->asic_type) {
1403 	case CHIP_BONAIRE:
1404 	case CHIP_HAWAII:
1405 		baco_reset = cik_asic_supports_baco(adev);
1406 		break;
1407 	default:
1408 		baco_reset = false;
1409 		break;
1410 	}
1411 
1412 	if (baco_reset)
1413 		return AMD_RESET_METHOD_BACO;
1414 	else
1415 		return AMD_RESET_METHOD_LEGACY;
1416 }
1417 
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429 	int r;
1430 
1431 	/* APUs don't have full asic reset */
1432 	if (adev->flags & AMD_IS_APU)
1433 		return 0;
1434 
1435 	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436 		dev_info(adev->dev, "BACO reset\n");
1437 		r = amdgpu_dpm_baco_reset(adev);
1438 	} else {
1439 		dev_info(adev->dev, "PCI CONFIG reset\n");
1440 		r = cik_asic_pci_config_reset(adev);
1441 	}
1442 
1443 	return r;
1444 }
1445 
1446 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447 {
1448 	return RREG32(mmCONFIG_MEMSIZE);
1449 }
1450 
1451 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452 			      u32 cntl_reg, u32 status_reg)
1453 {
1454 	int r, i;
1455 	struct atom_clock_dividers dividers;
1456 	uint32_t tmp;
1457 
1458 	r = amdgpu_atombios_get_clock_dividers(adev,
1459 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460 					       clock, false, &dividers);
1461 	if (r)
1462 		return r;
1463 
1464 	tmp = RREG32_SMC(cntl_reg);
1465 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467 	tmp |= dividers.post_divider;
1468 	WREG32_SMC(cntl_reg, tmp);
1469 
1470 	for (i = 0; i < 100; i++) {
1471 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472 			break;
1473 		mdelay(10);
1474 	}
1475 	if (i == 100)
1476 		return -ETIMEDOUT;
1477 
1478 	return 0;
1479 }
1480 
1481 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482 {
1483 	int r = 0;
1484 
1485 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486 	if (r)
1487 		return r;
1488 
1489 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490 	return r;
1491 }
1492 
1493 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494 {
1495 	int r, i;
1496 	struct atom_clock_dividers dividers;
1497 	u32 tmp;
1498 
1499 	r = amdgpu_atombios_get_clock_dividers(adev,
1500 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501 					       ecclk, false, &dividers);
1502 	if (r)
1503 		return r;
1504 
1505 	for (i = 0; i < 100; i++) {
1506 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507 			break;
1508 		mdelay(10);
1509 	}
1510 	if (i == 100)
1511 		return -ETIMEDOUT;
1512 
1513 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516 	tmp |= dividers.post_divider;
1517 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518 
1519 	for (i = 0; i < 100; i++) {
1520 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521 			break;
1522 		mdelay(10);
1523 	}
1524 	if (i == 100)
1525 		return -ETIMEDOUT;
1526 
1527 	return 0;
1528 }
1529 
1530 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531 {
1532 	struct pci_dev *root = adev->pdev->bus->self;
1533 	u32 speed_cntl, current_data_rate;
1534 	int i;
1535 	u16 tmp16;
1536 
1537 	if (pci_is_root_bus(adev->pdev->bus))
1538 		return;
1539 
1540 	if (amdgpu_pcie_gen2 == 0)
1541 		return;
1542 
1543 	if (adev->flags & AMD_IS_APU)
1544 		return;
1545 
1546 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548 		return;
1549 
1550 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554 		if (current_data_rate == 2) {
1555 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556 			return;
1557 		}
1558 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560 		if (current_data_rate == 1) {
1561 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562 			return;
1563 		}
1564 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565 	}
1566 
1567 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568 		return;
1569 
1570 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571 		/* re-try equalization if gen3 is not already enabled */
1572 		if (current_data_rate != 2) {
1573 			u16 bridge_cfg, gpu_cfg;
1574 			u16 bridge_cfg2, gpu_cfg2;
1575 			u32 max_lw, current_lw, tmp;
1576 
1577 			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1578 			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1579 
1580 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1581 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1582 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1583 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1584 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1585 
1586 			if (current_lw < max_lw) {
1587 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1588 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1589 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1590 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1591 					tmp |= (max_lw <<
1592 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1593 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1594 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1595 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1596 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1597 				}
1598 			}
1599 
1600 			for (i = 0; i < 10; i++) {
1601 				/* check status */
1602 				pcie_capability_read_word(adev->pdev,
1603 							  PCI_EXP_DEVSTA,
1604 							  &tmp16);
1605 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1606 					break;
1607 
1608 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1609 							  &bridge_cfg);
1610 				pcie_capability_read_word(adev->pdev,
1611 							  PCI_EXP_LNKCTL,
1612 							  &gpu_cfg);
1613 
1614 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1615 							  &bridge_cfg2);
1616 				pcie_capability_read_word(adev->pdev,
1617 							  PCI_EXP_LNKCTL2,
1618 							  &gpu_cfg2);
1619 
1620 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1621 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1622 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1623 
1624 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1625 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1626 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1627 
1628 				msleep(100);
1629 
1630 				/* linkctl */
1631 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1632 								   PCI_EXP_LNKCTL_HAWD,
1633 								   bridge_cfg &
1634 								   PCI_EXP_LNKCTL_HAWD);
1635 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1636 								   PCI_EXP_LNKCTL_HAWD,
1637 								   gpu_cfg &
1638 								   PCI_EXP_LNKCTL_HAWD);
1639 
1640 				/* linkctl2 */
1641 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1642 							  &tmp16);
1643 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1644 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1645 				tmp16 |= (bridge_cfg2 &
1646 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1647 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1648 				pcie_capability_write_word(root,
1649 							   PCI_EXP_LNKCTL2,
1650 							   tmp16);
1651 
1652 				pcie_capability_read_word(adev->pdev,
1653 							  PCI_EXP_LNKCTL2,
1654 							  &tmp16);
1655 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1656 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1657 				tmp16 |= (gpu_cfg2 &
1658 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1659 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1660 				pcie_capability_write_word(adev->pdev,
1661 							   PCI_EXP_LNKCTL2,
1662 							   tmp16);
1663 
1664 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1665 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1666 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1667 			}
1668 		}
1669 	}
1670 
1671 	/* set the link speed */
1672 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1673 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1674 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1675 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1676 
1677 	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1678 	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1679 
1680 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1681 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1682 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1683 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1684 	else
1685 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1686 	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1687 
1688 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1689 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1690 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1691 
1692 	for (i = 0; i < adev->usec_timeout; i++) {
1693 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1694 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1695 			break;
1696 		udelay(1);
1697 	}
1698 }
1699 
1700 static void cik_program_aspm(struct amdgpu_device *adev)
1701 {
1702 	u32 data, orig;
1703 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1704 	bool disable_clkreq = false;
1705 
1706 	if (!amdgpu_device_should_use_aspm(adev))
1707 		return;
1708 
1709 	if (pci_is_root_bus(adev->pdev->bus))
1710 		return;
1711 
1712 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1713 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1714 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1715 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1716 	if (orig != data)
1717 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1718 
1719 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1720 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1721 	if (orig != data)
1722 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1723 
1724 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1725 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1726 	if (orig != data)
1727 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1728 
1729 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1730 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1731 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1732 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1733 	if (!disable_l0s)
1734 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1735 
1736 	if (!disable_l1) {
1737 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1738 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1739 		if (orig != data)
1740 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1741 
1742 		if (!disable_plloff_in_l1) {
1743 			bool clk_req_support;
1744 
1745 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1746 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1747 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1748 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1749 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1750 			if (orig != data)
1751 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1752 
1753 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1754 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1755 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1756 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1757 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1758 			if (orig != data)
1759 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1760 
1761 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1762 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1763 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1764 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1765 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1766 			if (orig != data)
1767 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1768 
1769 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1770 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1771 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1772 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1773 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1774 			if (orig != data)
1775 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1776 
1777 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1778 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1779 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1780 			if (orig != data)
1781 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1782 
1783 			if (!disable_clkreq) {
1784 				struct pci_dev *root = adev->pdev->bus->self;
1785 				u32 lnkcap;
1786 
1787 				clk_req_support = false;
1788 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1789 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1790 					clk_req_support = true;
1791 			} else {
1792 				clk_req_support = false;
1793 			}
1794 
1795 			if (clk_req_support) {
1796 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1797 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1798 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1799 				if (orig != data)
1800 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1801 
1802 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1803 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1804 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1805 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1806 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1807 				if (orig != data)
1808 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1809 
1810 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1811 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1812 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1813 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1814 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1815 				if (orig != data)
1816 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1817 
1818 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1819 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1820 				if (orig != data)
1821 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1822 
1823 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1824 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1825 				if (orig != data)
1826 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1827 
1828 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1829 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1830 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1831 				if (orig != data)
1832 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1833 			}
1834 		}
1835 	} else {
1836 		if (orig != data)
1837 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1838 	}
1839 
1840 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1841 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1842 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1843 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1844 	if (orig != data)
1845 		WREG32_PCIE(ixPCIE_CNTL2, data);
1846 
1847 	if (!disable_l0s) {
1848 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1849 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1850 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1851 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1852 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1853 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1854 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1855 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1856 				if (orig != data)
1857 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1858 			}
1859 		}
1860 	}
1861 }
1862 
1863 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1864 {
1865 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1866 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1867 }
1868 
1869 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1870 {
1871 	if (!ring || !ring->funcs->emit_wreg) {
1872 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1873 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1874 	} else {
1875 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1876 	}
1877 }
1878 
1879 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1880 			       struct amdgpu_ring *ring)
1881 {
1882 	if (!ring || !ring->funcs->emit_wreg) {
1883 		WREG32(mmHDP_DEBUG0, 1);
1884 		RREG32(mmHDP_DEBUG0);
1885 	} else {
1886 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1887 	}
1888 }
1889 
1890 static bool cik_need_full_reset(struct amdgpu_device *adev)
1891 {
1892 	/* change this when we support soft reset */
1893 	return true;
1894 }
1895 
1896 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1897 			       uint64_t *count1)
1898 {
1899 	uint32_t perfctr = 0;
1900 	uint64_t cnt0_of, cnt1_of;
1901 	int tmp;
1902 
1903 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1904 	 * that may or may not be different from their GPU counterparts
1905 	 */
1906 	if (adev->flags & AMD_IS_APU)
1907 		return;
1908 
1909 	/* Set the 2 events that we wish to watch, defined above */
1910 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1911 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1912 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1913 
1914 	/* Write to enable desired perf counters */
1915 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1916 	/* Zero out and enable the perf counters
1917 	 * Write 0x5:
1918 	 * Bit 0 = Start all counters(1)
1919 	 * Bit 2 = Global counter reset enable(1)
1920 	 */
1921 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1922 
1923 	msleep(1000);
1924 
1925 	/* Load the shadow and disable the perf counters
1926 	 * Write 0x2:
1927 	 * Bit 0 = Stop counters(0)
1928 	 * Bit 1 = Load the shadow counters(1)
1929 	 */
1930 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1931 
1932 	/* Read register values to get any >32bit overflow */
1933 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1934 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1935 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1936 
1937 	/* Get the values and add the overflow */
1938 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1939 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1940 }
1941 
1942 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1943 {
1944 	u32 clock_cntl, pc;
1945 
1946 	if (adev->flags & AMD_IS_APU)
1947 		return false;
1948 
1949 	/* check if the SMC is already running */
1950 	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1951 	pc = RREG32_SMC(ixSMC_PC_C);
1952 	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1953 	    (0x20100 <= pc))
1954 		return true;
1955 
1956 	return false;
1957 }
1958 
1959 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1960 {
1961 	uint64_t nak_r, nak_g;
1962 
1963 	/* Get the number of NAKs received and generated */
1964 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1965 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1966 
1967 	/* Add the total number of NAKs, i.e the number of replays */
1968 	return (nak_r + nak_g);
1969 }
1970 
1971 static void cik_pre_asic_init(struct amdgpu_device *adev)
1972 {
1973 }
1974 
1975 static const struct amdgpu_asic_funcs cik_asic_funcs =
1976 {
1977 	.read_disabled_bios = &cik_read_disabled_bios,
1978 	.read_bios_from_rom = &cik_read_bios_from_rom,
1979 	.read_register = &cik_read_register,
1980 	.reset = &cik_asic_reset,
1981 	.reset_method = &cik_asic_reset_method,
1982 	.set_vga_state = &cik_vga_set_state,
1983 	.get_xclk = &cik_get_xclk,
1984 	.set_uvd_clocks = &cik_set_uvd_clocks,
1985 	.set_vce_clocks = &cik_set_vce_clocks,
1986 	.get_config_memsize = &cik_get_config_memsize,
1987 	.flush_hdp = &cik_flush_hdp,
1988 	.invalidate_hdp = &cik_invalidate_hdp,
1989 	.need_full_reset = &cik_need_full_reset,
1990 	.init_doorbell_index = &legacy_doorbell_index_init,
1991 	.get_pcie_usage = &cik_get_pcie_usage,
1992 	.need_reset_on_init = &cik_need_reset_on_init,
1993 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1994 	.supports_baco = &cik_asic_supports_baco,
1995 	.pre_asic_init = &cik_pre_asic_init,
1996 	.query_video_codecs = &cik_query_video_codecs,
1997 };
1998 
1999 static int cik_common_early_init(void *handle)
2000 {
2001 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2002 
2003 	adev->smc_rreg = &cik_smc_rreg;
2004 	adev->smc_wreg = &cik_smc_wreg;
2005 	adev->pcie_rreg = &cik_pcie_rreg;
2006 	adev->pcie_wreg = &cik_pcie_wreg;
2007 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2008 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2009 	adev->didt_rreg = &cik_didt_rreg;
2010 	adev->didt_wreg = &cik_didt_wreg;
2011 
2012 	adev->asic_funcs = &cik_asic_funcs;
2013 
2014 	adev->rev_id = cik_get_rev_id(adev);
2015 	adev->external_rev_id = 0xFF;
2016 	switch (adev->asic_type) {
2017 	case CHIP_BONAIRE:
2018 		adev->cg_flags =
2019 			AMD_CG_SUPPORT_GFX_MGCG |
2020 			AMD_CG_SUPPORT_GFX_MGLS |
2021 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2022 			AMD_CG_SUPPORT_GFX_CGLS |
2023 			AMD_CG_SUPPORT_GFX_CGTS |
2024 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2025 			AMD_CG_SUPPORT_GFX_CP_LS |
2026 			AMD_CG_SUPPORT_MC_LS |
2027 			AMD_CG_SUPPORT_MC_MGCG |
2028 			AMD_CG_SUPPORT_SDMA_MGCG |
2029 			AMD_CG_SUPPORT_SDMA_LS |
2030 			AMD_CG_SUPPORT_BIF_LS |
2031 			AMD_CG_SUPPORT_VCE_MGCG |
2032 			AMD_CG_SUPPORT_UVD_MGCG |
2033 			AMD_CG_SUPPORT_HDP_LS |
2034 			AMD_CG_SUPPORT_HDP_MGCG;
2035 		adev->pg_flags = 0;
2036 		adev->external_rev_id = adev->rev_id + 0x14;
2037 		break;
2038 	case CHIP_HAWAII:
2039 		adev->cg_flags =
2040 			AMD_CG_SUPPORT_GFX_MGCG |
2041 			AMD_CG_SUPPORT_GFX_MGLS |
2042 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2043 			AMD_CG_SUPPORT_GFX_CGLS |
2044 			AMD_CG_SUPPORT_GFX_CGTS |
2045 			AMD_CG_SUPPORT_GFX_CP_LS |
2046 			AMD_CG_SUPPORT_MC_LS |
2047 			AMD_CG_SUPPORT_MC_MGCG |
2048 			AMD_CG_SUPPORT_SDMA_MGCG |
2049 			AMD_CG_SUPPORT_SDMA_LS |
2050 			AMD_CG_SUPPORT_BIF_LS |
2051 			AMD_CG_SUPPORT_VCE_MGCG |
2052 			AMD_CG_SUPPORT_UVD_MGCG |
2053 			AMD_CG_SUPPORT_HDP_LS |
2054 			AMD_CG_SUPPORT_HDP_MGCG;
2055 		adev->pg_flags = 0;
2056 		adev->external_rev_id = 0x28;
2057 		break;
2058 	case CHIP_KAVERI:
2059 		adev->cg_flags =
2060 			AMD_CG_SUPPORT_GFX_MGCG |
2061 			AMD_CG_SUPPORT_GFX_MGLS |
2062 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2063 			AMD_CG_SUPPORT_GFX_CGLS |
2064 			AMD_CG_SUPPORT_GFX_CGTS |
2065 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2066 			AMD_CG_SUPPORT_GFX_CP_LS |
2067 			AMD_CG_SUPPORT_SDMA_MGCG |
2068 			AMD_CG_SUPPORT_SDMA_LS |
2069 			AMD_CG_SUPPORT_BIF_LS |
2070 			AMD_CG_SUPPORT_VCE_MGCG |
2071 			AMD_CG_SUPPORT_UVD_MGCG |
2072 			AMD_CG_SUPPORT_HDP_LS |
2073 			AMD_CG_SUPPORT_HDP_MGCG;
2074 		adev->pg_flags =
2075 			/*AMD_PG_SUPPORT_GFX_PG |
2076 			  AMD_PG_SUPPORT_GFX_SMG |
2077 			  AMD_PG_SUPPORT_GFX_DMG |*/
2078 			AMD_PG_SUPPORT_UVD |
2079 			AMD_PG_SUPPORT_VCE |
2080 			/*  AMD_PG_SUPPORT_CP |
2081 			  AMD_PG_SUPPORT_GDS |
2082 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2083 			  AMD_PG_SUPPORT_ACP |
2084 			  AMD_PG_SUPPORT_SAMU |*/
2085 			0;
2086 		if (adev->pdev->device == 0x1312 ||
2087 			adev->pdev->device == 0x1316 ||
2088 			adev->pdev->device == 0x1317)
2089 			adev->external_rev_id = 0x41;
2090 		else
2091 			adev->external_rev_id = 0x1;
2092 		break;
2093 	case CHIP_KABINI:
2094 	case CHIP_MULLINS:
2095 		adev->cg_flags =
2096 			AMD_CG_SUPPORT_GFX_MGCG |
2097 			AMD_CG_SUPPORT_GFX_MGLS |
2098 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2099 			AMD_CG_SUPPORT_GFX_CGLS |
2100 			AMD_CG_SUPPORT_GFX_CGTS |
2101 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2102 			AMD_CG_SUPPORT_GFX_CP_LS |
2103 			AMD_CG_SUPPORT_SDMA_MGCG |
2104 			AMD_CG_SUPPORT_SDMA_LS |
2105 			AMD_CG_SUPPORT_BIF_LS |
2106 			AMD_CG_SUPPORT_VCE_MGCG |
2107 			AMD_CG_SUPPORT_UVD_MGCG |
2108 			AMD_CG_SUPPORT_HDP_LS |
2109 			AMD_CG_SUPPORT_HDP_MGCG;
2110 		adev->pg_flags =
2111 			/*AMD_PG_SUPPORT_GFX_PG |
2112 			  AMD_PG_SUPPORT_GFX_SMG | */
2113 			AMD_PG_SUPPORT_UVD |
2114 			/*AMD_PG_SUPPORT_VCE |
2115 			  AMD_PG_SUPPORT_CP |
2116 			  AMD_PG_SUPPORT_GDS |
2117 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2118 			  AMD_PG_SUPPORT_SAMU |*/
2119 			0;
2120 		if (adev->asic_type == CHIP_KABINI) {
2121 			if (adev->rev_id == 0)
2122 				adev->external_rev_id = 0x81;
2123 			else if (adev->rev_id == 1)
2124 				adev->external_rev_id = 0x82;
2125 			else if (adev->rev_id == 2)
2126 				adev->external_rev_id = 0x85;
2127 		} else
2128 			adev->external_rev_id = adev->rev_id + 0xa1;
2129 		break;
2130 	default:
2131 		/* FIXME: not supported yet */
2132 		return -EINVAL;
2133 	}
2134 
2135 	return 0;
2136 }
2137 
2138 static int cik_common_sw_init(void *handle)
2139 {
2140 	return 0;
2141 }
2142 
2143 static int cik_common_sw_fini(void *handle)
2144 {
2145 	return 0;
2146 }
2147 
2148 static int cik_common_hw_init(void *handle)
2149 {
2150 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2151 
2152 	/* move the golden regs per IP block */
2153 	cik_init_golden_registers(adev);
2154 	/* enable pcie gen2/3 link */
2155 	cik_pcie_gen3_enable(adev);
2156 	/* enable aspm */
2157 	cik_program_aspm(adev);
2158 
2159 	return 0;
2160 }
2161 
2162 static int cik_common_hw_fini(void *handle)
2163 {
2164 	return 0;
2165 }
2166 
2167 static int cik_common_suspend(void *handle)
2168 {
2169 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2170 
2171 	return cik_common_hw_fini(adev);
2172 }
2173 
2174 static int cik_common_resume(void *handle)
2175 {
2176 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2177 
2178 	return cik_common_hw_init(adev);
2179 }
2180 
2181 static bool cik_common_is_idle(void *handle)
2182 {
2183 	return true;
2184 }
2185 
2186 static int cik_common_wait_for_idle(void *handle)
2187 {
2188 	return 0;
2189 }
2190 
2191 static int cik_common_soft_reset(void *handle)
2192 {
2193 	/* XXX hard reset?? */
2194 	return 0;
2195 }
2196 
2197 static int cik_common_set_clockgating_state(void *handle,
2198 					    enum amd_clockgating_state state)
2199 {
2200 	return 0;
2201 }
2202 
2203 static int cik_common_set_powergating_state(void *handle,
2204 					    enum amd_powergating_state state)
2205 {
2206 	return 0;
2207 }
2208 
2209 static const struct amd_ip_funcs cik_common_ip_funcs = {
2210 	.name = "cik_common",
2211 	.early_init = cik_common_early_init,
2212 	.late_init = NULL,
2213 	.sw_init = cik_common_sw_init,
2214 	.sw_fini = cik_common_sw_fini,
2215 	.hw_init = cik_common_hw_init,
2216 	.hw_fini = cik_common_hw_fini,
2217 	.suspend = cik_common_suspend,
2218 	.resume = cik_common_resume,
2219 	.is_idle = cik_common_is_idle,
2220 	.wait_for_idle = cik_common_wait_for_idle,
2221 	.soft_reset = cik_common_soft_reset,
2222 	.set_clockgating_state = cik_common_set_clockgating_state,
2223 	.set_powergating_state = cik_common_set_powergating_state,
2224 };
2225 
2226 static const struct amdgpu_ip_block_version cik_common_ip_block =
2227 {
2228 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2229 	.major = 1,
2230 	.minor = 0,
2231 	.rev = 0,
2232 	.funcs = &cik_common_ip_funcs,
2233 };
2234 
2235 int cik_set_ip_blocks(struct amdgpu_device *adev)
2236 {
2237 	switch (adev->asic_type) {
2238 	case CHIP_BONAIRE:
2239 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2240 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2241 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2242 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2243 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2244 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2245 		if (adev->enable_virtual_display)
2246 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2247 #if defined(CONFIG_DRM_AMD_DC)
2248 		else if (amdgpu_device_has_dc_support(adev))
2249 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2250 #endif
2251 		else
2252 			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2253 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2254 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2255 		break;
2256 	case CHIP_HAWAII:
2257 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2258 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2259 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2260 		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2261 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2262 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2263 		if (adev->enable_virtual_display)
2264 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2265 #if defined(CONFIG_DRM_AMD_DC)
2266 		else if (amdgpu_device_has_dc_support(adev))
2267 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2268 #endif
2269 		else
2270 			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2271 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2272 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2273 		break;
2274 	case CHIP_KAVERI:
2275 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2276 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2277 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2278 		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2279 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2280 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2281 		if (adev->enable_virtual_display)
2282 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2283 #if defined(CONFIG_DRM_AMD_DC)
2284 		else if (amdgpu_device_has_dc_support(adev))
2285 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2286 #endif
2287 		else
2288 			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2289 
2290 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2291 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2292 		break;
2293 	case CHIP_KABINI:
2294 	case CHIP_MULLINS:
2295 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2296 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2297 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2298 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2299 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2300 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2301 		if (adev->enable_virtual_display)
2302 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2303 #if defined(CONFIG_DRM_AMD_DC)
2304 		else if (amdgpu_device_has_dc_support(adev))
2305 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2306 #endif
2307 		else
2308 			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2309 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2310 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2311 		break;
2312 	default:
2313 		/* FIXME: not supported yet */
2314 		return -EINVAL;
2315 	}
2316 	return 0;
2317 }
2318