xref: /linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision 0883c2c06fb5bcf5b9e008270827e63c09a88c1e) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "amdgpu.h"
29 #include "amdgpu_atombios.h"
30 #include "amdgpu_ih.h"
31 #include "amdgpu_uvd.h"
32 #include "amdgpu_vce.h"
33 #include "cikd.h"
34 #include "atom.h"
35 #include "amd_pcie.h"
36 
37 #include "cik.h"
38 #include "gmc_v7_0.h"
39 #include "cik_ih.h"
40 #include "dce_v8_0.h"
41 #include "gfx_v7_0.h"
42 #include "cik_sdma.h"
43 #include "uvd_v4_2.h"
44 #include "vce_v2_0.h"
45 #include "cik_dpm.h"
46 
47 #include "uvd/uvd_4_2_d.h"
48 
49 #include "smu/smu_7_0_1_d.h"
50 #include "smu/smu_7_0_1_sh_mask.h"
51 
52 #include "dce/dce_8_0_d.h"
53 #include "dce/dce_8_0_sh_mask.h"
54 
55 #include "bif/bif_4_1_d.h"
56 #include "bif/bif_4_1_sh_mask.h"
57 
58 #include "gca/gfx_7_2_d.h"
59 #include "gca/gfx_7_2_enum.h"
60 #include "gca/gfx_7_2_sh_mask.h"
61 
62 #include "gmc/gmc_7_1_d.h"
63 #include "gmc/gmc_7_1_sh_mask.h"
64 
65 #include "oss/oss_2_0_d.h"
66 #include "oss/oss_2_0_sh_mask.h"
67 
68 #include "amdgpu_amdkfd.h"
69 #include "amdgpu_powerplay.h"
70 
71 /*
72  * Indirect registers accessor
73  */
74 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
75 {
76 	unsigned long flags;
77 	u32 r;
78 
79 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
80 	WREG32(mmPCIE_INDEX, reg);
81 	(void)RREG32(mmPCIE_INDEX);
82 	r = RREG32(mmPCIE_DATA);
83 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
84 	return r;
85 }
86 
87 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
88 {
89 	unsigned long flags;
90 
91 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
92 	WREG32(mmPCIE_INDEX, reg);
93 	(void)RREG32(mmPCIE_INDEX);
94 	WREG32(mmPCIE_DATA, v);
95 	(void)RREG32(mmPCIE_DATA);
96 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
97 }
98 
99 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
100 {
101 	unsigned long flags;
102 	u32 r;
103 
104 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
105 	WREG32(mmSMC_IND_INDEX_0, (reg));
106 	r = RREG32(mmSMC_IND_DATA_0);
107 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
108 	return r;
109 }
110 
111 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
112 {
113 	unsigned long flags;
114 
115 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
116 	WREG32(mmSMC_IND_INDEX_0, (reg));
117 	WREG32(mmSMC_IND_DATA_0, (v));
118 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
119 }
120 
121 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
122 {
123 	unsigned long flags;
124 	u32 r;
125 
126 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
127 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
128 	r = RREG32(mmUVD_CTX_DATA);
129 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
130 	return r;
131 }
132 
133 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
134 {
135 	unsigned long flags;
136 
137 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
138 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
139 	WREG32(mmUVD_CTX_DATA, (v));
140 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
141 }
142 
143 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
144 {
145 	unsigned long flags;
146 	u32 r;
147 
148 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
149 	WREG32(mmDIDT_IND_INDEX, (reg));
150 	r = RREG32(mmDIDT_IND_DATA);
151 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
152 	return r;
153 }
154 
155 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
156 {
157 	unsigned long flags;
158 
159 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
160 	WREG32(mmDIDT_IND_INDEX, (reg));
161 	WREG32(mmDIDT_IND_DATA, (v));
162 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
163 }
164 
165 static const u32 bonaire_golden_spm_registers[] =
166 {
167 	0xc200, 0xe0ffffff, 0xe0000000
168 };
169 
170 static const u32 bonaire_golden_common_registers[] =
171 {
172 	0x31dc, 0xffffffff, 0x00000800,
173 	0x31dd, 0xffffffff, 0x00000800,
174 	0x31e6, 0xffffffff, 0x00007fbf,
175 	0x31e7, 0xffffffff, 0x00007faf
176 };
177 
178 static const u32 bonaire_golden_registers[] =
179 {
180 	0xcd5, 0x00000333, 0x00000333,
181 	0xcd4, 0x000c0fc0, 0x00040200,
182 	0x2684, 0x00010000, 0x00058208,
183 	0xf000, 0xffff1fff, 0x00140000,
184 	0xf080, 0xfdfc0fff, 0x00000100,
185 	0xf08d, 0x40000000, 0x40000200,
186 	0x260c, 0xffffffff, 0x00000000,
187 	0x260d, 0xf00fffff, 0x00000400,
188 	0x260e, 0x0002021c, 0x00020200,
189 	0x31e, 0x00000080, 0x00000000,
190 	0x16ec, 0x000000f0, 0x00000070,
191 	0x16f0, 0xf0311fff, 0x80300000,
192 	0x263e, 0x73773777, 0x12010001,
193 	0xd43, 0x00810000, 0x408af000,
194 	0x1c0c, 0x31000111, 0x00000011,
195 	0xbd2, 0x73773777, 0x12010001,
196 	0x883, 0x00007fb6, 0x0021a1b1,
197 	0x884, 0x00007fb6, 0x002021b1,
198 	0x860, 0x00007fb6, 0x00002191,
199 	0x886, 0x00007fb6, 0x002121b1,
200 	0x887, 0x00007fb6, 0x002021b1,
201 	0x877, 0x00007fb6, 0x00002191,
202 	0x878, 0x00007fb6, 0x00002191,
203 	0xd8a, 0x0000003f, 0x0000000a,
204 	0xd8b, 0x0000003f, 0x0000000a,
205 	0xab9, 0x00073ffe, 0x000022a2,
206 	0x903, 0x000007ff, 0x00000000,
207 	0x2285, 0xf000003f, 0x00000007,
208 	0x22fc, 0x00002001, 0x00000001,
209 	0x22c9, 0xffffffff, 0x00ffffff,
210 	0xc281, 0x0000ff0f, 0x00000000,
211 	0xa293, 0x07ffffff, 0x06000000,
212 	0x136, 0x00000fff, 0x00000100,
213 	0xf9e, 0x00000001, 0x00000002,
214 	0x2440, 0x03000000, 0x0362c688,
215 	0x2300, 0x000000ff, 0x00000001,
216 	0x390, 0x00001fff, 0x00001fff,
217 	0x2418, 0x0000007f, 0x00000020,
218 	0x2542, 0x00010000, 0x00010000,
219 	0x2b05, 0x000003ff, 0x000000f3,
220 	0x2b03, 0xffffffff, 0x00001032
221 };
222 
223 static const u32 bonaire_mgcg_cgcg_init[] =
224 {
225 	0x3108, 0xffffffff, 0xfffffffc,
226 	0xc200, 0xffffffff, 0xe0000000,
227 	0xf0a8, 0xffffffff, 0x00000100,
228 	0xf082, 0xffffffff, 0x00000100,
229 	0xf0b0, 0xffffffff, 0xc0000100,
230 	0xf0b2, 0xffffffff, 0xc0000100,
231 	0xf0b1, 0xffffffff, 0xc0000100,
232 	0x1579, 0xffffffff, 0x00600100,
233 	0xf0a0, 0xffffffff, 0x00000100,
234 	0xf085, 0xffffffff, 0x06000100,
235 	0xf088, 0xffffffff, 0x00000100,
236 	0xf086, 0xffffffff, 0x06000100,
237 	0xf081, 0xffffffff, 0x00000100,
238 	0xf0b8, 0xffffffff, 0x00000100,
239 	0xf089, 0xffffffff, 0x00000100,
240 	0xf080, 0xffffffff, 0x00000100,
241 	0xf08c, 0xffffffff, 0x00000100,
242 	0xf08d, 0xffffffff, 0x00000100,
243 	0xf094, 0xffffffff, 0x00000100,
244 	0xf095, 0xffffffff, 0x00000100,
245 	0xf096, 0xffffffff, 0x00000100,
246 	0xf097, 0xffffffff, 0x00000100,
247 	0xf098, 0xffffffff, 0x00000100,
248 	0xf09f, 0xffffffff, 0x00000100,
249 	0xf09e, 0xffffffff, 0x00000100,
250 	0xf084, 0xffffffff, 0x06000100,
251 	0xf0a4, 0xffffffff, 0x00000100,
252 	0xf09d, 0xffffffff, 0x00000100,
253 	0xf0ad, 0xffffffff, 0x00000100,
254 	0xf0ac, 0xffffffff, 0x00000100,
255 	0xf09c, 0xffffffff, 0x00000100,
256 	0xc200, 0xffffffff, 0xe0000000,
257 	0xf008, 0xffffffff, 0x00010000,
258 	0xf009, 0xffffffff, 0x00030002,
259 	0xf00a, 0xffffffff, 0x00040007,
260 	0xf00b, 0xffffffff, 0x00060005,
261 	0xf00c, 0xffffffff, 0x00090008,
262 	0xf00d, 0xffffffff, 0x00010000,
263 	0xf00e, 0xffffffff, 0x00030002,
264 	0xf00f, 0xffffffff, 0x00040007,
265 	0xf010, 0xffffffff, 0x00060005,
266 	0xf011, 0xffffffff, 0x00090008,
267 	0xf012, 0xffffffff, 0x00010000,
268 	0xf013, 0xffffffff, 0x00030002,
269 	0xf014, 0xffffffff, 0x00040007,
270 	0xf015, 0xffffffff, 0x00060005,
271 	0xf016, 0xffffffff, 0x00090008,
272 	0xf017, 0xffffffff, 0x00010000,
273 	0xf018, 0xffffffff, 0x00030002,
274 	0xf019, 0xffffffff, 0x00040007,
275 	0xf01a, 0xffffffff, 0x00060005,
276 	0xf01b, 0xffffffff, 0x00090008,
277 	0xf01c, 0xffffffff, 0x00010000,
278 	0xf01d, 0xffffffff, 0x00030002,
279 	0xf01e, 0xffffffff, 0x00040007,
280 	0xf01f, 0xffffffff, 0x00060005,
281 	0xf020, 0xffffffff, 0x00090008,
282 	0xf021, 0xffffffff, 0x00010000,
283 	0xf022, 0xffffffff, 0x00030002,
284 	0xf023, 0xffffffff, 0x00040007,
285 	0xf024, 0xffffffff, 0x00060005,
286 	0xf025, 0xffffffff, 0x00090008,
287 	0xf026, 0xffffffff, 0x00010000,
288 	0xf027, 0xffffffff, 0x00030002,
289 	0xf028, 0xffffffff, 0x00040007,
290 	0xf029, 0xffffffff, 0x00060005,
291 	0xf02a, 0xffffffff, 0x00090008,
292 	0xf000, 0xffffffff, 0x96e00200,
293 	0x21c2, 0xffffffff, 0x00900100,
294 	0x3109, 0xffffffff, 0x0020003f,
295 	0xe, 0xffffffff, 0x0140001c,
296 	0xf, 0x000f0000, 0x000f0000,
297 	0x88, 0xffffffff, 0xc060000c,
298 	0x89, 0xc0000fff, 0x00000100,
299 	0x3e4, 0xffffffff, 0x00000100,
300 	0x3e6, 0x00000101, 0x00000000,
301 	0x82a, 0xffffffff, 0x00000104,
302 	0x1579, 0xff000fff, 0x00000100,
303 	0xc33, 0xc0000fff, 0x00000104,
304 	0x3079, 0x00000001, 0x00000001,
305 	0x3403, 0xff000ff0, 0x00000100,
306 	0x3603, 0xff000ff0, 0x00000100
307 };
308 
309 static const u32 spectre_golden_spm_registers[] =
310 {
311 	0xc200, 0xe0ffffff, 0xe0000000
312 };
313 
314 static const u32 spectre_golden_common_registers[] =
315 {
316 	0x31dc, 0xffffffff, 0x00000800,
317 	0x31dd, 0xffffffff, 0x00000800,
318 	0x31e6, 0xffffffff, 0x00007fbf,
319 	0x31e7, 0xffffffff, 0x00007faf
320 };
321 
322 static const u32 spectre_golden_registers[] =
323 {
324 	0xf000, 0xffff1fff, 0x96940200,
325 	0xf003, 0xffff0001, 0xff000000,
326 	0xf080, 0xfffc0fff, 0x00000100,
327 	0x1bb6, 0x00010101, 0x00010000,
328 	0x260d, 0xf00fffff, 0x00000400,
329 	0x260e, 0xfffffffc, 0x00020200,
330 	0x16ec, 0x000000f0, 0x00000070,
331 	0x16f0, 0xf0311fff, 0x80300000,
332 	0x263e, 0x73773777, 0x12010001,
333 	0x26df, 0x00ff0000, 0x00fc0000,
334 	0xbd2, 0x73773777, 0x12010001,
335 	0x2285, 0xf000003f, 0x00000007,
336 	0x22c9, 0xffffffff, 0x00ffffff,
337 	0xa0d4, 0x3f3f3fff, 0x00000082,
338 	0xa0d5, 0x0000003f, 0x00000000,
339 	0xf9e, 0x00000001, 0x00000002,
340 	0x244f, 0xffff03df, 0x00000004,
341 	0x31da, 0x00000008, 0x00000008,
342 	0x2300, 0x000008ff, 0x00000800,
343 	0x2542, 0x00010000, 0x00010000,
344 	0x2b03, 0xffffffff, 0x54763210,
345 	0x853e, 0x01ff01ff, 0x00000002,
346 	0x8526, 0x007ff800, 0x00200000,
347 	0x8057, 0xffffffff, 0x00000f40,
348 	0xc24d, 0xffffffff, 0x00000001
349 };
350 
351 static const u32 spectre_mgcg_cgcg_init[] =
352 {
353 	0x3108, 0xffffffff, 0xfffffffc,
354 	0xc200, 0xffffffff, 0xe0000000,
355 	0xf0a8, 0xffffffff, 0x00000100,
356 	0xf082, 0xffffffff, 0x00000100,
357 	0xf0b0, 0xffffffff, 0x00000100,
358 	0xf0b2, 0xffffffff, 0x00000100,
359 	0xf0b1, 0xffffffff, 0x00000100,
360 	0x1579, 0xffffffff, 0x00600100,
361 	0xf0a0, 0xffffffff, 0x00000100,
362 	0xf085, 0xffffffff, 0x06000100,
363 	0xf088, 0xffffffff, 0x00000100,
364 	0xf086, 0xffffffff, 0x06000100,
365 	0xf081, 0xffffffff, 0x00000100,
366 	0xf0b8, 0xffffffff, 0x00000100,
367 	0xf089, 0xffffffff, 0x00000100,
368 	0xf080, 0xffffffff, 0x00000100,
369 	0xf08c, 0xffffffff, 0x00000100,
370 	0xf08d, 0xffffffff, 0x00000100,
371 	0xf094, 0xffffffff, 0x00000100,
372 	0xf095, 0xffffffff, 0x00000100,
373 	0xf096, 0xffffffff, 0x00000100,
374 	0xf097, 0xffffffff, 0x00000100,
375 	0xf098, 0xffffffff, 0x00000100,
376 	0xf09f, 0xffffffff, 0x00000100,
377 	0xf09e, 0xffffffff, 0x00000100,
378 	0xf084, 0xffffffff, 0x06000100,
379 	0xf0a4, 0xffffffff, 0x00000100,
380 	0xf09d, 0xffffffff, 0x00000100,
381 	0xf0ad, 0xffffffff, 0x00000100,
382 	0xf0ac, 0xffffffff, 0x00000100,
383 	0xf09c, 0xffffffff, 0x00000100,
384 	0xc200, 0xffffffff, 0xe0000000,
385 	0xf008, 0xffffffff, 0x00010000,
386 	0xf009, 0xffffffff, 0x00030002,
387 	0xf00a, 0xffffffff, 0x00040007,
388 	0xf00b, 0xffffffff, 0x00060005,
389 	0xf00c, 0xffffffff, 0x00090008,
390 	0xf00d, 0xffffffff, 0x00010000,
391 	0xf00e, 0xffffffff, 0x00030002,
392 	0xf00f, 0xffffffff, 0x00040007,
393 	0xf010, 0xffffffff, 0x00060005,
394 	0xf011, 0xffffffff, 0x00090008,
395 	0xf012, 0xffffffff, 0x00010000,
396 	0xf013, 0xffffffff, 0x00030002,
397 	0xf014, 0xffffffff, 0x00040007,
398 	0xf015, 0xffffffff, 0x00060005,
399 	0xf016, 0xffffffff, 0x00090008,
400 	0xf017, 0xffffffff, 0x00010000,
401 	0xf018, 0xffffffff, 0x00030002,
402 	0xf019, 0xffffffff, 0x00040007,
403 	0xf01a, 0xffffffff, 0x00060005,
404 	0xf01b, 0xffffffff, 0x00090008,
405 	0xf01c, 0xffffffff, 0x00010000,
406 	0xf01d, 0xffffffff, 0x00030002,
407 	0xf01e, 0xffffffff, 0x00040007,
408 	0xf01f, 0xffffffff, 0x00060005,
409 	0xf020, 0xffffffff, 0x00090008,
410 	0xf021, 0xffffffff, 0x00010000,
411 	0xf022, 0xffffffff, 0x00030002,
412 	0xf023, 0xffffffff, 0x00040007,
413 	0xf024, 0xffffffff, 0x00060005,
414 	0xf025, 0xffffffff, 0x00090008,
415 	0xf026, 0xffffffff, 0x00010000,
416 	0xf027, 0xffffffff, 0x00030002,
417 	0xf028, 0xffffffff, 0x00040007,
418 	0xf029, 0xffffffff, 0x00060005,
419 	0xf02a, 0xffffffff, 0x00090008,
420 	0xf02b, 0xffffffff, 0x00010000,
421 	0xf02c, 0xffffffff, 0x00030002,
422 	0xf02d, 0xffffffff, 0x00040007,
423 	0xf02e, 0xffffffff, 0x00060005,
424 	0xf02f, 0xffffffff, 0x00090008,
425 	0xf000, 0xffffffff, 0x96e00200,
426 	0x21c2, 0xffffffff, 0x00900100,
427 	0x3109, 0xffffffff, 0x0020003f,
428 	0xe, 0xffffffff, 0x0140001c,
429 	0xf, 0x000f0000, 0x000f0000,
430 	0x88, 0xffffffff, 0xc060000c,
431 	0x89, 0xc0000fff, 0x00000100,
432 	0x3e4, 0xffffffff, 0x00000100,
433 	0x3e6, 0x00000101, 0x00000000,
434 	0x82a, 0xffffffff, 0x00000104,
435 	0x1579, 0xff000fff, 0x00000100,
436 	0xc33, 0xc0000fff, 0x00000104,
437 	0x3079, 0x00000001, 0x00000001,
438 	0x3403, 0xff000ff0, 0x00000100,
439 	0x3603, 0xff000ff0, 0x00000100
440 };
441 
442 static const u32 kalindi_golden_spm_registers[] =
443 {
444 	0xc200, 0xe0ffffff, 0xe0000000
445 };
446 
447 static const u32 kalindi_golden_common_registers[] =
448 {
449 	0x31dc, 0xffffffff, 0x00000800,
450 	0x31dd, 0xffffffff, 0x00000800,
451 	0x31e6, 0xffffffff, 0x00007fbf,
452 	0x31e7, 0xffffffff, 0x00007faf
453 };
454 
455 static const u32 kalindi_golden_registers[] =
456 {
457 	0xf000, 0xffffdfff, 0x6e944040,
458 	0x1579, 0xff607fff, 0xfc000100,
459 	0xf088, 0xff000fff, 0x00000100,
460 	0xf089, 0xff000fff, 0x00000100,
461 	0xf080, 0xfffc0fff, 0x00000100,
462 	0x1bb6, 0x00010101, 0x00010000,
463 	0x260c, 0xffffffff, 0x00000000,
464 	0x260d, 0xf00fffff, 0x00000400,
465 	0x16ec, 0x000000f0, 0x00000070,
466 	0x16f0, 0xf0311fff, 0x80300000,
467 	0x263e, 0x73773777, 0x12010001,
468 	0x263f, 0xffffffff, 0x00000010,
469 	0x26df, 0x00ff0000, 0x00fc0000,
470 	0x200c, 0x00001f0f, 0x0000100a,
471 	0xbd2, 0x73773777, 0x12010001,
472 	0x902, 0x000fffff, 0x000c007f,
473 	0x2285, 0xf000003f, 0x00000007,
474 	0x22c9, 0x3fff3fff, 0x00ffcfff,
475 	0xc281, 0x0000ff0f, 0x00000000,
476 	0xa293, 0x07ffffff, 0x06000000,
477 	0x136, 0x00000fff, 0x00000100,
478 	0xf9e, 0x00000001, 0x00000002,
479 	0x31da, 0x00000008, 0x00000008,
480 	0x2300, 0x000000ff, 0x00000003,
481 	0x853e, 0x01ff01ff, 0x00000002,
482 	0x8526, 0x007ff800, 0x00200000,
483 	0x8057, 0xffffffff, 0x00000f40,
484 	0x2231, 0x001f3ae3, 0x00000082,
485 	0x2235, 0x0000001f, 0x00000010,
486 	0xc24d, 0xffffffff, 0x00000000
487 };
488 
489 static const u32 kalindi_mgcg_cgcg_init[] =
490 {
491 	0x3108, 0xffffffff, 0xfffffffc,
492 	0xc200, 0xffffffff, 0xe0000000,
493 	0xf0a8, 0xffffffff, 0x00000100,
494 	0xf082, 0xffffffff, 0x00000100,
495 	0xf0b0, 0xffffffff, 0x00000100,
496 	0xf0b2, 0xffffffff, 0x00000100,
497 	0xf0b1, 0xffffffff, 0x00000100,
498 	0x1579, 0xffffffff, 0x00600100,
499 	0xf0a0, 0xffffffff, 0x00000100,
500 	0xf085, 0xffffffff, 0x06000100,
501 	0xf088, 0xffffffff, 0x00000100,
502 	0xf086, 0xffffffff, 0x06000100,
503 	0xf081, 0xffffffff, 0x00000100,
504 	0xf0b8, 0xffffffff, 0x00000100,
505 	0xf089, 0xffffffff, 0x00000100,
506 	0xf080, 0xffffffff, 0x00000100,
507 	0xf08c, 0xffffffff, 0x00000100,
508 	0xf08d, 0xffffffff, 0x00000100,
509 	0xf094, 0xffffffff, 0x00000100,
510 	0xf095, 0xffffffff, 0x00000100,
511 	0xf096, 0xffffffff, 0x00000100,
512 	0xf097, 0xffffffff, 0x00000100,
513 	0xf098, 0xffffffff, 0x00000100,
514 	0xf09f, 0xffffffff, 0x00000100,
515 	0xf09e, 0xffffffff, 0x00000100,
516 	0xf084, 0xffffffff, 0x06000100,
517 	0xf0a4, 0xffffffff, 0x00000100,
518 	0xf09d, 0xffffffff, 0x00000100,
519 	0xf0ad, 0xffffffff, 0x00000100,
520 	0xf0ac, 0xffffffff, 0x00000100,
521 	0xf09c, 0xffffffff, 0x00000100,
522 	0xc200, 0xffffffff, 0xe0000000,
523 	0xf008, 0xffffffff, 0x00010000,
524 	0xf009, 0xffffffff, 0x00030002,
525 	0xf00a, 0xffffffff, 0x00040007,
526 	0xf00b, 0xffffffff, 0x00060005,
527 	0xf00c, 0xffffffff, 0x00090008,
528 	0xf00d, 0xffffffff, 0x00010000,
529 	0xf00e, 0xffffffff, 0x00030002,
530 	0xf00f, 0xffffffff, 0x00040007,
531 	0xf010, 0xffffffff, 0x00060005,
532 	0xf011, 0xffffffff, 0x00090008,
533 	0xf000, 0xffffffff, 0x96e00200,
534 	0x21c2, 0xffffffff, 0x00900100,
535 	0x3109, 0xffffffff, 0x0020003f,
536 	0xe, 0xffffffff, 0x0140001c,
537 	0xf, 0x000f0000, 0x000f0000,
538 	0x88, 0xffffffff, 0xc060000c,
539 	0x89, 0xc0000fff, 0x00000100,
540 	0x82a, 0xffffffff, 0x00000104,
541 	0x1579, 0xff000fff, 0x00000100,
542 	0xc33, 0xc0000fff, 0x00000104,
543 	0x3079, 0x00000001, 0x00000001,
544 	0x3403, 0xff000ff0, 0x00000100,
545 	0x3603, 0xff000ff0, 0x00000100
546 };
547 
548 static const u32 hawaii_golden_spm_registers[] =
549 {
550 	0xc200, 0xe0ffffff, 0xe0000000
551 };
552 
553 static const u32 hawaii_golden_common_registers[] =
554 {
555 	0xc200, 0xffffffff, 0xe0000000,
556 	0xa0d4, 0xffffffff, 0x3a00161a,
557 	0xa0d5, 0xffffffff, 0x0000002e,
558 	0x2684, 0xffffffff, 0x00018208,
559 	0x263e, 0xffffffff, 0x12011003
560 };
561 
562 static const u32 hawaii_golden_registers[] =
563 {
564 	0xcd5, 0x00000333, 0x00000333,
565 	0x2684, 0x00010000, 0x00058208,
566 	0x260c, 0xffffffff, 0x00000000,
567 	0x260d, 0xf00fffff, 0x00000400,
568 	0x260e, 0x0002021c, 0x00020200,
569 	0x31e, 0x00000080, 0x00000000,
570 	0x16ec, 0x000000f0, 0x00000070,
571 	0x16f0, 0xf0311fff, 0x80300000,
572 	0xd43, 0x00810000, 0x408af000,
573 	0x1c0c, 0x31000111, 0x00000011,
574 	0xbd2, 0x73773777, 0x12010001,
575 	0x848, 0x0000007f, 0x0000001b,
576 	0x877, 0x00007fb6, 0x00002191,
577 	0xd8a, 0x0000003f, 0x0000000a,
578 	0xd8b, 0x0000003f, 0x0000000a,
579 	0xab9, 0x00073ffe, 0x000022a2,
580 	0x903, 0x000007ff, 0x00000000,
581 	0x22fc, 0x00002001, 0x00000001,
582 	0x22c9, 0xffffffff, 0x00ffffff,
583 	0xc281, 0x0000ff0f, 0x00000000,
584 	0xa293, 0x07ffffff, 0x06000000,
585 	0xf9e, 0x00000001, 0x00000002,
586 	0x31da, 0x00000008, 0x00000008,
587 	0x31dc, 0x00000f00, 0x00000800,
588 	0x31dd, 0x00000f00, 0x00000800,
589 	0x31e6, 0x00ffffff, 0x00ff7fbf,
590 	0x31e7, 0x00ffffff, 0x00ff7faf,
591 	0x2300, 0x000000ff, 0x00000800,
592 	0x390, 0x00001fff, 0x00001fff,
593 	0x2418, 0x0000007f, 0x00000020,
594 	0x2542, 0x00010000, 0x00010000,
595 	0x2b80, 0x00100000, 0x000ff07c,
596 	0x2b05, 0x000003ff, 0x0000000f,
597 	0x2b04, 0xffffffff, 0x7564fdec,
598 	0x2b03, 0xffffffff, 0x3120b9a8,
599 	0x2b02, 0x20000000, 0x0f9c0000
600 };
601 
602 static const u32 hawaii_mgcg_cgcg_init[] =
603 {
604 	0x3108, 0xffffffff, 0xfffffffd,
605 	0xc200, 0xffffffff, 0xe0000000,
606 	0xf0a8, 0xffffffff, 0x00000100,
607 	0xf082, 0xffffffff, 0x00000100,
608 	0xf0b0, 0xffffffff, 0x00000100,
609 	0xf0b2, 0xffffffff, 0x00000100,
610 	0xf0b1, 0xffffffff, 0x00000100,
611 	0x1579, 0xffffffff, 0x00200100,
612 	0xf0a0, 0xffffffff, 0x00000100,
613 	0xf085, 0xffffffff, 0x06000100,
614 	0xf088, 0xffffffff, 0x00000100,
615 	0xf086, 0xffffffff, 0x06000100,
616 	0xf081, 0xffffffff, 0x00000100,
617 	0xf0b8, 0xffffffff, 0x00000100,
618 	0xf089, 0xffffffff, 0x00000100,
619 	0xf080, 0xffffffff, 0x00000100,
620 	0xf08c, 0xffffffff, 0x00000100,
621 	0xf08d, 0xffffffff, 0x00000100,
622 	0xf094, 0xffffffff, 0x00000100,
623 	0xf095, 0xffffffff, 0x00000100,
624 	0xf096, 0xffffffff, 0x00000100,
625 	0xf097, 0xffffffff, 0x00000100,
626 	0xf098, 0xffffffff, 0x00000100,
627 	0xf09f, 0xffffffff, 0x00000100,
628 	0xf09e, 0xffffffff, 0x00000100,
629 	0xf084, 0xffffffff, 0x06000100,
630 	0xf0a4, 0xffffffff, 0x00000100,
631 	0xf09d, 0xffffffff, 0x00000100,
632 	0xf0ad, 0xffffffff, 0x00000100,
633 	0xf0ac, 0xffffffff, 0x00000100,
634 	0xf09c, 0xffffffff, 0x00000100,
635 	0xc200, 0xffffffff, 0xe0000000,
636 	0xf008, 0xffffffff, 0x00010000,
637 	0xf009, 0xffffffff, 0x00030002,
638 	0xf00a, 0xffffffff, 0x00040007,
639 	0xf00b, 0xffffffff, 0x00060005,
640 	0xf00c, 0xffffffff, 0x00090008,
641 	0xf00d, 0xffffffff, 0x00010000,
642 	0xf00e, 0xffffffff, 0x00030002,
643 	0xf00f, 0xffffffff, 0x00040007,
644 	0xf010, 0xffffffff, 0x00060005,
645 	0xf011, 0xffffffff, 0x00090008,
646 	0xf012, 0xffffffff, 0x00010000,
647 	0xf013, 0xffffffff, 0x00030002,
648 	0xf014, 0xffffffff, 0x00040007,
649 	0xf015, 0xffffffff, 0x00060005,
650 	0xf016, 0xffffffff, 0x00090008,
651 	0xf017, 0xffffffff, 0x00010000,
652 	0xf018, 0xffffffff, 0x00030002,
653 	0xf019, 0xffffffff, 0x00040007,
654 	0xf01a, 0xffffffff, 0x00060005,
655 	0xf01b, 0xffffffff, 0x00090008,
656 	0xf01c, 0xffffffff, 0x00010000,
657 	0xf01d, 0xffffffff, 0x00030002,
658 	0xf01e, 0xffffffff, 0x00040007,
659 	0xf01f, 0xffffffff, 0x00060005,
660 	0xf020, 0xffffffff, 0x00090008,
661 	0xf021, 0xffffffff, 0x00010000,
662 	0xf022, 0xffffffff, 0x00030002,
663 	0xf023, 0xffffffff, 0x00040007,
664 	0xf024, 0xffffffff, 0x00060005,
665 	0xf025, 0xffffffff, 0x00090008,
666 	0xf026, 0xffffffff, 0x00010000,
667 	0xf027, 0xffffffff, 0x00030002,
668 	0xf028, 0xffffffff, 0x00040007,
669 	0xf029, 0xffffffff, 0x00060005,
670 	0xf02a, 0xffffffff, 0x00090008,
671 	0xf02b, 0xffffffff, 0x00010000,
672 	0xf02c, 0xffffffff, 0x00030002,
673 	0xf02d, 0xffffffff, 0x00040007,
674 	0xf02e, 0xffffffff, 0x00060005,
675 	0xf02f, 0xffffffff, 0x00090008,
676 	0xf030, 0xffffffff, 0x00010000,
677 	0xf031, 0xffffffff, 0x00030002,
678 	0xf032, 0xffffffff, 0x00040007,
679 	0xf033, 0xffffffff, 0x00060005,
680 	0xf034, 0xffffffff, 0x00090008,
681 	0xf035, 0xffffffff, 0x00010000,
682 	0xf036, 0xffffffff, 0x00030002,
683 	0xf037, 0xffffffff, 0x00040007,
684 	0xf038, 0xffffffff, 0x00060005,
685 	0xf039, 0xffffffff, 0x00090008,
686 	0xf03a, 0xffffffff, 0x00010000,
687 	0xf03b, 0xffffffff, 0x00030002,
688 	0xf03c, 0xffffffff, 0x00040007,
689 	0xf03d, 0xffffffff, 0x00060005,
690 	0xf03e, 0xffffffff, 0x00090008,
691 	0x30c6, 0xffffffff, 0x00020200,
692 	0xcd4, 0xffffffff, 0x00000200,
693 	0x570, 0xffffffff, 0x00000400,
694 	0x157a, 0xffffffff, 0x00000000,
695 	0xbd4, 0xffffffff, 0x00000902,
696 	0xf000, 0xffffffff, 0x96940200,
697 	0x21c2, 0xffffffff, 0x00900100,
698 	0x3109, 0xffffffff, 0x0020003f,
699 	0xe, 0xffffffff, 0x0140001c,
700 	0xf, 0x000f0000, 0x000f0000,
701 	0x88, 0xffffffff, 0xc060000c,
702 	0x89, 0xc0000fff, 0x00000100,
703 	0x3e4, 0xffffffff, 0x00000100,
704 	0x3e6, 0x00000101, 0x00000000,
705 	0x82a, 0xffffffff, 0x00000104,
706 	0x1579, 0xff000fff, 0x00000100,
707 	0xc33, 0xc0000fff, 0x00000104,
708 	0x3079, 0x00000001, 0x00000001,
709 	0x3403, 0xff000ff0, 0x00000100,
710 	0x3603, 0xff000ff0, 0x00000100
711 };
712 
713 static const u32 godavari_golden_registers[] =
714 {
715 	0x1579, 0xff607fff, 0xfc000100,
716 	0x1bb6, 0x00010101, 0x00010000,
717 	0x260c, 0xffffffff, 0x00000000,
718 	0x260c0, 0xf00fffff, 0x00000400,
719 	0x184c, 0xffffffff, 0x00010000,
720 	0x16ec, 0x000000f0, 0x00000070,
721 	0x16f0, 0xf0311fff, 0x80300000,
722 	0x263e, 0x73773777, 0x12010001,
723 	0x263f, 0xffffffff, 0x00000010,
724 	0x200c, 0x00001f0f, 0x0000100a,
725 	0xbd2, 0x73773777, 0x12010001,
726 	0x902, 0x000fffff, 0x000c007f,
727 	0x2285, 0xf000003f, 0x00000007,
728 	0x22c9, 0xffffffff, 0x00ff0fff,
729 	0xc281, 0x0000ff0f, 0x00000000,
730 	0xa293, 0x07ffffff, 0x06000000,
731 	0x136, 0x00000fff, 0x00000100,
732 	0x3405, 0x00010000, 0x00810001,
733 	0x3605, 0x00010000, 0x00810001,
734 	0xf9e, 0x00000001, 0x00000002,
735 	0x31da, 0x00000008, 0x00000008,
736 	0x31dc, 0x00000f00, 0x00000800,
737 	0x31dd, 0x00000f00, 0x00000800,
738 	0x31e6, 0x00ffffff, 0x00ff7fbf,
739 	0x31e7, 0x00ffffff, 0x00ff7faf,
740 	0x2300, 0x000000ff, 0x00000001,
741 	0x853e, 0x01ff01ff, 0x00000002,
742 	0x8526, 0x007ff800, 0x00200000,
743 	0x8057, 0xffffffff, 0x00000f40,
744 	0x2231, 0x001f3ae3, 0x00000082,
745 	0x2235, 0x0000001f, 0x00000010,
746 	0xc24d, 0xffffffff, 0x00000000
747 };
748 
749 static void cik_init_golden_registers(struct amdgpu_device *adev)
750 {
751 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
752 	mutex_lock(&adev->grbm_idx_mutex);
753 
754 	switch (adev->asic_type) {
755 	case CHIP_BONAIRE:
756 		amdgpu_program_register_sequence(adev,
757 						 bonaire_mgcg_cgcg_init,
758 						 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
759 		amdgpu_program_register_sequence(adev,
760 						 bonaire_golden_registers,
761 						 (const u32)ARRAY_SIZE(bonaire_golden_registers));
762 		amdgpu_program_register_sequence(adev,
763 						 bonaire_golden_common_registers,
764 						 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
765 		amdgpu_program_register_sequence(adev,
766 						 bonaire_golden_spm_registers,
767 						 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
768 		break;
769 	case CHIP_KABINI:
770 		amdgpu_program_register_sequence(adev,
771 						 kalindi_mgcg_cgcg_init,
772 						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
773 		amdgpu_program_register_sequence(adev,
774 						 kalindi_golden_registers,
775 						 (const u32)ARRAY_SIZE(kalindi_golden_registers));
776 		amdgpu_program_register_sequence(adev,
777 						 kalindi_golden_common_registers,
778 						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
779 		amdgpu_program_register_sequence(adev,
780 						 kalindi_golden_spm_registers,
781 						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
782 		break;
783 	case CHIP_MULLINS:
784 		amdgpu_program_register_sequence(adev,
785 						 kalindi_mgcg_cgcg_init,
786 						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
787 		amdgpu_program_register_sequence(adev,
788 						 godavari_golden_registers,
789 						 (const u32)ARRAY_SIZE(godavari_golden_registers));
790 		amdgpu_program_register_sequence(adev,
791 						 kalindi_golden_common_registers,
792 						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
793 		amdgpu_program_register_sequence(adev,
794 						 kalindi_golden_spm_registers,
795 						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
796 		break;
797 	case CHIP_KAVERI:
798 		amdgpu_program_register_sequence(adev,
799 						 spectre_mgcg_cgcg_init,
800 						 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
801 		amdgpu_program_register_sequence(adev,
802 						 spectre_golden_registers,
803 						 (const u32)ARRAY_SIZE(spectre_golden_registers));
804 		amdgpu_program_register_sequence(adev,
805 						 spectre_golden_common_registers,
806 						 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
807 		amdgpu_program_register_sequence(adev,
808 						 spectre_golden_spm_registers,
809 						 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
810 		break;
811 	case CHIP_HAWAII:
812 		amdgpu_program_register_sequence(adev,
813 						 hawaii_mgcg_cgcg_init,
814 						 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
815 		amdgpu_program_register_sequence(adev,
816 						 hawaii_golden_registers,
817 						 (const u32)ARRAY_SIZE(hawaii_golden_registers));
818 		amdgpu_program_register_sequence(adev,
819 						 hawaii_golden_common_registers,
820 						 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
821 		amdgpu_program_register_sequence(adev,
822 						 hawaii_golden_spm_registers,
823 						 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
824 		break;
825 	default:
826 		break;
827 	}
828 	mutex_unlock(&adev->grbm_idx_mutex);
829 }
830 
831 /**
832  * cik_get_xclk - get the xclk
833  *
834  * @adev: amdgpu_device pointer
835  *
836  * Returns the reference clock used by the gfx engine
837  * (CIK).
838  */
839 static u32 cik_get_xclk(struct amdgpu_device *adev)
840 {
841 	u32 reference_clock = adev->clock.spll.reference_freq;
842 
843 	if (adev->flags & AMD_IS_APU) {
844 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
845 			return reference_clock / 2;
846 	} else {
847 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
848 			return reference_clock / 4;
849 	}
850 	return reference_clock;
851 }
852 
853 /**
854  * cik_srbm_select - select specific register instances
855  *
856  * @adev: amdgpu_device pointer
857  * @me: selected ME (micro engine)
858  * @pipe: pipe
859  * @queue: queue
860  * @vmid: VMID
861  *
862  * Switches the currently active registers instances.  Some
863  * registers are instanced per VMID, others are instanced per
864  * me/pipe/queue combination.
865  */
866 void cik_srbm_select(struct amdgpu_device *adev,
867 		     u32 me, u32 pipe, u32 queue, u32 vmid)
868 {
869 	u32 srbm_gfx_cntl =
870 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
871 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
872 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
873 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
874 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
875 }
876 
877 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
878 {
879 	uint32_t tmp;
880 
881 	tmp = RREG32(mmCONFIG_CNTL);
882 	if (state == false)
883 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
884 	else
885 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
886 	WREG32(mmCONFIG_CNTL, tmp);
887 }
888 
889 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
890 {
891 	u32 bus_cntl;
892 	u32 d1vga_control = 0;
893 	u32 d2vga_control = 0;
894 	u32 vga_render_control = 0;
895 	u32 rom_cntl;
896 	bool r;
897 
898 	bus_cntl = RREG32(mmBUS_CNTL);
899 	if (adev->mode_info.num_crtc) {
900 		d1vga_control = RREG32(mmD1VGA_CONTROL);
901 		d2vga_control = RREG32(mmD2VGA_CONTROL);
902 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
903 	}
904 	rom_cntl = RREG32_SMC(ixROM_CNTL);
905 
906 	/* enable the rom */
907 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
908 	if (adev->mode_info.num_crtc) {
909 		/* Disable VGA mode */
910 		WREG32(mmD1VGA_CONTROL,
911 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
912 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
913 		WREG32(mmD2VGA_CONTROL,
914 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
915 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
916 		WREG32(mmVGA_RENDER_CONTROL,
917 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
918 	}
919 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
920 
921 	r = amdgpu_read_bios(adev);
922 
923 	/* restore regs */
924 	WREG32(mmBUS_CNTL, bus_cntl);
925 	if (adev->mode_info.num_crtc) {
926 		WREG32(mmD1VGA_CONTROL, d1vga_control);
927 		WREG32(mmD2VGA_CONTROL, d2vga_control);
928 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
929 	}
930 	WREG32_SMC(ixROM_CNTL, rom_cntl);
931 	return r;
932 }
933 
934 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
935 				   u8 *bios, u32 length_bytes)
936 {
937 	u32 *dw_ptr;
938 	unsigned long flags;
939 	u32 i, length_dw;
940 
941 	if (bios == NULL)
942 		return false;
943 	if (length_bytes == 0)
944 		return false;
945 	/* APU vbios image is part of sbios image */
946 	if (adev->flags & AMD_IS_APU)
947 		return false;
948 
949 	dw_ptr = (u32 *)bios;
950 	length_dw = ALIGN(length_bytes, 4) / 4;
951 	/* take the smc lock since we are using the smc index */
952 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
953 	/* set rom index to 0 */
954 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
955 	WREG32(mmSMC_IND_DATA_0, 0);
956 	/* set index to data for continous read */
957 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
958 	for (i = 0; i < length_dw; i++)
959 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
960 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
961 
962 	return true;
963 }
964 
965 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
966 	{mmGRBM_STATUS, false},
967 	{mmGB_ADDR_CONFIG, false},
968 	{mmMC_ARB_RAMCFG, false},
969 	{mmGB_TILE_MODE0, false},
970 	{mmGB_TILE_MODE1, false},
971 	{mmGB_TILE_MODE2, false},
972 	{mmGB_TILE_MODE3, false},
973 	{mmGB_TILE_MODE4, false},
974 	{mmGB_TILE_MODE5, false},
975 	{mmGB_TILE_MODE6, false},
976 	{mmGB_TILE_MODE7, false},
977 	{mmGB_TILE_MODE8, false},
978 	{mmGB_TILE_MODE9, false},
979 	{mmGB_TILE_MODE10, false},
980 	{mmGB_TILE_MODE11, false},
981 	{mmGB_TILE_MODE12, false},
982 	{mmGB_TILE_MODE13, false},
983 	{mmGB_TILE_MODE14, false},
984 	{mmGB_TILE_MODE15, false},
985 	{mmGB_TILE_MODE16, false},
986 	{mmGB_TILE_MODE17, false},
987 	{mmGB_TILE_MODE18, false},
988 	{mmGB_TILE_MODE19, false},
989 	{mmGB_TILE_MODE20, false},
990 	{mmGB_TILE_MODE21, false},
991 	{mmGB_TILE_MODE22, false},
992 	{mmGB_TILE_MODE23, false},
993 	{mmGB_TILE_MODE24, false},
994 	{mmGB_TILE_MODE25, false},
995 	{mmGB_TILE_MODE26, false},
996 	{mmGB_TILE_MODE27, false},
997 	{mmGB_TILE_MODE28, false},
998 	{mmGB_TILE_MODE29, false},
999 	{mmGB_TILE_MODE30, false},
1000 	{mmGB_TILE_MODE31, false},
1001 	{mmGB_MACROTILE_MODE0, false},
1002 	{mmGB_MACROTILE_MODE1, false},
1003 	{mmGB_MACROTILE_MODE2, false},
1004 	{mmGB_MACROTILE_MODE3, false},
1005 	{mmGB_MACROTILE_MODE4, false},
1006 	{mmGB_MACROTILE_MODE5, false},
1007 	{mmGB_MACROTILE_MODE6, false},
1008 	{mmGB_MACROTILE_MODE7, false},
1009 	{mmGB_MACROTILE_MODE8, false},
1010 	{mmGB_MACROTILE_MODE9, false},
1011 	{mmGB_MACROTILE_MODE10, false},
1012 	{mmGB_MACROTILE_MODE11, false},
1013 	{mmGB_MACROTILE_MODE12, false},
1014 	{mmGB_MACROTILE_MODE13, false},
1015 	{mmGB_MACROTILE_MODE14, false},
1016 	{mmGB_MACROTILE_MODE15, false},
1017 	{mmCC_RB_BACKEND_DISABLE, false, true},
1018 	{mmGC_USER_RB_BACKEND_DISABLE, false, true},
1019 	{mmGB_BACKEND_MAP, false, false},
1020 	{mmPA_SC_RASTER_CONFIG, false, true},
1021 	{mmPA_SC_RASTER_CONFIG_1, false, true},
1022 };
1023 
1024 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1025 					  u32 se_num, u32 sh_num,
1026 					  u32 reg_offset)
1027 {
1028 	uint32_t val;
1029 
1030 	mutex_lock(&adev->grbm_idx_mutex);
1031 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1032 		gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1033 
1034 	val = RREG32(reg_offset);
1035 
1036 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1037 		gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1038 	mutex_unlock(&adev->grbm_idx_mutex);
1039 	return val;
1040 }
1041 
1042 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1043 			     u32 sh_num, u32 reg_offset, u32 *value)
1044 {
1045 	uint32_t i;
1046 
1047 	*value = 0;
1048 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1049 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1050 			continue;
1051 
1052 		if (!cik_allowed_read_registers[i].untouched)
1053 			*value = cik_allowed_read_registers[i].grbm_indexed ?
1054 				 cik_read_indexed_register(adev, se_num,
1055 							   sh_num, reg_offset) :
1056 				 RREG32(reg_offset);
1057 		return 0;
1058 	}
1059 	return -EINVAL;
1060 }
1061 
1062 struct kv_reset_save_regs {
1063 	u32 gmcon_reng_execute;
1064 	u32 gmcon_misc;
1065 	u32 gmcon_misc3;
1066 };
1067 
1068 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1069 				   struct kv_reset_save_regs *save)
1070 {
1071 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1072 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1073 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1074 
1075 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1076 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1077 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1078 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1079 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1080 }
1081 
1082 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1083 				      struct kv_reset_save_regs *save)
1084 {
1085 	int i;
1086 
1087 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1088 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1089 
1090 	for (i = 0; i < 5; i++)
1091 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1092 
1093 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1094 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1095 
1096 	for (i = 0; i < 5; i++)
1097 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1098 
1099 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1100 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1101 
1102 	for (i = 0; i < 5; i++)
1103 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1104 
1105 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1106 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1107 
1108 	for (i = 0; i < 5; i++)
1109 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1110 
1111 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1112 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1113 
1114 	for (i = 0; i < 5; i++)
1115 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1116 
1117 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1118 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1119 
1120 	for (i = 0; i < 5; i++)
1121 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1122 
1123 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1124 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1125 
1126 	for (i = 0; i < 5; i++)
1127 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1128 
1129 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1130 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1131 
1132 	for (i = 0; i < 5; i++)
1133 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1134 
1135 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1136 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1137 
1138 	for (i = 0; i < 5; i++)
1139 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1140 
1141 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1142 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1143 
1144 	for (i = 0; i < 5; i++)
1145 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1146 
1147 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1148 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1149 
1150 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1151 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1152 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1153 }
1154 
1155 static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1156 {
1157 	struct kv_reset_save_regs kv_save = { 0 };
1158 	u32 i;
1159 
1160 	dev_info(adev->dev, "GPU pci config reset\n");
1161 
1162 	if (adev->flags & AMD_IS_APU)
1163 		kv_save_regs_for_reset(adev, &kv_save);
1164 
1165 	/* disable BM */
1166 	pci_clear_master(adev->pdev);
1167 	/* reset */
1168 	amdgpu_pci_config_reset(adev);
1169 
1170 	udelay(100);
1171 
1172 	/* wait for asic to come out of reset */
1173 	for (i = 0; i < adev->usec_timeout; i++) {
1174 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
1175 			break;
1176 		udelay(1);
1177 	}
1178 
1179 	/* does asic init need to be run first??? */
1180 	if (adev->flags & AMD_IS_APU)
1181 		kv_restore_regs_for_reset(adev, &kv_save);
1182 }
1183 
1184 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1185 {
1186 	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1187 
1188 	if (hung)
1189 		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1190 	else
1191 		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1192 
1193 	WREG32(mmBIOS_SCRATCH_3, tmp);
1194 }
1195 
1196 /**
1197  * cik_asic_reset - soft reset GPU
1198  *
1199  * @adev: amdgpu_device pointer
1200  *
1201  * Look up which blocks are hung and attempt
1202  * to reset them.
1203  * Returns 0 for success.
1204  */
1205 static int cik_asic_reset(struct amdgpu_device *adev)
1206 {
1207 	cik_set_bios_scratch_engine_hung(adev, true);
1208 
1209 	cik_gpu_pci_config_reset(adev);
1210 
1211 	cik_set_bios_scratch_engine_hung(adev, false);
1212 
1213 	return 0;
1214 }
1215 
1216 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1217 			      u32 cntl_reg, u32 status_reg)
1218 {
1219 	int r, i;
1220 	struct atom_clock_dividers dividers;
1221 	uint32_t tmp;
1222 
1223 	r = amdgpu_atombios_get_clock_dividers(adev,
1224 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1225 					       clock, false, &dividers);
1226 	if (r)
1227 		return r;
1228 
1229 	tmp = RREG32_SMC(cntl_reg);
1230 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1231 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1232 	tmp |= dividers.post_divider;
1233 	WREG32_SMC(cntl_reg, tmp);
1234 
1235 	for (i = 0; i < 100; i++) {
1236 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1237 			break;
1238 		mdelay(10);
1239 	}
1240 	if (i == 100)
1241 		return -ETIMEDOUT;
1242 
1243 	return 0;
1244 }
1245 
1246 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1247 {
1248 	int r = 0;
1249 
1250 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1251 	if (r)
1252 		return r;
1253 
1254 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1255 	return r;
1256 }
1257 
1258 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1259 {
1260 	int r, i;
1261 	struct atom_clock_dividers dividers;
1262 	u32 tmp;
1263 
1264 	r = amdgpu_atombios_get_clock_dividers(adev,
1265 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1266 					       ecclk, false, &dividers);
1267 	if (r)
1268 		return r;
1269 
1270 	for (i = 0; i < 100; i++) {
1271 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1272 			break;
1273 		mdelay(10);
1274 	}
1275 	if (i == 100)
1276 		return -ETIMEDOUT;
1277 
1278 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1279 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1280 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1281 	tmp |= dividers.post_divider;
1282 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1283 
1284 	for (i = 0; i < 100; i++) {
1285 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1286 			break;
1287 		mdelay(10);
1288 	}
1289 	if (i == 100)
1290 		return -ETIMEDOUT;
1291 
1292 	return 0;
1293 }
1294 
1295 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1296 {
1297 	struct pci_dev *root = adev->pdev->bus->self;
1298 	int bridge_pos, gpu_pos;
1299 	u32 speed_cntl, current_data_rate;
1300 	int i;
1301 	u16 tmp16;
1302 
1303 	if (pci_is_root_bus(adev->pdev->bus))
1304 		return;
1305 
1306 	if (amdgpu_pcie_gen2 == 0)
1307 		return;
1308 
1309 	if (adev->flags & AMD_IS_APU)
1310 		return;
1311 
1312 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1313 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1314 		return;
1315 
1316 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1317 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1318 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1319 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1320 		if (current_data_rate == 2) {
1321 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1322 			return;
1323 		}
1324 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1325 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1326 		if (current_data_rate == 1) {
1327 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1328 			return;
1329 		}
1330 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1331 	}
1332 
1333 	bridge_pos = pci_pcie_cap(root);
1334 	if (!bridge_pos)
1335 		return;
1336 
1337 	gpu_pos = pci_pcie_cap(adev->pdev);
1338 	if (!gpu_pos)
1339 		return;
1340 
1341 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1342 		/* re-try equalization if gen3 is not already enabled */
1343 		if (current_data_rate != 2) {
1344 			u16 bridge_cfg, gpu_cfg;
1345 			u16 bridge_cfg2, gpu_cfg2;
1346 			u32 max_lw, current_lw, tmp;
1347 
1348 			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1349 			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1350 
1351 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1352 			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1353 
1354 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1355 			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1356 
1357 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1358 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1359 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1360 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1361 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1362 
1363 			if (current_lw < max_lw) {
1364 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1365 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1366 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1367 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1368 					tmp |= (max_lw <<
1369 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1370 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1371 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1372 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1373 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1374 				}
1375 			}
1376 
1377 			for (i = 0; i < 10; i++) {
1378 				/* check status */
1379 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1380 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1381 					break;
1382 
1383 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1384 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1385 
1386 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1387 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1388 
1389 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1390 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1391 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1392 
1393 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1394 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1395 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1396 
1397 				mdelay(100);
1398 
1399 				/* linkctl */
1400 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1401 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1402 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1403 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1404 
1405 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1406 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1407 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1408 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1409 
1410 				/* linkctl2 */
1411 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1412 				tmp16 &= ~((1 << 4) | (7 << 9));
1413 				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1414 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1415 
1416 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1417 				tmp16 &= ~((1 << 4) | (7 << 9));
1418 				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1419 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1420 
1421 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1422 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1423 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1424 			}
1425 		}
1426 	}
1427 
1428 	/* set the link speed */
1429 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1430 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1431 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1432 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1433 
1434 	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1435 	tmp16 &= ~0xf;
1436 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1437 		tmp16 |= 3; /* gen3 */
1438 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1439 		tmp16 |= 2; /* gen2 */
1440 	else
1441 		tmp16 |= 1; /* gen1 */
1442 	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1443 
1444 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1445 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1446 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1447 
1448 	for (i = 0; i < adev->usec_timeout; i++) {
1449 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1450 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1451 			break;
1452 		udelay(1);
1453 	}
1454 }
1455 
1456 static void cik_program_aspm(struct amdgpu_device *adev)
1457 {
1458 	u32 data, orig;
1459 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1460 	bool disable_clkreq = false;
1461 
1462 	if (amdgpu_aspm == 0)
1463 		return;
1464 
1465 	if (pci_is_root_bus(adev->pdev->bus))
1466 		return;
1467 
1468 	/* XXX double check APUs */
1469 	if (adev->flags & AMD_IS_APU)
1470 		return;
1471 
1472 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1473 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1474 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1475 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1476 	if (orig != data)
1477 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1478 
1479 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1480 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1481 	if (orig != data)
1482 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1483 
1484 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1485 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1486 	if (orig != data)
1487 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1488 
1489 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1490 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1491 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1492 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1493 	if (!disable_l0s)
1494 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1495 
1496 	if (!disable_l1) {
1497 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1498 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1499 		if (orig != data)
1500 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1501 
1502 		if (!disable_plloff_in_l1) {
1503 			bool clk_req_support;
1504 
1505 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1506 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1507 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1508 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1509 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1510 			if (orig != data)
1511 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1512 
1513 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1514 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1515 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1516 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1517 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1518 			if (orig != data)
1519 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1520 
1521 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1522 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1523 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1524 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1525 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1526 			if (orig != data)
1527 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1528 
1529 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1530 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1531 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1532 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1533 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1534 			if (orig != data)
1535 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1536 
1537 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1538 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1539 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1540 			if (orig != data)
1541 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1542 
1543 			if (!disable_clkreq) {
1544 				struct pci_dev *root = adev->pdev->bus->self;
1545 				u32 lnkcap;
1546 
1547 				clk_req_support = false;
1548 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1549 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1550 					clk_req_support = true;
1551 			} else {
1552 				clk_req_support = false;
1553 			}
1554 
1555 			if (clk_req_support) {
1556 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1557 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1558 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1559 				if (orig != data)
1560 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1561 
1562 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1563 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1564 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1565 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1566 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1567 				if (orig != data)
1568 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1569 
1570 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1571 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1572 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1573 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1574 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1575 				if (orig != data)
1576 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1577 
1578 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1579 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1580 				if (orig != data)
1581 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1582 
1583 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1584 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1585 				if (orig != data)
1586 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1587 
1588 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1589 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1590 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1591 				if (orig != data)
1592 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1593 			}
1594 		}
1595 	} else {
1596 		if (orig != data)
1597 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1598 	}
1599 
1600 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1601 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1602 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1603 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1604 	if (orig != data)
1605 		WREG32_PCIE(ixPCIE_CNTL2, data);
1606 
1607 	if (!disable_l0s) {
1608 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1609 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1610 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1611 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1612 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1613 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1614 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1615 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1616 				if (orig != data)
1617 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1618 			}
1619 		}
1620 	}
1621 }
1622 
1623 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1624 {
1625 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1626 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1627 }
1628 
1629 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1630 {
1631 	/* ORDER MATTERS! */
1632 	{
1633 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1634 		.major = 1,
1635 		.minor = 0,
1636 		.rev = 0,
1637 		.funcs = &cik_common_ip_funcs,
1638 	},
1639 	{
1640 		.type = AMD_IP_BLOCK_TYPE_GMC,
1641 		.major = 7,
1642 		.minor = 0,
1643 		.rev = 0,
1644 		.funcs = &gmc_v7_0_ip_funcs,
1645 	},
1646 	{
1647 		.type = AMD_IP_BLOCK_TYPE_IH,
1648 		.major = 2,
1649 		.minor = 0,
1650 		.rev = 0,
1651 		.funcs = &cik_ih_ip_funcs,
1652 	},
1653 	{
1654 		.type = AMD_IP_BLOCK_TYPE_SMC,
1655 		.major = 7,
1656 		.minor = 0,
1657 		.rev = 0,
1658 		.funcs = &amdgpu_pp_ip_funcs,
1659 	},
1660 	{
1661 		.type = AMD_IP_BLOCK_TYPE_DCE,
1662 		.major = 8,
1663 		.minor = 2,
1664 		.rev = 0,
1665 		.funcs = &dce_v8_0_ip_funcs,
1666 	},
1667 	{
1668 		.type = AMD_IP_BLOCK_TYPE_GFX,
1669 		.major = 7,
1670 		.minor = 2,
1671 		.rev = 0,
1672 		.funcs = &gfx_v7_0_ip_funcs,
1673 	},
1674 	{
1675 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1676 		.major = 2,
1677 		.minor = 0,
1678 		.rev = 0,
1679 		.funcs = &cik_sdma_ip_funcs,
1680 	},
1681 	{
1682 		.type = AMD_IP_BLOCK_TYPE_UVD,
1683 		.major = 4,
1684 		.minor = 2,
1685 		.rev = 0,
1686 		.funcs = &uvd_v4_2_ip_funcs,
1687 	},
1688 	{
1689 		.type = AMD_IP_BLOCK_TYPE_VCE,
1690 		.major = 2,
1691 		.minor = 0,
1692 		.rev = 0,
1693 		.funcs = &vce_v2_0_ip_funcs,
1694 	},
1695 };
1696 
1697 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1698 {
1699 	/* ORDER MATTERS! */
1700 	{
1701 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1702 		.major = 1,
1703 		.minor = 0,
1704 		.rev = 0,
1705 		.funcs = &cik_common_ip_funcs,
1706 	},
1707 	{
1708 		.type = AMD_IP_BLOCK_TYPE_GMC,
1709 		.major = 7,
1710 		.minor = 0,
1711 		.rev = 0,
1712 		.funcs = &gmc_v7_0_ip_funcs,
1713 	},
1714 	{
1715 		.type = AMD_IP_BLOCK_TYPE_IH,
1716 		.major = 2,
1717 		.minor = 0,
1718 		.rev = 0,
1719 		.funcs = &cik_ih_ip_funcs,
1720 	},
1721 	{
1722 		.type = AMD_IP_BLOCK_TYPE_SMC,
1723 		.major = 7,
1724 		.minor = 0,
1725 		.rev = 0,
1726 		.funcs = &amdgpu_pp_ip_funcs,
1727 	},
1728 	{
1729 		.type = AMD_IP_BLOCK_TYPE_DCE,
1730 		.major = 8,
1731 		.minor = 5,
1732 		.rev = 0,
1733 		.funcs = &dce_v8_0_ip_funcs,
1734 	},
1735 	{
1736 		.type = AMD_IP_BLOCK_TYPE_GFX,
1737 		.major = 7,
1738 		.minor = 3,
1739 		.rev = 0,
1740 		.funcs = &gfx_v7_0_ip_funcs,
1741 	},
1742 	{
1743 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1744 		.major = 2,
1745 		.minor = 0,
1746 		.rev = 0,
1747 		.funcs = &cik_sdma_ip_funcs,
1748 	},
1749 	{
1750 		.type = AMD_IP_BLOCK_TYPE_UVD,
1751 		.major = 4,
1752 		.minor = 2,
1753 		.rev = 0,
1754 		.funcs = &uvd_v4_2_ip_funcs,
1755 	},
1756 	{
1757 		.type = AMD_IP_BLOCK_TYPE_VCE,
1758 		.major = 2,
1759 		.minor = 0,
1760 		.rev = 0,
1761 		.funcs = &vce_v2_0_ip_funcs,
1762 	},
1763 };
1764 
1765 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
1766 {
1767 	/* ORDER MATTERS! */
1768 	{
1769 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1770 		.major = 1,
1771 		.minor = 0,
1772 		.rev = 0,
1773 		.funcs = &cik_common_ip_funcs,
1774 	},
1775 	{
1776 		.type = AMD_IP_BLOCK_TYPE_GMC,
1777 		.major = 7,
1778 		.minor = 0,
1779 		.rev = 0,
1780 		.funcs = &gmc_v7_0_ip_funcs,
1781 	},
1782 	{
1783 		.type = AMD_IP_BLOCK_TYPE_IH,
1784 		.major = 2,
1785 		.minor = 0,
1786 		.rev = 0,
1787 		.funcs = &cik_ih_ip_funcs,
1788 	},
1789 	{
1790 		.type = AMD_IP_BLOCK_TYPE_SMC,
1791 		.major = 7,
1792 		.minor = 0,
1793 		.rev = 0,
1794 		.funcs = &amdgpu_pp_ip_funcs,
1795 	},
1796 	{
1797 		.type = AMD_IP_BLOCK_TYPE_DCE,
1798 		.major = 8,
1799 		.minor = 3,
1800 		.rev = 0,
1801 		.funcs = &dce_v8_0_ip_funcs,
1802 	},
1803 	{
1804 		.type = AMD_IP_BLOCK_TYPE_GFX,
1805 		.major = 7,
1806 		.minor = 2,
1807 		.rev = 0,
1808 		.funcs = &gfx_v7_0_ip_funcs,
1809 	},
1810 	{
1811 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1812 		.major = 2,
1813 		.minor = 0,
1814 		.rev = 0,
1815 		.funcs = &cik_sdma_ip_funcs,
1816 	},
1817 	{
1818 		.type = AMD_IP_BLOCK_TYPE_UVD,
1819 		.major = 4,
1820 		.minor = 2,
1821 		.rev = 0,
1822 		.funcs = &uvd_v4_2_ip_funcs,
1823 	},
1824 	{
1825 		.type = AMD_IP_BLOCK_TYPE_VCE,
1826 		.major = 2,
1827 		.minor = 0,
1828 		.rev = 0,
1829 		.funcs = &vce_v2_0_ip_funcs,
1830 	},
1831 };
1832 
1833 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
1834 {
1835 	/* ORDER MATTERS! */
1836 	{
1837 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1838 		.major = 1,
1839 		.minor = 0,
1840 		.rev = 0,
1841 		.funcs = &cik_common_ip_funcs,
1842 	},
1843 	{
1844 		.type = AMD_IP_BLOCK_TYPE_GMC,
1845 		.major = 7,
1846 		.minor = 0,
1847 		.rev = 0,
1848 		.funcs = &gmc_v7_0_ip_funcs,
1849 	},
1850 	{
1851 		.type = AMD_IP_BLOCK_TYPE_IH,
1852 		.major = 2,
1853 		.minor = 0,
1854 		.rev = 0,
1855 		.funcs = &cik_ih_ip_funcs,
1856 	},
1857 	{
1858 		.type = AMD_IP_BLOCK_TYPE_SMC,
1859 		.major = 7,
1860 		.minor = 0,
1861 		.rev = 0,
1862 		.funcs = &amdgpu_pp_ip_funcs,
1863 	},
1864 	{
1865 		.type = AMD_IP_BLOCK_TYPE_DCE,
1866 		.major = 8,
1867 		.minor = 3,
1868 		.rev = 0,
1869 		.funcs = &dce_v8_0_ip_funcs,
1870 	},
1871 	{
1872 		.type = AMD_IP_BLOCK_TYPE_GFX,
1873 		.major = 7,
1874 		.minor = 2,
1875 		.rev = 0,
1876 		.funcs = &gfx_v7_0_ip_funcs,
1877 	},
1878 	{
1879 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1880 		.major = 2,
1881 		.minor = 0,
1882 		.rev = 0,
1883 		.funcs = &cik_sdma_ip_funcs,
1884 	},
1885 	{
1886 		.type = AMD_IP_BLOCK_TYPE_UVD,
1887 		.major = 4,
1888 		.minor = 2,
1889 		.rev = 0,
1890 		.funcs = &uvd_v4_2_ip_funcs,
1891 	},
1892 	{
1893 		.type = AMD_IP_BLOCK_TYPE_VCE,
1894 		.major = 2,
1895 		.minor = 0,
1896 		.rev = 0,
1897 		.funcs = &vce_v2_0_ip_funcs,
1898 	},
1899 };
1900 
1901 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
1902 {
1903 	/* ORDER MATTERS! */
1904 	{
1905 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1906 		.major = 1,
1907 		.minor = 0,
1908 		.rev = 0,
1909 		.funcs = &cik_common_ip_funcs,
1910 	},
1911 	{
1912 		.type = AMD_IP_BLOCK_TYPE_GMC,
1913 		.major = 7,
1914 		.minor = 0,
1915 		.rev = 0,
1916 		.funcs = &gmc_v7_0_ip_funcs,
1917 	},
1918 	{
1919 		.type = AMD_IP_BLOCK_TYPE_IH,
1920 		.major = 2,
1921 		.minor = 0,
1922 		.rev = 0,
1923 		.funcs = &cik_ih_ip_funcs,
1924 	},
1925 	{
1926 		.type = AMD_IP_BLOCK_TYPE_SMC,
1927 		.major = 7,
1928 		.minor = 0,
1929 		.rev = 0,
1930 		.funcs = &amdgpu_pp_ip_funcs,
1931 	},
1932 	{
1933 		.type = AMD_IP_BLOCK_TYPE_DCE,
1934 		.major = 8,
1935 		.minor = 1,
1936 		.rev = 0,
1937 		.funcs = &dce_v8_0_ip_funcs,
1938 	},
1939 	{
1940 		.type = AMD_IP_BLOCK_TYPE_GFX,
1941 		.major = 7,
1942 		.minor = 1,
1943 		.rev = 0,
1944 		.funcs = &gfx_v7_0_ip_funcs,
1945 	},
1946 	{
1947 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1948 		.major = 2,
1949 		.minor = 0,
1950 		.rev = 0,
1951 		.funcs = &cik_sdma_ip_funcs,
1952 	},
1953 	{
1954 		.type = AMD_IP_BLOCK_TYPE_UVD,
1955 		.major = 4,
1956 		.minor = 2,
1957 		.rev = 0,
1958 		.funcs = &uvd_v4_2_ip_funcs,
1959 	},
1960 	{
1961 		.type = AMD_IP_BLOCK_TYPE_VCE,
1962 		.major = 2,
1963 		.minor = 0,
1964 		.rev = 0,
1965 		.funcs = &vce_v2_0_ip_funcs,
1966 	},
1967 };
1968 
1969 int cik_set_ip_blocks(struct amdgpu_device *adev)
1970 {
1971 	switch (adev->asic_type) {
1972 	case CHIP_BONAIRE:
1973 		adev->ip_blocks = bonaire_ip_blocks;
1974 		adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
1975 		break;
1976 	case CHIP_HAWAII:
1977 		adev->ip_blocks = hawaii_ip_blocks;
1978 		adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
1979 		break;
1980 	case CHIP_KAVERI:
1981 		adev->ip_blocks = kaveri_ip_blocks;
1982 		adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
1983 		break;
1984 	case CHIP_KABINI:
1985 		adev->ip_blocks = kabini_ip_blocks;
1986 		adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
1987 		break;
1988 	case CHIP_MULLINS:
1989 		adev->ip_blocks = mullins_ip_blocks;
1990 		adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
1991 		break;
1992 	default:
1993 		/* FIXME: not supported yet */
1994 		return -EINVAL;
1995 	}
1996 
1997 	return 0;
1998 }
1999 
2000 static const struct amdgpu_asic_funcs cik_asic_funcs =
2001 {
2002 	.read_disabled_bios = &cik_read_disabled_bios,
2003 	.read_bios_from_rom = &cik_read_bios_from_rom,
2004 	.read_register = &cik_read_register,
2005 	.reset = &cik_asic_reset,
2006 	.set_vga_state = &cik_vga_set_state,
2007 	.get_xclk = &cik_get_xclk,
2008 	.set_uvd_clocks = &cik_set_uvd_clocks,
2009 	.set_vce_clocks = &cik_set_vce_clocks,
2010 	/* these should be moved to their own ip modules */
2011 	.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2012 	.wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2013 };
2014 
2015 static int cik_common_early_init(void *handle)
2016 {
2017 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2018 
2019 	adev->smc_rreg = &cik_smc_rreg;
2020 	adev->smc_wreg = &cik_smc_wreg;
2021 	adev->pcie_rreg = &cik_pcie_rreg;
2022 	adev->pcie_wreg = &cik_pcie_wreg;
2023 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2024 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2025 	adev->didt_rreg = &cik_didt_rreg;
2026 	adev->didt_wreg = &cik_didt_wreg;
2027 
2028 	adev->asic_funcs = &cik_asic_funcs;
2029 
2030 	adev->rev_id = cik_get_rev_id(adev);
2031 	adev->external_rev_id = 0xFF;
2032 	switch (adev->asic_type) {
2033 	case CHIP_BONAIRE:
2034 		adev->cg_flags =
2035 			AMD_CG_SUPPORT_GFX_MGCG |
2036 			AMD_CG_SUPPORT_GFX_MGLS |
2037 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2038 			AMD_CG_SUPPORT_GFX_CGLS |
2039 			AMD_CG_SUPPORT_GFX_CGTS |
2040 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2041 			AMD_CG_SUPPORT_GFX_CP_LS |
2042 			AMD_CG_SUPPORT_MC_LS |
2043 			AMD_CG_SUPPORT_MC_MGCG |
2044 			AMD_CG_SUPPORT_SDMA_MGCG |
2045 			AMD_CG_SUPPORT_SDMA_LS |
2046 			AMD_CG_SUPPORT_BIF_LS |
2047 			AMD_CG_SUPPORT_VCE_MGCG |
2048 			AMD_CG_SUPPORT_UVD_MGCG |
2049 			AMD_CG_SUPPORT_HDP_LS |
2050 			AMD_CG_SUPPORT_HDP_MGCG;
2051 		adev->pg_flags = 0;
2052 		adev->external_rev_id = adev->rev_id + 0x14;
2053 		break;
2054 	case CHIP_HAWAII:
2055 		adev->cg_flags =
2056 			AMD_CG_SUPPORT_GFX_MGCG |
2057 			AMD_CG_SUPPORT_GFX_MGLS |
2058 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2059 			AMD_CG_SUPPORT_GFX_CGLS |
2060 			AMD_CG_SUPPORT_GFX_CGTS |
2061 			AMD_CG_SUPPORT_GFX_CP_LS |
2062 			AMD_CG_SUPPORT_MC_LS |
2063 			AMD_CG_SUPPORT_MC_MGCG |
2064 			AMD_CG_SUPPORT_SDMA_MGCG |
2065 			AMD_CG_SUPPORT_SDMA_LS |
2066 			AMD_CG_SUPPORT_BIF_LS |
2067 			AMD_CG_SUPPORT_VCE_MGCG |
2068 			AMD_CG_SUPPORT_UVD_MGCG |
2069 			AMD_CG_SUPPORT_HDP_LS |
2070 			AMD_CG_SUPPORT_HDP_MGCG;
2071 		adev->pg_flags = 0;
2072 		adev->external_rev_id = 0x28;
2073 		break;
2074 	case CHIP_KAVERI:
2075 		adev->cg_flags =
2076 			AMD_CG_SUPPORT_GFX_MGCG |
2077 			AMD_CG_SUPPORT_GFX_MGLS |
2078 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2079 			AMD_CG_SUPPORT_GFX_CGLS |
2080 			AMD_CG_SUPPORT_GFX_CGTS |
2081 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2082 			AMD_CG_SUPPORT_GFX_CP_LS |
2083 			AMD_CG_SUPPORT_SDMA_MGCG |
2084 			AMD_CG_SUPPORT_SDMA_LS |
2085 			AMD_CG_SUPPORT_BIF_LS |
2086 			AMD_CG_SUPPORT_VCE_MGCG |
2087 			AMD_CG_SUPPORT_UVD_MGCG |
2088 			AMD_CG_SUPPORT_HDP_LS |
2089 			AMD_CG_SUPPORT_HDP_MGCG;
2090 		adev->pg_flags =
2091 			/*AMD_PG_SUPPORT_GFX_PG |
2092 			  AMD_PG_SUPPORT_GFX_SMG |
2093 			  AMD_PG_SUPPORT_GFX_DMG |*/
2094 			AMD_PG_SUPPORT_UVD |
2095 			/*AMD_PG_SUPPORT_VCE |
2096 			  AMD_PG_SUPPORT_CP |
2097 			  AMD_PG_SUPPORT_GDS |
2098 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2099 			  AMD_PG_SUPPORT_ACP |
2100 			  AMD_PG_SUPPORT_SAMU |*/
2101 			0;
2102 		if (adev->pdev->device == 0x1312 ||
2103 			adev->pdev->device == 0x1316 ||
2104 			adev->pdev->device == 0x1317)
2105 			adev->external_rev_id = 0x41;
2106 		else
2107 			adev->external_rev_id = 0x1;
2108 		break;
2109 	case CHIP_KABINI:
2110 	case CHIP_MULLINS:
2111 		adev->cg_flags =
2112 			AMD_CG_SUPPORT_GFX_MGCG |
2113 			AMD_CG_SUPPORT_GFX_MGLS |
2114 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2115 			AMD_CG_SUPPORT_GFX_CGLS |
2116 			AMD_CG_SUPPORT_GFX_CGTS |
2117 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2118 			AMD_CG_SUPPORT_GFX_CP_LS |
2119 			AMD_CG_SUPPORT_SDMA_MGCG |
2120 			AMD_CG_SUPPORT_SDMA_LS |
2121 			AMD_CG_SUPPORT_BIF_LS |
2122 			AMD_CG_SUPPORT_VCE_MGCG |
2123 			AMD_CG_SUPPORT_UVD_MGCG |
2124 			AMD_CG_SUPPORT_HDP_LS |
2125 			AMD_CG_SUPPORT_HDP_MGCG;
2126 		adev->pg_flags =
2127 			/*AMD_PG_SUPPORT_GFX_PG |
2128 			  AMD_PG_SUPPORT_GFX_SMG | */
2129 			AMD_PG_SUPPORT_UVD |
2130 			/*AMD_PG_SUPPORT_VCE |
2131 			  AMD_PG_SUPPORT_CP |
2132 			  AMD_PG_SUPPORT_GDS |
2133 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2134 			  AMD_PG_SUPPORT_SAMU |*/
2135 			0;
2136 		if (adev->asic_type == CHIP_KABINI) {
2137 			if (adev->rev_id == 0)
2138 				adev->external_rev_id = 0x81;
2139 			else if (adev->rev_id == 1)
2140 				adev->external_rev_id = 0x82;
2141 			else if (adev->rev_id == 2)
2142 				adev->external_rev_id = 0x85;
2143 		} else
2144 			adev->external_rev_id = adev->rev_id + 0xa1;
2145 		break;
2146 	default:
2147 		/* FIXME: not supported yet */
2148 		return -EINVAL;
2149 	}
2150 
2151 	amdgpu_get_pcie_info(adev);
2152 
2153 	return 0;
2154 }
2155 
2156 static int cik_common_sw_init(void *handle)
2157 {
2158 	return 0;
2159 }
2160 
2161 static int cik_common_sw_fini(void *handle)
2162 {
2163 	return 0;
2164 }
2165 
2166 static int cik_common_hw_init(void *handle)
2167 {
2168 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2169 
2170 	/* move the golden regs per IP block */
2171 	cik_init_golden_registers(adev);
2172 	/* enable pcie gen2/3 link */
2173 	cik_pcie_gen3_enable(adev);
2174 	/* enable aspm */
2175 	cik_program_aspm(adev);
2176 
2177 	return 0;
2178 }
2179 
2180 static int cik_common_hw_fini(void *handle)
2181 {
2182 	return 0;
2183 }
2184 
2185 static int cik_common_suspend(void *handle)
2186 {
2187 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2188 
2189 	amdgpu_amdkfd_suspend(adev);
2190 
2191 	return cik_common_hw_fini(adev);
2192 }
2193 
2194 static int cik_common_resume(void *handle)
2195 {
2196 	int r;
2197 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2198 
2199 	r = cik_common_hw_init(adev);
2200 	if (r)
2201 		return r;
2202 
2203 	return amdgpu_amdkfd_resume(adev);
2204 }
2205 
2206 static bool cik_common_is_idle(void *handle)
2207 {
2208 	return true;
2209 }
2210 
2211 static int cik_common_wait_for_idle(void *handle)
2212 {
2213 	return 0;
2214 }
2215 
2216 static int cik_common_soft_reset(void *handle)
2217 {
2218 	/* XXX hard reset?? */
2219 	return 0;
2220 }
2221 
2222 static int cik_common_set_clockgating_state(void *handle,
2223 					    enum amd_clockgating_state state)
2224 {
2225 	return 0;
2226 }
2227 
2228 static int cik_common_set_powergating_state(void *handle,
2229 					    enum amd_powergating_state state)
2230 {
2231 	return 0;
2232 }
2233 
2234 const struct amd_ip_funcs cik_common_ip_funcs = {
2235 	.name = "cik_common",
2236 	.early_init = cik_common_early_init,
2237 	.late_init = NULL,
2238 	.sw_init = cik_common_sw_init,
2239 	.sw_fini = cik_common_sw_fini,
2240 	.hw_init = cik_common_hw_init,
2241 	.hw_fini = cik_common_hw_fini,
2242 	.suspend = cik_common_suspend,
2243 	.resume = cik_common_resume,
2244 	.is_idle = cik_common_is_idle,
2245 	.wait_for_idle = cik_common_wait_for_idle,
2246 	.soft_reset = cik_common_soft_reset,
2247 	.set_clockgating_state = cik_common_set_clockgating_state,
2248 	.set_powergating_state = cik_common_set_powergating_state,
2249 };
2250