xref: /linux/drivers/gpu/drm/amd/amdgpu/si.c (revision 32786fdc9506aeba98278c1844d4bfb766863832) 
1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "amdgpu.h"
29 #include "amdgpu_atombios.h"
30 #include "amdgpu_ih.h"
31 #include "amdgpu_uvd.h"
32 #include "amdgpu_vce.h"
33 #include "atom.h"
34 #include "amdgpu_powerplay.h"
35 #include "si/sid.h"
36 #include "si_ih.h"
37 #include "gfx_v6_0.h"
38 #include "gmc_v6_0.h"
39 #include "si_dma.h"
40 #include "dce_v6_0.h"
41 #include "si.h"
42 #include "dce_virtual.h"
43 
44 static const u32 tahiti_golden_registers[] =
45 {
46 	0x2684, 0x00010000, 0x00018208,
47 	0x260c, 0xffffffff, 0x00000000,
48 	0x260d, 0xf00fffff, 0x00000400,
49 	0x260e, 0x0002021c, 0x00020200,
50 	0x031e, 0x00000080, 0x00000000,
51 	0x340c, 0x000300c0, 0x00800040,
52 	0x360c, 0x000300c0, 0x00800040,
53 	0x16ec, 0x000000f0, 0x00000070,
54 	0x16f0, 0x00200000, 0x50100000,
55 	0x1c0c, 0x31000311, 0x00000011,
56 	0x09df, 0x00000003, 0x000007ff,
57 	0x0903, 0x000007ff, 0x00000000,
58 	0x2285, 0xf000001f, 0x00000007,
59 	0x22c9, 0xffffffff, 0x00ffffff,
60 	0x22c4, 0x0000ff0f, 0x00000000,
61 	0xa293, 0x07ffffff, 0x4e000000,
62 	0xa0d4, 0x3f3f3fff, 0x2a00126a,
63 	0x000c, 0x000000ff, 0x0040,
64 	0x000d, 0x00000040, 0x00004040,
65 	0x2440, 0x07ffffff, 0x03000000,
66 	0x23a2, 0x01ff1f3f, 0x00000000,
67 	0x23a1, 0x01ff1f3f, 0x00000000,
68 	0x2418, 0x0000007f, 0x00000020,
69 	0x2542, 0x00010000, 0x00010000,
70 	0x2b05, 0x00000200, 0x000002fb,
71 	0x2b04, 0xffffffff, 0x0000543b,
72 	0x2b03, 0xffffffff, 0xa9210876,
73 	0x2234, 0xffffffff, 0x000fff40,
74 	0x2235, 0x0000001f, 0x00000010,
75 	0x0504, 0x20000000, 0x20fffed8,
76 	0x0570, 0x000c0fc0, 0x000c0400
77 };
78 
79 static const u32 tahiti_golden_registers2[] =
80 {
81 	0x0319, 0x00000001, 0x00000001
82 };
83 
84 static const u32 tahiti_golden_rlc_registers[] =
85 {
86 	0x3109, 0xffffffff, 0x00601005,
87 	0x311f, 0xffffffff, 0x10104040,
88 	0x3122, 0xffffffff, 0x0100000a,
89 	0x30c5, 0xffffffff, 0x00000800,
90 	0x30c3, 0xffffffff, 0x800000f4,
91 	0x3d2a, 0xffffffff, 0x00000000
92 };
93 
94 static const u32 pitcairn_golden_registers[] =
95 {
96 	0x2684, 0x00010000, 0x00018208,
97 	0x260c, 0xffffffff, 0x00000000,
98 	0x260d, 0xf00fffff, 0x00000400,
99 	0x260e, 0x0002021c, 0x00020200,
100 	0x031e, 0x00000080, 0x00000000,
101 	0x340c, 0x000300c0, 0x00800040,
102 	0x360c, 0x000300c0, 0x00800040,
103 	0x16ec, 0x000000f0, 0x00000070,
104 	0x16f0, 0x00200000, 0x50100000,
105 	0x1c0c, 0x31000311, 0x00000011,
106 	0x0ab9, 0x00073ffe, 0x000022a2,
107 	0x0903, 0x000007ff, 0x00000000,
108 	0x2285, 0xf000001f, 0x00000007,
109 	0x22c9, 0xffffffff, 0x00ffffff,
110 	0x22c4, 0x0000ff0f, 0x00000000,
111 	0xa293, 0x07ffffff, 0x4e000000,
112 	0xa0d4, 0x3f3f3fff, 0x2a00126a,
113 	0x000c, 0x000000ff, 0x0040,
114 	0x000d, 0x00000040, 0x00004040,
115 	0x2440, 0x07ffffff, 0x03000000,
116 	0x2418, 0x0000007f, 0x00000020,
117 	0x2542, 0x00010000, 0x00010000,
118 	0x2b05, 0x000003ff, 0x000000f7,
119 	0x2b04, 0xffffffff, 0x00000000,
120 	0x2b03, 0xffffffff, 0x32761054,
121 	0x2235, 0x0000001f, 0x00000010,
122 	0x0570, 0x000c0fc0, 0x000c0400
123 };
124 
125 static const u32 pitcairn_golden_rlc_registers[] =
126 {
127 	0x3109, 0xffffffff, 0x00601004,
128 	0x311f, 0xffffffff, 0x10102020,
129 	0x3122, 0xffffffff, 0x01000020,
130 	0x30c5, 0xffffffff, 0x00000800,
131 	0x30c3, 0xffffffff, 0x800000a4
132 };
133 
134 static const u32 verde_pg_init[] =
135 {
136 	0xd4f, 0xffffffff, 0x40000,
137 	0xd4e, 0xffffffff, 0x200010ff,
138 	0xd4f, 0xffffffff, 0x0,
139 	0xd4f, 0xffffffff, 0x0,
140 	0xd4f, 0xffffffff, 0x0,
141 	0xd4f, 0xffffffff, 0x0,
142 	0xd4f, 0xffffffff, 0x0,
143 	0xd4f, 0xffffffff, 0x7007,
144 	0xd4e, 0xffffffff, 0x300010ff,
145 	0xd4f, 0xffffffff, 0x0,
146 	0xd4f, 0xffffffff, 0x0,
147 	0xd4f, 0xffffffff, 0x0,
148 	0xd4f, 0xffffffff, 0x0,
149 	0xd4f, 0xffffffff, 0x0,
150 	0xd4f, 0xffffffff, 0x400000,
151 	0xd4e, 0xffffffff, 0x100010ff,
152 	0xd4f, 0xffffffff, 0x0,
153 	0xd4f, 0xffffffff, 0x0,
154 	0xd4f, 0xffffffff, 0x0,
155 	0xd4f, 0xffffffff, 0x0,
156 	0xd4f, 0xffffffff, 0x0,
157 	0xd4f, 0xffffffff, 0x120200,
158 	0xd4e, 0xffffffff, 0x500010ff,
159 	0xd4f, 0xffffffff, 0x0,
160 	0xd4f, 0xffffffff, 0x0,
161 	0xd4f, 0xffffffff, 0x0,
162 	0xd4f, 0xffffffff, 0x0,
163 	0xd4f, 0xffffffff, 0x0,
164 	0xd4f, 0xffffffff, 0x1e1e16,
165 	0xd4e, 0xffffffff, 0x600010ff,
166 	0xd4f, 0xffffffff, 0x0,
167 	0xd4f, 0xffffffff, 0x0,
168 	0xd4f, 0xffffffff, 0x0,
169 	0xd4f, 0xffffffff, 0x0,
170 	0xd4f, 0xffffffff, 0x0,
171 	0xd4f, 0xffffffff, 0x171f1e,
172 	0xd4e, 0xffffffff, 0x700010ff,
173 	0xd4f, 0xffffffff, 0x0,
174 	0xd4f, 0xffffffff, 0x0,
175 	0xd4f, 0xffffffff, 0x0,
176 	0xd4f, 0xffffffff, 0x0,
177 	0xd4f, 0xffffffff, 0x0,
178 	0xd4f, 0xffffffff, 0x0,
179 	0xd4e, 0xffffffff, 0x9ff,
180 	0xd40, 0xffffffff, 0x0,
181 	0xd41, 0xffffffff, 0x10000800,
182 	0xd41, 0xffffffff, 0xf,
183 	0xd41, 0xffffffff, 0xf,
184 	0xd40, 0xffffffff, 0x4,
185 	0xd41, 0xffffffff, 0x1000051e,
186 	0xd41, 0xffffffff, 0xffff,
187 	0xd41, 0xffffffff, 0xffff,
188 	0xd40, 0xffffffff, 0x8,
189 	0xd41, 0xffffffff, 0x80500,
190 	0xd40, 0xffffffff, 0x12,
191 	0xd41, 0xffffffff, 0x9050c,
192 	0xd40, 0xffffffff, 0x1d,
193 	0xd41, 0xffffffff, 0xb052c,
194 	0xd40, 0xffffffff, 0x2a,
195 	0xd41, 0xffffffff, 0x1053e,
196 	0xd40, 0xffffffff, 0x2d,
197 	0xd41, 0xffffffff, 0x10546,
198 	0xd40, 0xffffffff, 0x30,
199 	0xd41, 0xffffffff, 0xa054e,
200 	0xd40, 0xffffffff, 0x3c,
201 	0xd41, 0xffffffff, 0x1055f,
202 	0xd40, 0xffffffff, 0x3f,
203 	0xd41, 0xffffffff, 0x10567,
204 	0xd40, 0xffffffff, 0x42,
205 	0xd41, 0xffffffff, 0x1056f,
206 	0xd40, 0xffffffff, 0x45,
207 	0xd41, 0xffffffff, 0x10572,
208 	0xd40, 0xffffffff, 0x48,
209 	0xd41, 0xffffffff, 0x20575,
210 	0xd40, 0xffffffff, 0x4c,
211 	0xd41, 0xffffffff, 0x190801,
212 	0xd40, 0xffffffff, 0x67,
213 	0xd41, 0xffffffff, 0x1082a,
214 	0xd40, 0xffffffff, 0x6a,
215 	0xd41, 0xffffffff, 0x1b082d,
216 	0xd40, 0xffffffff, 0x87,
217 	0xd41, 0xffffffff, 0x310851,
218 	0xd40, 0xffffffff, 0xba,
219 	0xd41, 0xffffffff, 0x891,
220 	0xd40, 0xffffffff, 0xbc,
221 	0xd41, 0xffffffff, 0x893,
222 	0xd40, 0xffffffff, 0xbe,
223 	0xd41, 0xffffffff, 0x20895,
224 	0xd40, 0xffffffff, 0xc2,
225 	0xd41, 0xffffffff, 0x20899,
226 	0xd40, 0xffffffff, 0xc6,
227 	0xd41, 0xffffffff, 0x2089d,
228 	0xd40, 0xffffffff, 0xca,
229 	0xd41, 0xffffffff, 0x8a1,
230 	0xd40, 0xffffffff, 0xcc,
231 	0xd41, 0xffffffff, 0x8a3,
232 	0xd40, 0xffffffff, 0xce,
233 	0xd41, 0xffffffff, 0x308a5,
234 	0xd40, 0xffffffff, 0xd3,
235 	0xd41, 0xffffffff, 0x6d08cd,
236 	0xd40, 0xffffffff, 0x142,
237 	0xd41, 0xffffffff, 0x2000095a,
238 	0xd41, 0xffffffff, 0x1,
239 	0xd40, 0xffffffff, 0x144,
240 	0xd41, 0xffffffff, 0x301f095b,
241 	0xd40, 0xffffffff, 0x165,
242 	0xd41, 0xffffffff, 0xc094d,
243 	0xd40, 0xffffffff, 0x173,
244 	0xd41, 0xffffffff, 0xf096d,
245 	0xd40, 0xffffffff, 0x184,
246 	0xd41, 0xffffffff, 0x15097f,
247 	0xd40, 0xffffffff, 0x19b,
248 	0xd41, 0xffffffff, 0xc0998,
249 	0xd40, 0xffffffff, 0x1a9,
250 	0xd41, 0xffffffff, 0x409a7,
251 	0xd40, 0xffffffff, 0x1af,
252 	0xd41, 0xffffffff, 0xcdc,
253 	0xd40, 0xffffffff, 0x1b1,
254 	0xd41, 0xffffffff, 0x800,
255 	0xd42, 0xffffffff, 0x6c9b2000,
256 	0xd44, 0xfc00, 0x2000,
257 	0xd51, 0xffffffff, 0xfc0,
258 	0xa35, 0x00000100, 0x100
259 };
260 
261 static const u32 verde_golden_rlc_registers[] =
262 {
263 	0x3109, 0xffffffff, 0x033f1005,
264 	0x311f, 0xffffffff, 0x10808020,
265 	0x3122, 0xffffffff, 0x00800008,
266 	0x30c5, 0xffffffff, 0x00001000,
267 	0x30c3, 0xffffffff, 0x80010014
268 };
269 
270 static const u32 verde_golden_registers[] =
271 {
272 	0x2684, 0x00010000, 0x00018208,
273 	0x260c, 0xffffffff, 0x00000000,
274 	0x260d, 0xf00fffff, 0x00000400,
275 	0x260e, 0x0002021c, 0x00020200,
276 	0x031e, 0x00000080, 0x00000000,
277 	0x340c, 0x000300c0, 0x00800040,
278 	0x340c, 0x000300c0, 0x00800040,
279 	0x360c, 0x000300c0, 0x00800040,
280 	0x360c, 0x000300c0, 0x00800040,
281 	0x16ec, 0x000000f0, 0x00000070,
282 	0x16f0, 0x00200000, 0x50100000,
283 
284 	0x1c0c, 0x31000311, 0x00000011,
285 	0x0ab9, 0x00073ffe, 0x000022a2,
286 	0x0ab9, 0x00073ffe, 0x000022a2,
287 	0x0ab9, 0x00073ffe, 0x000022a2,
288 	0x0903, 0x000007ff, 0x00000000,
289 	0x0903, 0x000007ff, 0x00000000,
290 	0x0903, 0x000007ff, 0x00000000,
291 	0x2285, 0xf000001f, 0x00000007,
292 	0x2285, 0xf000001f, 0x00000007,
293 	0x2285, 0xf000001f, 0x00000007,
294 	0x2285, 0xffffffff, 0x00ffffff,
295 	0x22c4, 0x0000ff0f, 0x00000000,
296 
297 	0xa293, 0x07ffffff, 0x4e000000,
298 	0xa0d4, 0x3f3f3fff, 0x0000124a,
299 	0xa0d4, 0x3f3f3fff, 0x0000124a,
300 	0xa0d4, 0x3f3f3fff, 0x0000124a,
301 	0x000c, 0x000000ff, 0x0040,
302 	0x000d, 0x00000040, 0x00004040,
303 	0x2440, 0x07ffffff, 0x03000000,
304 	0x2440, 0x07ffffff, 0x03000000,
305 	0x23a2, 0x01ff1f3f, 0x00000000,
306 	0x23a3, 0x01ff1f3f, 0x00000000,
307 	0x23a2, 0x01ff1f3f, 0x00000000,
308 	0x23a1, 0x01ff1f3f, 0x00000000,
309 	0x23a1, 0x01ff1f3f, 0x00000000,
310 
311 	0x23a1, 0x01ff1f3f, 0x00000000,
312 	0x2418, 0x0000007f, 0x00000020,
313 	0x2542, 0x00010000, 0x00010000,
314 	0x2b01, 0x000003ff, 0x00000003,
315 	0x2b05, 0x000003ff, 0x00000003,
316 	0x2b05, 0x000003ff, 0x00000003,
317 	0x2b04, 0xffffffff, 0x00000000,
318 	0x2b04, 0xffffffff, 0x00000000,
319 	0x2b04, 0xffffffff, 0x00000000,
320 	0x2b03, 0xffffffff, 0x00001032,
321 	0x2b03, 0xffffffff, 0x00001032,
322 	0x2b03, 0xffffffff, 0x00001032,
323 	0x2235, 0x0000001f, 0x00000010,
324 	0x2235, 0x0000001f, 0x00000010,
325 	0x2235, 0x0000001f, 0x00000010,
326 	0x0570, 0x000c0fc0, 0x000c0400
327 };
328 
329 static const u32 oland_golden_registers[] =
330 {
331 	0x2684, 0x00010000, 0x00018208,
332 	0x260c, 0xffffffff, 0x00000000,
333 	0x260d, 0xf00fffff, 0x00000400,
334 	0x260e, 0x0002021c, 0x00020200,
335 	0x031e, 0x00000080, 0x00000000,
336 	0x340c, 0x000300c0, 0x00800040,
337 	0x360c, 0x000300c0, 0x00800040,
338 	0x16ec, 0x000000f0, 0x00000070,
339 	0x16f9, 0x00200000, 0x50100000,
340 	0x1c0c, 0x31000311, 0x00000011,
341 	0x0ab9, 0x00073ffe, 0x000022a2,
342 	0x0903, 0x000007ff, 0x00000000,
343 	0x2285, 0xf000001f, 0x00000007,
344 	0x22c9, 0xffffffff, 0x00ffffff,
345 	0x22c4, 0x0000ff0f, 0x00000000,
346 	0xa293, 0x07ffffff, 0x4e000000,
347 	0xa0d4, 0x3f3f3fff, 0x00000082,
348 	0x000c, 0x000000ff, 0x0040,
349 	0x000d, 0x00000040, 0x00004040,
350 	0x2440, 0x07ffffff, 0x03000000,
351 	0x2418, 0x0000007f, 0x00000020,
352 	0x2542, 0x00010000, 0x00010000,
353 	0x2b05, 0x000003ff, 0x000000f3,
354 	0x2b04, 0xffffffff, 0x00000000,
355 	0x2b03, 0xffffffff, 0x00003210,
356 	0x2235, 0x0000001f, 0x00000010,
357 	0x0570, 0x000c0fc0, 0x000c0400
358 };
359 
360 static const u32 oland_golden_rlc_registers[] =
361 {
362 	0x3109, 0xffffffff, 0x00601005,
363 	0x311f, 0xffffffff, 0x10104040,
364 	0x3122, 0xffffffff, 0x0100000a,
365 	0x30c5, 0xffffffff, 0x00000800,
366 	0x30c3, 0xffffffff, 0x800000f4
367 };
368 
369 static const u32 hainan_golden_registers[] =
370 {
371 	0x2684, 0x00010000, 0x00018208,
372 	0x260c, 0xffffffff, 0x00000000,
373 	0x260d, 0xf00fffff, 0x00000400,
374 	0x260e, 0x0002021c, 0x00020200,
375 	0x4595, 0xff000fff, 0x00000100,
376 	0x340c, 0x000300c0, 0x00800040,
377 	0x3630, 0xff000fff, 0x00000100,
378 	0x360c, 0x000300c0, 0x00800040,
379 	0x0ab9, 0x00073ffe, 0x000022a2,
380 	0x0903, 0x000007ff, 0x00000000,
381 	0x2285, 0xf000001f, 0x00000007,
382 	0x22c9, 0xffffffff, 0x00ffffff,
383 	0x22c4, 0x0000ff0f, 0x00000000,
384 	0xa393, 0x07ffffff, 0x4e000000,
385 	0xa0d4, 0x3f3f3fff, 0x00000000,
386 	0x000c, 0x000000ff, 0x0040,
387 	0x000d, 0x00000040, 0x00004040,
388 	0x2440, 0x03e00000, 0x03600000,
389 	0x2418, 0x0000007f, 0x00000020,
390 	0x2542, 0x00010000, 0x00010000,
391 	0x2b05, 0x000003ff, 0x000000f1,
392 	0x2b04, 0xffffffff, 0x00000000,
393 	0x2b03, 0xffffffff, 0x00003210,
394 	0x2235, 0x0000001f, 0x00000010,
395 	0x0570, 0x000c0fc0, 0x000c0400
396 };
397 
398 static const u32 hainan_golden_registers2[] =
399 {
400 	0x263e, 0xffffffff, 0x02010001
401 };
402 
403 static const u32 tahiti_mgcg_cgcg_init[] =
404 {
405 	0x3100, 0xffffffff, 0xfffffffc,
406 	0x200b, 0xffffffff, 0xe0000000,
407 	0x2698, 0xffffffff, 0x00000100,
408 	0x24a9, 0xffffffff, 0x00000100,
409 	0x3059, 0xffffffff, 0x00000100,
410 	0x25dd, 0xffffffff, 0x00000100,
411 	0x2261, 0xffffffff, 0x06000100,
412 	0x2286, 0xffffffff, 0x00000100,
413 	0x24a8, 0xffffffff, 0x00000100,
414 	0x30e0, 0xffffffff, 0x00000100,
415 	0x22ca, 0xffffffff, 0x00000100,
416 	0x2451, 0xffffffff, 0x00000100,
417 	0x2362, 0xffffffff, 0x00000100,
418 	0x2363, 0xffffffff, 0x00000100,
419 	0x240c, 0xffffffff, 0x00000100,
420 	0x240d, 0xffffffff, 0x00000100,
421 	0x240e, 0xffffffff, 0x00000100,
422 	0x240f, 0xffffffff, 0x00000100,
423 	0x2b60, 0xffffffff, 0x00000100,
424 	0x2b15, 0xffffffff, 0x00000100,
425 	0x225f, 0xffffffff, 0x06000100,
426 	0x261a, 0xffffffff, 0x00000100,
427 	0x2544, 0xffffffff, 0x00000100,
428 	0x2bc1, 0xffffffff, 0x00000100,
429 	0x2b81, 0xffffffff, 0x00000100,
430 	0x2527, 0xffffffff, 0x00000100,
431 	0x200b, 0xffffffff, 0xe0000000,
432 	0x2458, 0xffffffff, 0x00010000,
433 	0x2459, 0xffffffff, 0x00030002,
434 	0x245a, 0xffffffff, 0x00040007,
435 	0x245b, 0xffffffff, 0x00060005,
436 	0x245c, 0xffffffff, 0x00090008,
437 	0x245d, 0xffffffff, 0x00020001,
438 	0x245e, 0xffffffff, 0x00040003,
439 	0x245f, 0xffffffff, 0x00000007,
440 	0x2460, 0xffffffff, 0x00060005,
441 	0x2461, 0xffffffff, 0x00090008,
442 	0x2462, 0xffffffff, 0x00030002,
443 	0x2463, 0xffffffff, 0x00050004,
444 	0x2464, 0xffffffff, 0x00000008,
445 	0x2465, 0xffffffff, 0x00070006,
446 	0x2466, 0xffffffff, 0x000a0009,
447 	0x2467, 0xffffffff, 0x00040003,
448 	0x2468, 0xffffffff, 0x00060005,
449 	0x2469, 0xffffffff, 0x00000009,
450 	0x246a, 0xffffffff, 0x00080007,
451 	0x246b, 0xffffffff, 0x000b000a,
452 	0x246c, 0xffffffff, 0x00050004,
453 	0x246d, 0xffffffff, 0x00070006,
454 	0x246e, 0xffffffff, 0x0008000b,
455 	0x246f, 0xffffffff, 0x000a0009,
456 	0x2470, 0xffffffff, 0x000d000c,
457 	0x2471, 0xffffffff, 0x00060005,
458 	0x2472, 0xffffffff, 0x00080007,
459 	0x2473, 0xffffffff, 0x0000000b,
460 	0x2474, 0xffffffff, 0x000a0009,
461 	0x2475, 0xffffffff, 0x000d000c,
462 	0x2476, 0xffffffff, 0x00070006,
463 	0x2477, 0xffffffff, 0x00090008,
464 	0x2478, 0xffffffff, 0x0000000c,
465 	0x2479, 0xffffffff, 0x000b000a,
466 	0x247a, 0xffffffff, 0x000e000d,
467 	0x247b, 0xffffffff, 0x00080007,
468 	0x247c, 0xffffffff, 0x000a0009,
469 	0x247d, 0xffffffff, 0x0000000d,
470 	0x247e, 0xffffffff, 0x000c000b,
471 	0x247f, 0xffffffff, 0x000f000e,
472 	0x2480, 0xffffffff, 0x00090008,
473 	0x2481, 0xffffffff, 0x000b000a,
474 	0x2482, 0xffffffff, 0x000c000f,
475 	0x2483, 0xffffffff, 0x000e000d,
476 	0x2484, 0xffffffff, 0x00110010,
477 	0x2485, 0xffffffff, 0x000a0009,
478 	0x2486, 0xffffffff, 0x000c000b,
479 	0x2487, 0xffffffff, 0x0000000f,
480 	0x2488, 0xffffffff, 0x000e000d,
481 	0x2489, 0xffffffff, 0x00110010,
482 	0x248a, 0xffffffff, 0x000b000a,
483 	0x248b, 0xffffffff, 0x000d000c,
484 	0x248c, 0xffffffff, 0x00000010,
485 	0x248d, 0xffffffff, 0x000f000e,
486 	0x248e, 0xffffffff, 0x00120011,
487 	0x248f, 0xffffffff, 0x000c000b,
488 	0x2490, 0xffffffff, 0x000e000d,
489 	0x2491, 0xffffffff, 0x00000011,
490 	0x2492, 0xffffffff, 0x0010000f,
491 	0x2493, 0xffffffff, 0x00130012,
492 	0x2494, 0xffffffff, 0x000d000c,
493 	0x2495, 0xffffffff, 0x000f000e,
494 	0x2496, 0xffffffff, 0x00100013,
495 	0x2497, 0xffffffff, 0x00120011,
496 	0x2498, 0xffffffff, 0x00150014,
497 	0x2499, 0xffffffff, 0x000e000d,
498 	0x249a, 0xffffffff, 0x0010000f,
499 	0x249b, 0xffffffff, 0x00000013,
500 	0x249c, 0xffffffff, 0x00120011,
501 	0x249d, 0xffffffff, 0x00150014,
502 	0x249e, 0xffffffff, 0x000f000e,
503 	0x249f, 0xffffffff, 0x00110010,
504 	0x24a0, 0xffffffff, 0x00000014,
505 	0x24a1, 0xffffffff, 0x00130012,
506 	0x24a2, 0xffffffff, 0x00160015,
507 	0x24a3, 0xffffffff, 0x0010000f,
508 	0x24a4, 0xffffffff, 0x00120011,
509 	0x24a5, 0xffffffff, 0x00000015,
510 	0x24a6, 0xffffffff, 0x00140013,
511 	0x24a7, 0xffffffff, 0x00170016,
512 	0x2454, 0xffffffff, 0x96940200,
513 	0x21c2, 0xffffffff, 0x00900100,
514 	0x311e, 0xffffffff, 0x00000080,
515 	0x3101, 0xffffffff, 0x0020003f,
516 	0xc, 0xffffffff, 0x0000001c,
517 	0xd, 0x000f0000, 0x000f0000,
518 	0x583, 0xffffffff, 0x00000100,
519 	0x409, 0xffffffff, 0x00000100,
520 	0x40b, 0x00000101, 0x00000000,
521 	0x82a, 0xffffffff, 0x00000104,
522 	0x993, 0x000c0000, 0x000c0000,
523 	0x992, 0x000c0000, 0x000c0000,
524 	0x1579, 0xff000fff, 0x00000100,
525 	0x157a, 0x00000001, 0x00000001,
526 	0xbd4, 0x00000001, 0x00000001,
527 	0xc33, 0xc0000fff, 0x00000104,
528 	0x3079, 0x00000001, 0x00000001,
529 	0x3430, 0xfffffff0, 0x00000100,
530 	0x3630, 0xfffffff0, 0x00000100
531 };
532 static const u32 pitcairn_mgcg_cgcg_init[] =
533 {
534 	0x3100, 0xffffffff, 0xfffffffc,
535 	0x200b, 0xffffffff, 0xe0000000,
536 	0x2698, 0xffffffff, 0x00000100,
537 	0x24a9, 0xffffffff, 0x00000100,
538 	0x3059, 0xffffffff, 0x00000100,
539 	0x25dd, 0xffffffff, 0x00000100,
540 	0x2261, 0xffffffff, 0x06000100,
541 	0x2286, 0xffffffff, 0x00000100,
542 	0x24a8, 0xffffffff, 0x00000100,
543 	0x30e0, 0xffffffff, 0x00000100,
544 	0x22ca, 0xffffffff, 0x00000100,
545 	0x2451, 0xffffffff, 0x00000100,
546 	0x2362, 0xffffffff, 0x00000100,
547 	0x2363, 0xffffffff, 0x00000100,
548 	0x240c, 0xffffffff, 0x00000100,
549 	0x240d, 0xffffffff, 0x00000100,
550 	0x240e, 0xffffffff, 0x00000100,
551 	0x240f, 0xffffffff, 0x00000100,
552 	0x2b60, 0xffffffff, 0x00000100,
553 	0x2b15, 0xffffffff, 0x00000100,
554 	0x225f, 0xffffffff, 0x06000100,
555 	0x261a, 0xffffffff, 0x00000100,
556 	0x2544, 0xffffffff, 0x00000100,
557 	0x2bc1, 0xffffffff, 0x00000100,
558 	0x2b81, 0xffffffff, 0x00000100,
559 	0x2527, 0xffffffff, 0x00000100,
560 	0x200b, 0xffffffff, 0xe0000000,
561 	0x2458, 0xffffffff, 0x00010000,
562 	0x2459, 0xffffffff, 0x00030002,
563 	0x245a, 0xffffffff, 0x00040007,
564 	0x245b, 0xffffffff, 0x00060005,
565 	0x245c, 0xffffffff, 0x00090008,
566 	0x245d, 0xffffffff, 0x00020001,
567 	0x245e, 0xffffffff, 0x00040003,
568 	0x245f, 0xffffffff, 0x00000007,
569 	0x2460, 0xffffffff, 0x00060005,
570 	0x2461, 0xffffffff, 0x00090008,
571 	0x2462, 0xffffffff, 0x00030002,
572 	0x2463, 0xffffffff, 0x00050004,
573 	0x2464, 0xffffffff, 0x00000008,
574 	0x2465, 0xffffffff, 0x00070006,
575 	0x2466, 0xffffffff, 0x000a0009,
576 	0x2467, 0xffffffff, 0x00040003,
577 	0x2468, 0xffffffff, 0x00060005,
578 	0x2469, 0xffffffff, 0x00000009,
579 	0x246a, 0xffffffff, 0x00080007,
580 	0x246b, 0xffffffff, 0x000b000a,
581 	0x246c, 0xffffffff, 0x00050004,
582 	0x246d, 0xffffffff, 0x00070006,
583 	0x246e, 0xffffffff, 0x0008000b,
584 	0x246f, 0xffffffff, 0x000a0009,
585 	0x2470, 0xffffffff, 0x000d000c,
586 	0x2480, 0xffffffff, 0x00090008,
587 	0x2481, 0xffffffff, 0x000b000a,
588 	0x2482, 0xffffffff, 0x000c000f,
589 	0x2483, 0xffffffff, 0x000e000d,
590 	0x2484, 0xffffffff, 0x00110010,
591 	0x2485, 0xffffffff, 0x000a0009,
592 	0x2486, 0xffffffff, 0x000c000b,
593 	0x2487, 0xffffffff, 0x0000000f,
594 	0x2488, 0xffffffff, 0x000e000d,
595 	0x2489, 0xffffffff, 0x00110010,
596 	0x248a, 0xffffffff, 0x000b000a,
597 	0x248b, 0xffffffff, 0x000d000c,
598 	0x248c, 0xffffffff, 0x00000010,
599 	0x248d, 0xffffffff, 0x000f000e,
600 	0x248e, 0xffffffff, 0x00120011,
601 	0x248f, 0xffffffff, 0x000c000b,
602 	0x2490, 0xffffffff, 0x000e000d,
603 	0x2491, 0xffffffff, 0x00000011,
604 	0x2492, 0xffffffff, 0x0010000f,
605 	0x2493, 0xffffffff, 0x00130012,
606 	0x2494, 0xffffffff, 0x000d000c,
607 	0x2495, 0xffffffff, 0x000f000e,
608 	0x2496, 0xffffffff, 0x00100013,
609 	0x2497, 0xffffffff, 0x00120011,
610 	0x2498, 0xffffffff, 0x00150014,
611 	0x2454, 0xffffffff, 0x96940200,
612 	0x21c2, 0xffffffff, 0x00900100,
613 	0x311e, 0xffffffff, 0x00000080,
614 	0x3101, 0xffffffff, 0x0020003f,
615 	0xc, 0xffffffff, 0x0000001c,
616 	0xd, 0x000f0000, 0x000f0000,
617 	0x583, 0xffffffff, 0x00000100,
618 	0x409, 0xffffffff, 0x00000100,
619 	0x40b, 0x00000101, 0x00000000,
620 	0x82a, 0xffffffff, 0x00000104,
621 	0x1579, 0xff000fff, 0x00000100,
622 	0x157a, 0x00000001, 0x00000001,
623 	0xbd4, 0x00000001, 0x00000001,
624 	0xc33, 0xc0000fff, 0x00000104,
625 	0x3079, 0x00000001, 0x00000001,
626 	0x3430, 0xfffffff0, 0x00000100,
627 	0x3630, 0xfffffff0, 0x00000100
628 };
629 static const u32 verde_mgcg_cgcg_init[] =
630 {
631 	0x3100, 0xffffffff, 0xfffffffc,
632 	0x200b, 0xffffffff, 0xe0000000,
633 	0x2698, 0xffffffff, 0x00000100,
634 	0x24a9, 0xffffffff, 0x00000100,
635 	0x3059, 0xffffffff, 0x00000100,
636 	0x25dd, 0xffffffff, 0x00000100,
637 	0x2261, 0xffffffff, 0x06000100,
638 	0x2286, 0xffffffff, 0x00000100,
639 	0x24a8, 0xffffffff, 0x00000100,
640 	0x30e0, 0xffffffff, 0x00000100,
641 	0x22ca, 0xffffffff, 0x00000100,
642 	0x2451, 0xffffffff, 0x00000100,
643 	0x2362, 0xffffffff, 0x00000100,
644 	0x2363, 0xffffffff, 0x00000100,
645 	0x240c, 0xffffffff, 0x00000100,
646 	0x240d, 0xffffffff, 0x00000100,
647 	0x240e, 0xffffffff, 0x00000100,
648 	0x240f, 0xffffffff, 0x00000100,
649 	0x2b60, 0xffffffff, 0x00000100,
650 	0x2b15, 0xffffffff, 0x00000100,
651 	0x225f, 0xffffffff, 0x06000100,
652 	0x261a, 0xffffffff, 0x00000100,
653 	0x2544, 0xffffffff, 0x00000100,
654 	0x2bc1, 0xffffffff, 0x00000100,
655 	0x2b81, 0xffffffff, 0x00000100,
656 	0x2527, 0xffffffff, 0x00000100,
657 	0x200b, 0xffffffff, 0xe0000000,
658 	0x2458, 0xffffffff, 0x00010000,
659 	0x2459, 0xffffffff, 0x00030002,
660 	0x245a, 0xffffffff, 0x00040007,
661 	0x245b, 0xffffffff, 0x00060005,
662 	0x245c, 0xffffffff, 0x00090008,
663 	0x245d, 0xffffffff, 0x00020001,
664 	0x245e, 0xffffffff, 0x00040003,
665 	0x245f, 0xffffffff, 0x00000007,
666 	0x2460, 0xffffffff, 0x00060005,
667 	0x2461, 0xffffffff, 0x00090008,
668 	0x2462, 0xffffffff, 0x00030002,
669 	0x2463, 0xffffffff, 0x00050004,
670 	0x2464, 0xffffffff, 0x00000008,
671 	0x2465, 0xffffffff, 0x00070006,
672 	0x2466, 0xffffffff, 0x000a0009,
673 	0x2467, 0xffffffff, 0x00040003,
674 	0x2468, 0xffffffff, 0x00060005,
675 	0x2469, 0xffffffff, 0x00000009,
676 	0x246a, 0xffffffff, 0x00080007,
677 	0x246b, 0xffffffff, 0x000b000a,
678 	0x246c, 0xffffffff, 0x00050004,
679 	0x246d, 0xffffffff, 0x00070006,
680 	0x246e, 0xffffffff, 0x0008000b,
681 	0x246f, 0xffffffff, 0x000a0009,
682 	0x2470, 0xffffffff, 0x000d000c,
683 	0x2480, 0xffffffff, 0x00090008,
684 	0x2481, 0xffffffff, 0x000b000a,
685 	0x2482, 0xffffffff, 0x000c000f,
686 	0x2483, 0xffffffff, 0x000e000d,
687 	0x2484, 0xffffffff, 0x00110010,
688 	0x2485, 0xffffffff, 0x000a0009,
689 	0x2486, 0xffffffff, 0x000c000b,
690 	0x2487, 0xffffffff, 0x0000000f,
691 	0x2488, 0xffffffff, 0x000e000d,
692 	0x2489, 0xffffffff, 0x00110010,
693 	0x248a, 0xffffffff, 0x000b000a,
694 	0x248b, 0xffffffff, 0x000d000c,
695 	0x248c, 0xffffffff, 0x00000010,
696 	0x248d, 0xffffffff, 0x000f000e,
697 	0x248e, 0xffffffff, 0x00120011,
698 	0x248f, 0xffffffff, 0x000c000b,
699 	0x2490, 0xffffffff, 0x000e000d,
700 	0x2491, 0xffffffff, 0x00000011,
701 	0x2492, 0xffffffff, 0x0010000f,
702 	0x2493, 0xffffffff, 0x00130012,
703 	0x2494, 0xffffffff, 0x000d000c,
704 	0x2495, 0xffffffff, 0x000f000e,
705 	0x2496, 0xffffffff, 0x00100013,
706 	0x2497, 0xffffffff, 0x00120011,
707 	0x2498, 0xffffffff, 0x00150014,
708 	0x2454, 0xffffffff, 0x96940200,
709 	0x21c2, 0xffffffff, 0x00900100,
710 	0x311e, 0xffffffff, 0x00000080,
711 	0x3101, 0xffffffff, 0x0020003f,
712 	0xc, 0xffffffff, 0x0000001c,
713 	0xd, 0x000f0000, 0x000f0000,
714 	0x583, 0xffffffff, 0x00000100,
715 	0x409, 0xffffffff, 0x00000100,
716 	0x40b, 0x00000101, 0x00000000,
717 	0x82a, 0xffffffff, 0x00000104,
718 	0x993, 0x000c0000, 0x000c0000,
719 	0x992, 0x000c0000, 0x000c0000,
720 	0x1579, 0xff000fff, 0x00000100,
721 	0x157a, 0x00000001, 0x00000001,
722 	0xbd4, 0x00000001, 0x00000001,
723 	0xc33, 0xc0000fff, 0x00000104,
724 	0x3079, 0x00000001, 0x00000001,
725 	0x3430, 0xfffffff0, 0x00000100,
726 	0x3630, 0xfffffff0, 0x00000100
727 };
728 static const u32 oland_mgcg_cgcg_init[] =
729 {
730 	0x3100, 0xffffffff, 0xfffffffc,
731 	0x200b, 0xffffffff, 0xe0000000,
732 	0x2698, 0xffffffff, 0x00000100,
733 	0x24a9, 0xffffffff, 0x00000100,
734 	0x3059, 0xffffffff, 0x00000100,
735 	0x25dd, 0xffffffff, 0x00000100,
736 	0x2261, 0xffffffff, 0x06000100,
737 	0x2286, 0xffffffff, 0x00000100,
738 	0x24a8, 0xffffffff, 0x00000100,
739 	0x30e0, 0xffffffff, 0x00000100,
740 	0x22ca, 0xffffffff, 0x00000100,
741 	0x2451, 0xffffffff, 0x00000100,
742 	0x2362, 0xffffffff, 0x00000100,
743 	0x2363, 0xffffffff, 0x00000100,
744 	0x240c, 0xffffffff, 0x00000100,
745 	0x240d, 0xffffffff, 0x00000100,
746 	0x240e, 0xffffffff, 0x00000100,
747 	0x240f, 0xffffffff, 0x00000100,
748 	0x2b60, 0xffffffff, 0x00000100,
749 	0x2b15, 0xffffffff, 0x00000100,
750 	0x225f, 0xffffffff, 0x06000100,
751 	0x261a, 0xffffffff, 0x00000100,
752 	0x2544, 0xffffffff, 0x00000100,
753 	0x2bc1, 0xffffffff, 0x00000100,
754 	0x2b81, 0xffffffff, 0x00000100,
755 	0x2527, 0xffffffff, 0x00000100,
756 	0x200b, 0xffffffff, 0xe0000000,
757 	0x2458, 0xffffffff, 0x00010000,
758 	0x2459, 0xffffffff, 0x00030002,
759 	0x245a, 0xffffffff, 0x00040007,
760 	0x245b, 0xffffffff, 0x00060005,
761 	0x245c, 0xffffffff, 0x00090008,
762 	0x245d, 0xffffffff, 0x00020001,
763 	0x245e, 0xffffffff, 0x00040003,
764 	0x245f, 0xffffffff, 0x00000007,
765 	0x2460, 0xffffffff, 0x00060005,
766 	0x2461, 0xffffffff, 0x00090008,
767 	0x2462, 0xffffffff, 0x00030002,
768 	0x2463, 0xffffffff, 0x00050004,
769 	0x2464, 0xffffffff, 0x00000008,
770 	0x2465, 0xffffffff, 0x00070006,
771 	0x2466, 0xffffffff, 0x000a0009,
772 	0x2467, 0xffffffff, 0x00040003,
773 	0x2468, 0xffffffff, 0x00060005,
774 	0x2469, 0xffffffff, 0x00000009,
775 	0x246a, 0xffffffff, 0x00080007,
776 	0x246b, 0xffffffff, 0x000b000a,
777 	0x246c, 0xffffffff, 0x00050004,
778 	0x246d, 0xffffffff, 0x00070006,
779 	0x246e, 0xffffffff, 0x0008000b,
780 	0x246f, 0xffffffff, 0x000a0009,
781 	0x2470, 0xffffffff, 0x000d000c,
782 	0x2471, 0xffffffff, 0x00060005,
783 	0x2472, 0xffffffff, 0x00080007,
784 	0x2473, 0xffffffff, 0x0000000b,
785 	0x2474, 0xffffffff, 0x000a0009,
786 	0x2475, 0xffffffff, 0x000d000c,
787 	0x2454, 0xffffffff, 0x96940200,
788 	0x21c2, 0xffffffff, 0x00900100,
789 	0x311e, 0xffffffff, 0x00000080,
790 	0x3101, 0xffffffff, 0x0020003f,
791 	0xc, 0xffffffff, 0x0000001c,
792 	0xd, 0x000f0000, 0x000f0000,
793 	0x583, 0xffffffff, 0x00000100,
794 	0x409, 0xffffffff, 0x00000100,
795 	0x40b, 0x00000101, 0x00000000,
796 	0x82a, 0xffffffff, 0x00000104,
797 	0x993, 0x000c0000, 0x000c0000,
798 	0x992, 0x000c0000, 0x000c0000,
799 	0x1579, 0xff000fff, 0x00000100,
800 	0x157a, 0x00000001, 0x00000001,
801 	0xbd4, 0x00000001, 0x00000001,
802 	0xc33, 0xc0000fff, 0x00000104,
803 	0x3079, 0x00000001, 0x00000001,
804 	0x3430, 0xfffffff0, 0x00000100,
805 	0x3630, 0xfffffff0, 0x00000100
806 };
807 static const u32 hainan_mgcg_cgcg_init[] =
808 {
809 	0x3100, 0xffffffff, 0xfffffffc,
810 	0x200b, 0xffffffff, 0xe0000000,
811 	0x2698, 0xffffffff, 0x00000100,
812 	0x24a9, 0xffffffff, 0x00000100,
813 	0x3059, 0xffffffff, 0x00000100,
814 	0x25dd, 0xffffffff, 0x00000100,
815 	0x2261, 0xffffffff, 0x06000100,
816 	0x2286, 0xffffffff, 0x00000100,
817 	0x24a8, 0xffffffff, 0x00000100,
818 	0x30e0, 0xffffffff, 0x00000100,
819 	0x22ca, 0xffffffff, 0x00000100,
820 	0x2451, 0xffffffff, 0x00000100,
821 	0x2362, 0xffffffff, 0x00000100,
822 	0x2363, 0xffffffff, 0x00000100,
823 	0x240c, 0xffffffff, 0x00000100,
824 	0x240d, 0xffffffff, 0x00000100,
825 	0x240e, 0xffffffff, 0x00000100,
826 	0x240f, 0xffffffff, 0x00000100,
827 	0x2b60, 0xffffffff, 0x00000100,
828 	0x2b15, 0xffffffff, 0x00000100,
829 	0x225f, 0xffffffff, 0x06000100,
830 	0x261a, 0xffffffff, 0x00000100,
831 	0x2544, 0xffffffff, 0x00000100,
832 	0x2bc1, 0xffffffff, 0x00000100,
833 	0x2b81, 0xffffffff, 0x00000100,
834 	0x2527, 0xffffffff, 0x00000100,
835 	0x200b, 0xffffffff, 0xe0000000,
836 	0x2458, 0xffffffff, 0x00010000,
837 	0x2459, 0xffffffff, 0x00030002,
838 	0x245a, 0xffffffff, 0x00040007,
839 	0x245b, 0xffffffff, 0x00060005,
840 	0x245c, 0xffffffff, 0x00090008,
841 	0x245d, 0xffffffff, 0x00020001,
842 	0x245e, 0xffffffff, 0x00040003,
843 	0x245f, 0xffffffff, 0x00000007,
844 	0x2460, 0xffffffff, 0x00060005,
845 	0x2461, 0xffffffff, 0x00090008,
846 	0x2462, 0xffffffff, 0x00030002,
847 	0x2463, 0xffffffff, 0x00050004,
848 	0x2464, 0xffffffff, 0x00000008,
849 	0x2465, 0xffffffff, 0x00070006,
850 	0x2466, 0xffffffff, 0x000a0009,
851 	0x2467, 0xffffffff, 0x00040003,
852 	0x2468, 0xffffffff, 0x00060005,
853 	0x2469, 0xffffffff, 0x00000009,
854 	0x246a, 0xffffffff, 0x00080007,
855 	0x246b, 0xffffffff, 0x000b000a,
856 	0x246c, 0xffffffff, 0x00050004,
857 	0x246d, 0xffffffff, 0x00070006,
858 	0x246e, 0xffffffff, 0x0008000b,
859 	0x246f, 0xffffffff, 0x000a0009,
860 	0x2470, 0xffffffff, 0x000d000c,
861 	0x2471, 0xffffffff, 0x00060005,
862 	0x2472, 0xffffffff, 0x00080007,
863 	0x2473, 0xffffffff, 0x0000000b,
864 	0x2474, 0xffffffff, 0x000a0009,
865 	0x2475, 0xffffffff, 0x000d000c,
866 	0x2454, 0xffffffff, 0x96940200,
867 	0x21c2, 0xffffffff, 0x00900100,
868 	0x311e, 0xffffffff, 0x00000080,
869 	0x3101, 0xffffffff, 0x0020003f,
870 	0xc, 0xffffffff, 0x0000001c,
871 	0xd, 0x000f0000, 0x000f0000,
872 	0x583, 0xffffffff, 0x00000100,
873 	0x409, 0xffffffff, 0x00000100,
874 	0x82a, 0xffffffff, 0x00000104,
875 	0x993, 0x000c0000, 0x000c0000,
876 	0x992, 0x000c0000, 0x000c0000,
877 	0xbd4, 0x00000001, 0x00000001,
878 	0xc33, 0xc0000fff, 0x00000104,
879 	0x3079, 0x00000001, 0x00000001,
880 	0x3430, 0xfffffff0, 0x00000100,
881 	0x3630, 0xfffffff0, 0x00000100
882 };
883 
884 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
885 {
886 	unsigned long flags;
887 	u32 r;
888 
889 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
890 	WREG32(AMDGPU_PCIE_INDEX, reg);
891 	(void)RREG32(AMDGPU_PCIE_INDEX);
892 	r = RREG32(AMDGPU_PCIE_DATA);
893 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
894 	return r;
895 }
896 
897 static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
898 {
899 	unsigned long flags;
900 
901 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
902 	WREG32(AMDGPU_PCIE_INDEX, reg);
903 	(void)RREG32(AMDGPU_PCIE_INDEX);
904 	WREG32(AMDGPU_PCIE_DATA, v);
905 	(void)RREG32(AMDGPU_PCIE_DATA);
906 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
907 }
908 
909 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
910 {
911 	unsigned long flags;
912 	u32 r;
913 
914 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
915 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
916 	(void)RREG32(PCIE_PORT_INDEX);
917 	r = RREG32(PCIE_PORT_DATA);
918 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
919 	return r;
920 }
921 
922 static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
923 {
924 	unsigned long flags;
925 
926 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
927 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
928 	(void)RREG32(PCIE_PORT_INDEX);
929 	WREG32(PCIE_PORT_DATA, (v));
930 	(void)RREG32(PCIE_PORT_DATA);
931 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
932 }
933 
934 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
935 {
936 	unsigned long flags;
937 	u32 r;
938 
939 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
940 	WREG32(SMC_IND_INDEX_0, (reg));
941 	r = RREG32(SMC_IND_DATA_0);
942 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
943 	return r;
944 }
945 
946 static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
947 {
948 	unsigned long flags;
949 
950 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
951 	WREG32(SMC_IND_INDEX_0, (reg));
952 	WREG32(SMC_IND_DATA_0, (v));
953 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
954 }
955 
956 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
957 	{GRBM_STATUS, false},
958 	{GB_ADDR_CONFIG, false},
959 	{MC_ARB_RAMCFG, false},
960 	{GB_TILE_MODE0, false},
961 	{GB_TILE_MODE1, false},
962 	{GB_TILE_MODE2, false},
963 	{GB_TILE_MODE3, false},
964 	{GB_TILE_MODE4, false},
965 	{GB_TILE_MODE5, false},
966 	{GB_TILE_MODE6, false},
967 	{GB_TILE_MODE7, false},
968 	{GB_TILE_MODE8, false},
969 	{GB_TILE_MODE9, false},
970 	{GB_TILE_MODE10, false},
971 	{GB_TILE_MODE11, false},
972 	{GB_TILE_MODE12, false},
973 	{GB_TILE_MODE13, false},
974 	{GB_TILE_MODE14, false},
975 	{GB_TILE_MODE15, false},
976 	{GB_TILE_MODE16, false},
977 	{GB_TILE_MODE17, false},
978 	{GB_TILE_MODE18, false},
979 	{GB_TILE_MODE19, false},
980 	{GB_TILE_MODE20, false},
981 	{GB_TILE_MODE21, false},
982 	{GB_TILE_MODE22, false},
983 	{GB_TILE_MODE23, false},
984 	{GB_TILE_MODE24, false},
985 	{GB_TILE_MODE25, false},
986 	{GB_TILE_MODE26, false},
987 	{GB_TILE_MODE27, false},
988 	{GB_TILE_MODE28, false},
989 	{GB_TILE_MODE29, false},
990 	{GB_TILE_MODE30, false},
991 	{GB_TILE_MODE31, false},
992 	{CC_RB_BACKEND_DISABLE, false, true},
993 	{GC_USER_RB_BACKEND_DISABLE, false, true},
994 	{PA_SC_RASTER_CONFIG, false, true},
995 };
996 
997 static uint32_t si_read_indexed_register(struct amdgpu_device *adev,
998 					  u32 se_num, u32 sh_num,
999 					  u32 reg_offset)
1000 {
1001 	uint32_t val;
1002 
1003 	mutex_lock(&adev->grbm_idx_mutex);
1004 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1005 		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1006 
1007 	val = RREG32(reg_offset);
1008 
1009 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1010 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1011 	mutex_unlock(&adev->grbm_idx_mutex);
1012 	return val;
1013 }
1014 
1015 static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1016 			     u32 sh_num, u32 reg_offset, u32 *value)
1017 {
1018 	uint32_t i;
1019 
1020 	*value = 0;
1021 	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1022 		if (reg_offset != si_allowed_read_registers[i].reg_offset)
1023 			continue;
1024 
1025 		if (!si_allowed_read_registers[i].untouched)
1026 			*value = si_allowed_read_registers[i].grbm_indexed ?
1027 				 si_read_indexed_register(adev, se_num,
1028 							   sh_num, reg_offset) :
1029 				 RREG32(reg_offset);
1030 		return 0;
1031 	}
1032 	return -EINVAL;
1033 }
1034 
1035 static bool si_read_disabled_bios(struct amdgpu_device *adev)
1036 {
1037 	u32 bus_cntl;
1038 	u32 d1vga_control = 0;
1039 	u32 d2vga_control = 0;
1040 	u32 vga_render_control = 0;
1041 	u32 rom_cntl;
1042 	bool r;
1043 
1044 	bus_cntl = RREG32(R600_BUS_CNTL);
1045 	if (adev->mode_info.num_crtc) {
1046 		d1vga_control = RREG32(AVIVO_D1VGA_CONTROL);
1047 		d2vga_control = RREG32(AVIVO_D2VGA_CONTROL);
1048 		vga_render_control = RREG32(VGA_RENDER_CONTROL);
1049 	}
1050 	rom_cntl = RREG32(R600_ROM_CNTL);
1051 
1052 	/* enable the rom */
1053 	WREG32(R600_BUS_CNTL, (bus_cntl & ~R600_BIOS_ROM_DIS));
1054 	if (adev->mode_info.num_crtc) {
1055 		/* Disable VGA mode */
1056 		WREG32(AVIVO_D1VGA_CONTROL,
1057 		       (d1vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1058 					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1059 		WREG32(AVIVO_D2VGA_CONTROL,
1060 		       (d2vga_control & ~(AVIVO_DVGA_CONTROL_MODE_ENABLE |
1061 					  AVIVO_DVGA_CONTROL_TIMING_SELECT)));
1062 		WREG32(VGA_RENDER_CONTROL,
1063 		       (vga_render_control & C_000300_VGA_VSTATUS_CNTL));
1064 	}
1065 	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1066 
1067 	r = amdgpu_read_bios(adev);
1068 
1069 	/* restore regs */
1070 	WREG32(R600_BUS_CNTL, bus_cntl);
1071 	if (adev->mode_info.num_crtc) {
1072 		WREG32(AVIVO_D1VGA_CONTROL, d1vga_control);
1073 		WREG32(AVIVO_D2VGA_CONTROL, d2vga_control);
1074 		WREG32(VGA_RENDER_CONTROL, vga_render_control);
1075 	}
1076 	WREG32(R600_ROM_CNTL, rom_cntl);
1077 	return r;
1078 }
1079 
1080 //xxx: not implemented
1081 static int si_asic_reset(struct amdgpu_device *adev)
1082 {
1083 	return 0;
1084 }
1085 
1086 static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1087 {
1088 	uint32_t temp;
1089 
1090 	temp = RREG32(CONFIG_CNTL);
1091 	if (state == false) {
1092 		temp &= ~(1<<0);
1093 		temp |= (1<<1);
1094 	} else {
1095 		temp &= ~(1<<1);
1096 	}
1097 	WREG32(CONFIG_CNTL, temp);
1098 }
1099 
1100 static u32 si_get_xclk(struct amdgpu_device *adev)
1101 {
1102         u32 reference_clock = adev->clock.spll.reference_freq;
1103 	u32 tmp;
1104 
1105 	tmp = RREG32(CG_CLKPIN_CNTL_2);
1106 	if (tmp & MUX_TCLK_TO_XCLK)
1107 		return TCLK;
1108 
1109 	tmp = RREG32(CG_CLKPIN_CNTL);
1110 	if (tmp & XTALIN_DIVIDE)
1111 		return reference_clock / 4;
1112 
1113 	return reference_clock;
1114 }
1115 
1116 //xxx:not implemented
1117 static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1118 {
1119 	return 0;
1120 }
1121 
1122 static void si_detect_hw_virtualization(struct amdgpu_device *adev)
1123 {
1124 	if (is_virtual_machine()) /* passthrough mode */
1125 		adev->virtualization.virtual_caps |= AMDGPU_PASSTHROUGH_MODE;
1126 }
1127 
1128 static const struct amdgpu_asic_funcs si_asic_funcs =
1129 {
1130 	.read_disabled_bios = &si_read_disabled_bios,
1131 	.detect_hw_virtualization = si_detect_hw_virtualization,
1132 	.read_register = &si_read_register,
1133 	.reset = &si_asic_reset,
1134 	.set_vga_state = &si_vga_set_state,
1135 	.get_xclk = &si_get_xclk,
1136 	.set_uvd_clocks = &si_set_uvd_clocks,
1137 	.set_vce_clocks = NULL,
1138 };
1139 
1140 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
1141 {
1142 	return (RREG32(CC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1143 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1144 }
1145 
1146 static int si_common_early_init(void *handle)
1147 {
1148 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1149 
1150 	adev->smc_rreg = &si_smc_rreg;
1151 	adev->smc_wreg = &si_smc_wreg;
1152 	adev->pcie_rreg = &si_pcie_rreg;
1153 	adev->pcie_wreg = &si_pcie_wreg;
1154 	adev->pciep_rreg = &si_pciep_rreg;
1155 	adev->pciep_wreg = &si_pciep_wreg;
1156 	adev->uvd_ctx_rreg = NULL;
1157 	adev->uvd_ctx_wreg = NULL;
1158 	adev->didt_rreg = NULL;
1159 	adev->didt_wreg = NULL;
1160 
1161 	adev->asic_funcs = &si_asic_funcs;
1162 
1163 	adev->rev_id = si_get_rev_id(adev);
1164 	adev->external_rev_id = 0xFF;
1165 	switch (adev->asic_type) {
1166 	case CHIP_TAHITI:
1167 		adev->cg_flags =
1168 			AMD_CG_SUPPORT_GFX_MGCG |
1169 			AMD_CG_SUPPORT_GFX_MGLS |
1170 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1171 			AMD_CG_SUPPORT_GFX_CGLS |
1172 			AMD_CG_SUPPORT_GFX_CGTS |
1173 			AMD_CG_SUPPORT_GFX_CP_LS |
1174 			AMD_CG_SUPPORT_MC_MGCG |
1175 			AMD_CG_SUPPORT_SDMA_MGCG |
1176 			AMD_CG_SUPPORT_BIF_LS |
1177 			AMD_CG_SUPPORT_VCE_MGCG |
1178 			AMD_CG_SUPPORT_UVD_MGCG |
1179 			AMD_CG_SUPPORT_HDP_LS |
1180 			AMD_CG_SUPPORT_HDP_MGCG;
1181 			adev->pg_flags = 0;
1182 		break;
1183 	case CHIP_PITCAIRN:
1184 		adev->cg_flags =
1185 			AMD_CG_SUPPORT_GFX_MGCG |
1186 			AMD_CG_SUPPORT_GFX_MGLS |
1187 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1188 			AMD_CG_SUPPORT_GFX_CGLS |
1189 			AMD_CG_SUPPORT_GFX_CGTS |
1190 			AMD_CG_SUPPORT_GFX_CP_LS |
1191 			AMD_CG_SUPPORT_GFX_RLC_LS |
1192 			AMD_CG_SUPPORT_MC_LS |
1193 			AMD_CG_SUPPORT_MC_MGCG |
1194 			AMD_CG_SUPPORT_SDMA_MGCG |
1195 			AMD_CG_SUPPORT_BIF_LS |
1196 			AMD_CG_SUPPORT_VCE_MGCG |
1197 			AMD_CG_SUPPORT_UVD_MGCG |
1198 			AMD_CG_SUPPORT_HDP_LS |
1199 			AMD_CG_SUPPORT_HDP_MGCG;
1200 		adev->pg_flags = 0;
1201 		break;
1202 
1203 	case CHIP_VERDE:
1204 		adev->cg_flags =
1205 			AMD_CG_SUPPORT_GFX_MGCG |
1206 			AMD_CG_SUPPORT_GFX_MGLS |
1207 			AMD_CG_SUPPORT_GFX_CGLS |
1208 			AMD_CG_SUPPORT_GFX_CGTS |
1209 			AMD_CG_SUPPORT_GFX_CGTS_LS |
1210 			AMD_CG_SUPPORT_GFX_CP_LS |
1211 			AMD_CG_SUPPORT_MC_LS |
1212 			AMD_CG_SUPPORT_MC_MGCG |
1213 			AMD_CG_SUPPORT_SDMA_MGCG |
1214 			AMD_CG_SUPPORT_SDMA_LS |
1215 			AMD_CG_SUPPORT_BIF_LS |
1216 			AMD_CG_SUPPORT_VCE_MGCG |
1217 			AMD_CG_SUPPORT_UVD_MGCG |
1218 			AMD_CG_SUPPORT_HDP_LS |
1219 			AMD_CG_SUPPORT_HDP_MGCG;
1220 		adev->pg_flags = 0;
1221 		//???
1222 		adev->external_rev_id = adev->rev_id + 0x14;
1223 		break;
1224 	case CHIP_OLAND:
1225 		adev->cg_flags =
1226 			AMD_CG_SUPPORT_GFX_MGCG |
1227 			AMD_CG_SUPPORT_GFX_MGLS |
1228 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1229 			AMD_CG_SUPPORT_GFX_CGLS |
1230 			AMD_CG_SUPPORT_GFX_CGTS |
1231 			AMD_CG_SUPPORT_GFX_CP_LS |
1232 			AMD_CG_SUPPORT_GFX_RLC_LS |
1233 			AMD_CG_SUPPORT_MC_LS |
1234 			AMD_CG_SUPPORT_MC_MGCG |
1235 			AMD_CG_SUPPORT_SDMA_MGCG |
1236 			AMD_CG_SUPPORT_BIF_LS |
1237 			AMD_CG_SUPPORT_UVD_MGCG |
1238 			AMD_CG_SUPPORT_HDP_LS |
1239 			AMD_CG_SUPPORT_HDP_MGCG;
1240 		adev->pg_flags = 0;
1241 		break;
1242 	case CHIP_HAINAN:
1243 		adev->cg_flags =
1244 			AMD_CG_SUPPORT_GFX_MGCG |
1245 			AMD_CG_SUPPORT_GFX_MGLS |
1246 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
1247 			AMD_CG_SUPPORT_GFX_CGLS |
1248 			AMD_CG_SUPPORT_GFX_CGTS |
1249 			AMD_CG_SUPPORT_GFX_CP_LS |
1250 			AMD_CG_SUPPORT_GFX_RLC_LS |
1251 			AMD_CG_SUPPORT_MC_LS |
1252 			AMD_CG_SUPPORT_MC_MGCG |
1253 			AMD_CG_SUPPORT_SDMA_MGCG |
1254 			AMD_CG_SUPPORT_BIF_LS |
1255 			AMD_CG_SUPPORT_HDP_LS |
1256 			AMD_CG_SUPPORT_HDP_MGCG;
1257 		adev->pg_flags = 0;
1258 		break;
1259 
1260 	default:
1261 		return -EINVAL;
1262 	}
1263 
1264 	return 0;
1265 }
1266 
1267 static int si_common_sw_init(void *handle)
1268 {
1269 	return 0;
1270 }
1271 
1272 static int si_common_sw_fini(void *handle)
1273 {
1274 	return 0;
1275 }
1276 
1277 
1278 static void si_init_golden_registers(struct amdgpu_device *adev)
1279 {
1280 	switch (adev->asic_type) {
1281 	case CHIP_TAHITI:
1282 		amdgpu_program_register_sequence(adev,
1283 						 tahiti_golden_registers,
1284 						 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1285 		amdgpu_program_register_sequence(adev,
1286 						 tahiti_golden_rlc_registers,
1287 						 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1288 		amdgpu_program_register_sequence(adev,
1289 						 tahiti_mgcg_cgcg_init,
1290 						 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1291 		amdgpu_program_register_sequence(adev,
1292 						 tahiti_golden_registers2,
1293 						 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1294 		break;
1295 	case CHIP_PITCAIRN:
1296 		amdgpu_program_register_sequence(adev,
1297 						 pitcairn_golden_registers,
1298 						 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1299 		amdgpu_program_register_sequence(adev,
1300 						 pitcairn_golden_rlc_registers,
1301 						 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1302 		amdgpu_program_register_sequence(adev,
1303 						 pitcairn_mgcg_cgcg_init,
1304 						 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1305 	case CHIP_VERDE:
1306 		amdgpu_program_register_sequence(adev,
1307 						 verde_golden_registers,
1308 						 (const u32)ARRAY_SIZE(verde_golden_registers));
1309 		amdgpu_program_register_sequence(adev,
1310 						 verde_golden_rlc_registers,
1311 						 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1312 		amdgpu_program_register_sequence(adev,
1313 						 verde_mgcg_cgcg_init,
1314 						 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1315 		amdgpu_program_register_sequence(adev,
1316 						 verde_pg_init,
1317 						 (const u32)ARRAY_SIZE(verde_pg_init));
1318 		break;
1319 	case CHIP_OLAND:
1320 		amdgpu_program_register_sequence(adev,
1321 						 oland_golden_registers,
1322 						 (const u32)ARRAY_SIZE(oland_golden_registers));
1323 		amdgpu_program_register_sequence(adev,
1324 						 oland_golden_rlc_registers,
1325 						 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1326 		amdgpu_program_register_sequence(adev,
1327 						 oland_mgcg_cgcg_init,
1328 						 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1329 	case CHIP_HAINAN:
1330 		amdgpu_program_register_sequence(adev,
1331 						 hainan_golden_registers,
1332 						 (const u32)ARRAY_SIZE(hainan_golden_registers));
1333 		amdgpu_program_register_sequence(adev,
1334 						 hainan_golden_registers2,
1335 						 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1336 		amdgpu_program_register_sequence(adev,
1337 						 hainan_mgcg_cgcg_init,
1338 						 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1339 		break;
1340 
1341 
1342 	default:
1343 		BUG();
1344 	}
1345 }
1346 
1347 static void si_pcie_gen3_enable(struct amdgpu_device *adev)
1348 {
1349 	struct pci_dev *root = adev->pdev->bus->self;
1350 	int bridge_pos, gpu_pos;
1351 	u32 speed_cntl, mask, current_data_rate;
1352 	int ret, i;
1353 	u16 tmp16;
1354 
1355 	if (pci_is_root_bus(adev->pdev->bus))
1356 		return;
1357 
1358 	if (amdgpu_pcie_gen2 == 0)
1359 		return;
1360 
1361 	if (adev->flags & AMD_IS_APU)
1362 		return;
1363 
1364 	ret = drm_pcie_get_speed_cap_mask(adev->ddev, &mask);
1365 	if (ret != 0)
1366 		return;
1367 
1368 	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
1369 		return;
1370 
1371 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1372 	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
1373 		LC_CURRENT_DATA_RATE_SHIFT;
1374 	if (mask & DRM_PCIE_SPEED_80) {
1375 		if (current_data_rate == 2) {
1376 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1377 			return;
1378 		}
1379 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1380 	} else if (mask & DRM_PCIE_SPEED_50) {
1381 		if (current_data_rate == 1) {
1382 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1383 			return;
1384 		}
1385 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1386 	}
1387 
1388 	bridge_pos = pci_pcie_cap(root);
1389 	if (!bridge_pos)
1390 		return;
1391 
1392 	gpu_pos = pci_pcie_cap(adev->pdev);
1393 	if (!gpu_pos)
1394 		return;
1395 
1396 	if (mask & DRM_PCIE_SPEED_80) {
1397 		if (current_data_rate != 2) {
1398 			u16 bridge_cfg, gpu_cfg;
1399 			u16 bridge_cfg2, gpu_cfg2;
1400 			u32 max_lw, current_lw, tmp;
1401 
1402 			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1403 			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1404 
1405 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1406 			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1407 
1408 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1409 			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1410 
1411 			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
1412 			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
1413 			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
1414 
1415 			if (current_lw < max_lw) {
1416 				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1417 				if (tmp & LC_RENEGOTIATION_SUPPORT) {
1418 					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
1419 					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
1420 					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
1421 					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
1422 				}
1423 			}
1424 
1425 			for (i = 0; i < 10; i++) {
1426 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1427 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1428 					break;
1429 
1430 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1431 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1432 
1433 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1434 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1435 
1436 				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1437 				tmp |= LC_SET_QUIESCE;
1438 				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1439 
1440 				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1441 				tmp |= LC_REDO_EQ;
1442 				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1443 
1444 				mdelay(100);
1445 
1446 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1447 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1448 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1449 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1450 
1451 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1452 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1453 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1454 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1455 
1456 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1457 				tmp16 &= ~((1 << 4) | (7 << 9));
1458 				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1459 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1460 
1461 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1462 				tmp16 &= ~((1 << 4) | (7 << 9));
1463 				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1464 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1465 
1466 				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
1467 				tmp &= ~LC_SET_QUIESCE;
1468 				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
1469 			}
1470 		}
1471 	}
1472 
1473 	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
1474 	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
1475 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1476 
1477 	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1478 	tmp16 &= ~0xf;
1479 	if (mask & DRM_PCIE_SPEED_80)
1480 		tmp16 |= 3;
1481 	else if (mask & DRM_PCIE_SPEED_50)
1482 		tmp16 |= 2;
1483 	else
1484 		tmp16 |= 1;
1485 	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1486 
1487 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1488 	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
1489 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
1490 
1491 	for (i = 0; i < adev->usec_timeout; i++) {
1492 		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1493 		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
1494 			break;
1495 		udelay(1);
1496 	}
1497 }
1498 
1499 static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
1500 {
1501 	unsigned long flags;
1502 	u32 r;
1503 
1504 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1505 	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1506 	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
1507 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1508 	return r;
1509 }
1510 
1511 static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1512 {
1513 	unsigned long flags;
1514 
1515 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1516 	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
1517 	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
1518 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1519 }
1520 
1521 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
1522 {
1523 	unsigned long flags;
1524 	u32 r;
1525 
1526 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1527 	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1528 	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
1529 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1530 	return r;
1531 }
1532 
1533 static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1534 {
1535 	unsigned long flags;
1536 
1537 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
1538 	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
1539 	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
1540 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
1541 }
1542 static void si_program_aspm(struct amdgpu_device *adev)
1543 {
1544 	u32 data, orig;
1545 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1546 	bool disable_clkreq = false;
1547 
1548 	if (amdgpu_aspm == 0)
1549 		return;
1550 
1551 	if (adev->flags & AMD_IS_APU)
1552 		return;
1553 	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1554 	data &= ~LC_XMIT_N_FTS_MASK;
1555 	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
1556 	if (orig != data)
1557 		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
1558 
1559 	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
1560 	data |= LC_GO_TO_RECOVERY;
1561 	if (orig != data)
1562 		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
1563 
1564 	orig = data = RREG32_PCIE(PCIE_P_CNTL);
1565 	data |= P_IGNORE_EDB_ERR;
1566 	if (orig != data)
1567 		WREG32_PCIE(PCIE_P_CNTL, data);
1568 
1569 	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1570 	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
1571 	data |= LC_PMI_TO_L1_DIS;
1572 	if (!disable_l0s)
1573 		data |= LC_L0S_INACTIVITY(7);
1574 
1575 	if (!disable_l1) {
1576 		data |= LC_L1_INACTIVITY(7);
1577 		data &= ~LC_PMI_TO_L1_DIS;
1578 		if (orig != data)
1579 			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1580 
1581 		if (!disable_plloff_in_l1) {
1582 			bool clk_req_support;
1583 
1584 			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1585 			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1586 			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1587 			if (orig != data)
1588 				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1589 
1590 			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1591 			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1592 			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1593 			if (orig != data)
1594 				si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1595 
1596 			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1597 			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
1598 			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
1599 			if (orig != data)
1600 				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1601 
1602 			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1603 			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
1604 			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
1605 			if (orig != data)
1606 				si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1607 
1608 			if ((adev->family != CHIP_OLAND) && (adev->family != CHIP_HAINAN)) {
1609 				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_0);
1610 				data &= ~PLL_RAMP_UP_TIME_0_MASK;
1611 				if (orig != data)
1612 					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_0, data);
1613 
1614 				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_1);
1615 				data &= ~PLL_RAMP_UP_TIME_1_MASK;
1616 				if (orig != data)
1617 					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_1, data);
1618 
1619 				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_2);
1620 				data &= ~PLL_RAMP_UP_TIME_2_MASK;
1621 				if (orig != data)
1622 					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_2, data);
1623 
1624 				orig = data = si_pif_phy0_rreg(adev,PB0_PIF_PWRDOWN_3);
1625 				data &= ~PLL_RAMP_UP_TIME_3_MASK;
1626 				if (orig != data)
1627 					si_pif_phy0_wreg(adev,PB0_PIF_PWRDOWN_3, data);
1628 
1629 				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_0);
1630 				data &= ~PLL_RAMP_UP_TIME_0_MASK;
1631 				if (orig != data)
1632 					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_0, data);
1633 
1634 				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_1);
1635 				data &= ~PLL_RAMP_UP_TIME_1_MASK;
1636 				if (orig != data)
1637 					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_1, data);
1638 
1639 				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_2);
1640 				data &= ~PLL_RAMP_UP_TIME_2_MASK;
1641 				if (orig != data)
1642 					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_2, data);
1643 
1644 				orig = data = si_pif_phy1_rreg(adev,PB1_PIF_PWRDOWN_3);
1645 				data &= ~PLL_RAMP_UP_TIME_3_MASK;
1646 				if (orig != data)
1647 					si_pif_phy1_wreg(adev,PB1_PIF_PWRDOWN_3, data);
1648 			}
1649 			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
1650 			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
1651 			data |= LC_DYN_LANES_PWR_STATE(3);
1652 			if (orig != data)
1653 				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
1654 
1655 			orig = data = si_pif_phy0_rreg(adev,PB0_PIF_CNTL);
1656 			data &= ~LS2_EXIT_TIME_MASK;
1657 			if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1658 				data |= LS2_EXIT_TIME(5);
1659 			if (orig != data)
1660 				si_pif_phy0_wreg(adev,PB0_PIF_CNTL, data);
1661 
1662 			orig = data = si_pif_phy1_rreg(adev,PB1_PIF_CNTL);
1663 			data &= ~LS2_EXIT_TIME_MASK;
1664 			if ((adev->family == CHIP_OLAND) || (adev->family == CHIP_HAINAN))
1665 				data |= LS2_EXIT_TIME(5);
1666 			if (orig != data)
1667 				si_pif_phy1_wreg(adev,PB1_PIF_CNTL, data);
1668 
1669 			if (!disable_clkreq &&
1670 			    !pci_is_root_bus(adev->pdev->bus)) {
1671 				struct pci_dev *root = adev->pdev->bus->self;
1672 				u32 lnkcap;
1673 
1674 				clk_req_support = false;
1675 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1676 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1677 					clk_req_support = true;
1678 			} else {
1679 				clk_req_support = false;
1680 			}
1681 
1682 			if (clk_req_support) {
1683 				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
1684 				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
1685 				if (orig != data)
1686 					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
1687 
1688 				orig = data = RREG32(THM_CLK_CNTL);
1689 				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
1690 				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
1691 				if (orig != data)
1692 					WREG32(THM_CLK_CNTL, data);
1693 
1694 				orig = data = RREG32(MISC_CLK_CNTL);
1695 				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
1696 				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
1697 				if (orig != data)
1698 					WREG32(MISC_CLK_CNTL, data);
1699 
1700 				orig = data = RREG32(CG_CLKPIN_CNTL);
1701 				data &= ~BCLK_AS_XCLK;
1702 				if (orig != data)
1703 					WREG32(CG_CLKPIN_CNTL, data);
1704 
1705 				orig = data = RREG32(CG_CLKPIN_CNTL_2);
1706 				data &= ~FORCE_BIF_REFCLK_EN;
1707 				if (orig != data)
1708 					WREG32(CG_CLKPIN_CNTL_2, data);
1709 
1710 				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
1711 				data &= ~MPLL_CLKOUT_SEL_MASK;
1712 				data |= MPLL_CLKOUT_SEL(4);
1713 				if (orig != data)
1714 					WREG32(MPLL_BYPASSCLK_SEL, data);
1715 
1716 				orig = data = RREG32(SPLL_CNTL_MODE);
1717 				data &= ~SPLL_REFCLK_SEL_MASK;
1718 				if (orig != data)
1719 					WREG32(SPLL_CNTL_MODE, data);
1720 			}
1721 		}
1722 	} else {
1723 		if (orig != data)
1724 			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1725 	}
1726 
1727 	orig = data = RREG32_PCIE(PCIE_CNTL2);
1728 	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
1729 	if (orig != data)
1730 		WREG32_PCIE(PCIE_CNTL2, data);
1731 
1732 	if (!disable_l0s) {
1733 		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
1734 		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
1735 			data = RREG32_PCIE(PCIE_LC_STATUS1);
1736 			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
1737 				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
1738 				data &= ~LC_L0S_INACTIVITY_MASK;
1739 				if (orig != data)
1740 					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
1741 			}
1742 		}
1743 	}
1744 }
1745 
1746 static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
1747 {
1748 	int readrq;
1749 	u16 v;
1750 
1751 	readrq = pcie_get_readrq(adev->pdev);
1752 	v = ffs(readrq) - 8;
1753 	if ((v == 0) || (v == 6) || (v == 7))
1754 		pcie_set_readrq(adev->pdev, 512);
1755 }
1756 
1757 static int si_common_hw_init(void *handle)
1758 {
1759 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1760 
1761 	si_fix_pci_max_read_req_size(adev);
1762 	si_init_golden_registers(adev);
1763 	si_pcie_gen3_enable(adev);
1764 	si_program_aspm(adev);
1765 
1766 	return 0;
1767 }
1768 
1769 static int si_common_hw_fini(void *handle)
1770 {
1771 	return 0;
1772 }
1773 
1774 static int si_common_suspend(void *handle)
1775 {
1776 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1777 
1778 	return si_common_hw_fini(adev);
1779 }
1780 
1781 static int si_common_resume(void *handle)
1782 {
1783 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1784 
1785 	return si_common_hw_init(adev);
1786 }
1787 
1788 static bool si_common_is_idle(void *handle)
1789 {
1790 	return true;
1791 }
1792 
1793 static int si_common_wait_for_idle(void *handle)
1794 {
1795 	return 0;
1796 }
1797 
1798 static int si_common_soft_reset(void *handle)
1799 {
1800 	return 0;
1801 }
1802 
1803 static int si_common_set_clockgating_state(void *handle,
1804 					    enum amd_clockgating_state state)
1805 {
1806 	return 0;
1807 }
1808 
1809 static int si_common_set_powergating_state(void *handle,
1810 					    enum amd_powergating_state state)
1811 {
1812 	return 0;
1813 }
1814 
1815 static const struct amd_ip_funcs si_common_ip_funcs = {
1816 	.name = "si_common",
1817 	.early_init = si_common_early_init,
1818 	.late_init = NULL,
1819 	.sw_init = si_common_sw_init,
1820 	.sw_fini = si_common_sw_fini,
1821 	.hw_init = si_common_hw_init,
1822 	.hw_fini = si_common_hw_fini,
1823 	.suspend = si_common_suspend,
1824 	.resume = si_common_resume,
1825 	.is_idle = si_common_is_idle,
1826 	.wait_for_idle = si_common_wait_for_idle,
1827 	.soft_reset = si_common_soft_reset,
1828 	.set_clockgating_state = si_common_set_clockgating_state,
1829 	.set_powergating_state = si_common_set_powergating_state,
1830 };
1831 
1832 static const struct amdgpu_ip_block_version si_common_ip_block =
1833 {
1834 	.type = AMD_IP_BLOCK_TYPE_COMMON,
1835 	.major = 1,
1836 	.minor = 0,
1837 	.rev = 0,
1838 	.funcs = &si_common_ip_funcs,
1839 };
1840 
1841 int si_set_ip_blocks(struct amdgpu_device *adev)
1842 {
1843 	switch (adev->asic_type) {
1844 	case CHIP_VERDE:
1845 	case CHIP_TAHITI:
1846 	case CHIP_PITCAIRN:
1847 		amdgpu_ip_block_add(adev, &si_common_ip_block);
1848 		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
1849 		amdgpu_ip_block_add(adev, &si_ih_ip_block);
1850 		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1851 		if (adev->enable_virtual_display)
1852 			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1853 		else
1854 			amdgpu_ip_block_add(adev, &dce_v6_0_ip_block);
1855 		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
1856 		amdgpu_ip_block_add(adev, &si_dma_ip_block);
1857 		/* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
1858 		/* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
1859 		break;
1860 	case CHIP_OLAND:
1861 		amdgpu_ip_block_add(adev, &si_common_ip_block);
1862 		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
1863 		amdgpu_ip_block_add(adev, &si_ih_ip_block);
1864 		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1865 		if (adev->enable_virtual_display)
1866 			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1867 		else
1868 			amdgpu_ip_block_add(adev, &dce_v6_4_ip_block);
1869 		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
1870 		amdgpu_ip_block_add(adev, &si_dma_ip_block);
1871 		/* amdgpu_ip_block_add(adev, &uvd_v3_1_ip_block); */
1872 		/* amdgpu_ip_block_add(adev, &vce_v1_0_ip_block); */
1873 		break;
1874 	case CHIP_HAINAN:
1875 		amdgpu_ip_block_add(adev, &si_common_ip_block);
1876 		amdgpu_ip_block_add(adev, &gmc_v6_0_ip_block);
1877 		amdgpu_ip_block_add(adev, &si_ih_ip_block);
1878 		amdgpu_ip_block_add(adev, &amdgpu_pp_ip_block);
1879 		if (adev->enable_virtual_display)
1880 			amdgpu_ip_block_add(adev, &dce_virtual_ip_block);
1881 		amdgpu_ip_block_add(adev, &gfx_v6_0_ip_block);
1882 		amdgpu_ip_block_add(adev, &si_dma_ip_block);
1883 		break;
1884 	default:
1885 		BUG();
1886 	}
1887 	return 0;
1888 }
1889 
1890