xref: /linux/drivers/gpu/drm/amd/amdgpu/si.c (revision 4dd10fbbeb9c33709edc89916e19c03c780a0751)
1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include <drm/amdgpu_drm.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_atombios.h"
33 #include "amdgpu_ih.h"
34 #include "amdgpu_uvd.h"
35 #include "amdgpu_vce.h"
36 #include "atom.h"
37 #include "amd_pcie.h"
38 #include "amdgpu_video_codecs.h"
39 
40 #include "si_dpm.h"
41 #include "sid.h"
42 #include "si_ih.h"
43 #include "gfx_v6_0.h"
44 #include "gmc_v6_0.h"
45 #include "si_dma.h"
46 #include "dce_v6_0.h"
47 #include "si.h"
48 #include "uvd_v3_1.h"
49 #include "vce_v1_0.h"
50 
51 #include "uvd/uvd_4_0_d.h"
52 
53 #include "smu/smu_6_0_d.h"
54 #include "smu/smu_6_0_sh_mask.h"
55 
56 #include "gca/gfx_6_0_d.h"
57 #include "gca/gfx_6_0_sh_mask.h"
58 
59 #include "oss/oss_1_0_d.h"
60 #include "oss/oss_1_0_sh_mask.h"
61 
62 #include "gmc/gmc_6_0_d.h"
63 #include"gmc/gmc_6_0_sh_mask.h"
64 
65 #include "dce/dce_6_0_d.h"
66 #include "dce/dce_6_0_sh_mask.h"
67 
68 #include "bif/bif_3_0_d.h"
69 #include "bif/bif_3_0_sh_mask.h"
70 #include "si_enums.h"
71 
72 #include "amdgpu_dm.h"
73 #include "amdgpu_vkms.h"
74 
75 static const u32 tahiti_golden_registers[] =
76 {
77 	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
78 	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
79 	mmDB_DEBUG, 0xffffffff, 0x00000000,
80 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
81 	mmDB_DEBUG3, 0x0002021c, 0x00020200,
82 	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
83 	0x340c, 0x000000c0, 0x00800040,
84 	0x360c, 0x000000c0, 0x00800040,
85 	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
86 	mmFBC_MISC, 0x00200000, 0x50100000,
87 	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
88 	mmMC_ARB_WTM_CNTL_RD, 0x00000003, 0x000007ff,
89 	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
90 	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
91 	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
92 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
93 	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
94 	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
95 	0x000c, 0xffffffff, 0x0040,
96 	0x000d, 0x00000040, 0x00004040,
97 	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
98 	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
99 	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
100 	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
101 	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
102 	mmTCP_ADDR_CONFIG, 0x00000200, 0x000002fb,
103 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
104 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
105 	mmVGT_FIFO_DEPTHS, 0xffffffff, 0x000fff40,
106 	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
107 	mmVM_CONTEXT0_CNTL, 0x20000000, 0x20fffed8,
108 	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
109 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
110 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
111 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
112 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
113 };
114 
115 static const u32 tahiti_golden_registers2[] =
116 {
117 	mmMCIF_MEM_CONTROL, 0x00000001, 0x00000001,
118 };
119 
120 static const u32 tahiti_golden_rlc_registers[] =
121 {
122 	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
123 	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
124 	0x311f, 0xffffffff, 0x10104040,
125 	0x3122, 0xffffffff, 0x0100000a,
126 	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
127 	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
128 	mmUVD_CGC_GATE, 0x00000008, 0x00000000,
129 };
130 
131 static const u32 pitcairn_golden_registers[] =
132 {
133 	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
134 	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
135 	mmDB_DEBUG, 0xffffffff, 0x00000000,
136 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
137 	mmDB_DEBUG3, 0x0002021c, 0x00020200,
138 	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
139 	0x340c, 0x000300c0, 0x00800040,
140 	0x360c, 0x000300c0, 0x00800040,
141 	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
142 	mmFBC_MISC, 0x00200000, 0x50100000,
143 	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
144 	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
145 	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
146 	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
147 	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
148 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
149 	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
150 	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x2a00126a,
151 	0x000c, 0xffffffff, 0x0040,
152 	0x000d, 0x00000040, 0x00004040,
153 	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
154 	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
155 	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
156 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
157 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
158 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
159 	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
160 	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
161 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
162 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
163 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
164 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
165 };
166 
167 static const u32 pitcairn_golden_rlc_registers[] =
168 {
169 	mmGB_ADDR_CONFIG, 0xffffffff, 0x12011003,
170 	mmRLC_LB_PARAMS, 0xffffffff, 0x00601004,
171 	0x311f, 0xffffffff, 0x10102020,
172 	0x3122, 0xffffffff, 0x01000020,
173 	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
174 	mmRLC_LB_CNTL, 0xffffffff, 0x800000a4,
175 };
176 
177 static const u32 verde_pg_init[] =
178 {
179 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x40000,
180 	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x200010ff,
181 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
182 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
183 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
184 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
185 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
186 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x7007,
187 	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x300010ff,
188 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
189 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
190 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
191 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
192 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
193 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x400000,
194 	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x100010ff,
195 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
196 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
197 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
198 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
199 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
200 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x120200,
201 	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x500010ff,
202 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
203 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
204 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
205 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
206 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
207 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x1e1e16,
208 	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x600010ff,
209 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
210 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
211 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
212 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
213 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
214 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x171f1e,
215 	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x700010ff,
216 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
217 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
218 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
219 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
220 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
221 	mmGMCON_PGFSM_WRITE, 0xffffffff, 0x0,
222 	mmGMCON_PGFSM_CONFIG, 0xffffffff, 0x9ff,
223 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x0,
224 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10000800,
225 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
226 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf,
227 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4,
228 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1000051e,
229 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
230 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xffff,
231 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x8,
232 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x80500,
233 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x12,
234 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x9050c,
235 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1d,
236 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xb052c,
237 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2a,
238 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1053e,
239 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x2d,
240 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10546,
241 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x30,
242 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xa054e,
243 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3c,
244 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1055f,
245 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x3f,
246 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10567,
247 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x42,
248 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1056f,
249 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x45,
250 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x10572,
251 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x48,
252 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20575,
253 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x4c,
254 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x190801,
255 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x67,
256 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1082a,
257 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x6a,
258 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1b082d,
259 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x87,
260 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x310851,
261 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xba,
262 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x891,
263 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbc,
264 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x893,
265 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xbe,
266 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20895,
267 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc2,
268 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x20899,
269 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xc6,
270 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2089d,
271 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xca,
272 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a1,
273 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xcc,
274 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x8a3,
275 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xce,
276 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x308a5,
277 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0xd3,
278 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x6d08cd,
279 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x142,
280 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x2000095a,
281 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x1,
282 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x144,
283 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x301f095b,
284 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x165,
285 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc094d,
286 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x173,
287 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xf096d,
288 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x184,
289 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x15097f,
290 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x19b,
291 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xc0998,
292 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1a9,
293 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x409a7,
294 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1af,
295 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0xcdc,
296 	mmGMCON_RENG_RAM_INDEX, 0xffffffff, 0x1b1,
297 	mmGMCON_RENG_RAM_DATA, 0xffffffff, 0x800,
298 	mmGMCON_RENG_EXECUTE, 0xffffffff, 0x6c9b2000,
299 	mmGMCON_MISC2, 0xfc00, 0x2000,
300 	mmGMCON_MISC3, 0xffffffff, 0xfc0,
301 	mmMC_PMG_AUTO_CFG, 0x00000100, 0x100,
302 };
303 
304 static const u32 verde_golden_rlc_registers[] =
305 {
306 	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
307 	mmRLC_LB_PARAMS, 0xffffffff, 0x033f1005,
308 	0x311f, 0xffffffff, 0x10808020,
309 	0x3122, 0xffffffff, 0x00800008,
310 	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00001000,
311 	mmRLC_LB_CNTL, 0xffffffff, 0x80010014,
312 };
313 
314 static const u32 verde_golden_registers[] =
315 {
316 	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
317 	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
318 	mmDB_DEBUG, 0xffffffff, 0x00000000,
319 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
320 	mmDB_DEBUG3, 0x0002021c, 0x00020200,
321 	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
322 	0x340c, 0x000300c0, 0x00800040,
323 	0x360c, 0x000300c0, 0x00800040,
324 	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
325 	mmFBC_MISC, 0x00200000, 0x50100000,
326 	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
327 	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
328 	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
329 	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
330 	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
331 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
332 	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
333 	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x0000124a,
334 	0x000c, 0xffffffff, 0x0040,
335 	0x000d, 0x00000040, 0x00004040,
336 	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
337 	mmSQ_DED_CNT, 0x01ff1f3f, 0x00000000,
338 	mmSQ_SEC_CNT, 0x01ff1f3f, 0x00000000,
339 	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
340 	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
341 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x00000003,
342 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
343 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001032,
344 	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
345 	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
346 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
347 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
348 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
349 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
350 };
351 
352 static const u32 oland_golden_registers[] =
353 {
354 	mmAZALIA_SCLK_CONTROL, 0x00000030, 0x00000011,
355 	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
356 	mmDB_DEBUG, 0xffffffff, 0x00000000,
357 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
358 	mmDB_DEBUG3, 0x0002021c, 0x00020200,
359 	mmDCI_CLK_CNTL, 0x00000080, 0x00000000,
360 	0x340c, 0x000300c0, 0x00800040,
361 	0x360c, 0x000300c0, 0x00800040,
362 	mmFBC_DEBUG_COMP, 0x000000f0, 0x00000070,
363 	mmFBC_MISC, 0x00200000, 0x50100000,
364 	mmDIG0_HDMI_CONTROL, 0x31000311, 0x00000011,
365 	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
366 	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
367 	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
368 	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
369 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
370 	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
371 	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000082,
372 	0x000c, 0xffffffff, 0x0040,
373 	0x000d, 0x00000040, 0x00004040,
374 	mmSPI_CONFIG_CNTL, 0x07ffffff, 0x03000000,
375 	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
376 	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
377 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
378 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
379 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
380 	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
381 	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
382 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
383 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
384 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
385 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
386 
387 };
388 
389 static const u32 oland_golden_rlc_registers[] =
390 {
391 	mmGB_ADDR_CONFIG, 0xffffffff, 0x02010002,
392 	mmRLC_LB_PARAMS, 0xffffffff, 0x00601005,
393 	0x311f, 0xffffffff, 0x10104040,
394 	0x3122, 0xffffffff, 0x0100000a,
395 	mmRLC_LB_CNTR_MAX, 0xffffffff, 0x00000800,
396 	mmRLC_LB_CNTL, 0xffffffff, 0x800000f4,
397 };
398 
399 static const u32 hainan_golden_registers[] =
400 {
401 	0x17bc, 0x00000030, 0x00000011,
402 	mmCB_HW_CONTROL, 0x00010000, 0x00018208,
403 	mmDB_DEBUG, 0xffffffff, 0x00000000,
404 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
405 	mmDB_DEBUG3, 0x0002021c, 0x00020200,
406 	0x031e, 0x00000080, 0x00000000,
407 	0x3430, 0xff000fff, 0x00000100,
408 	0x340c, 0x000300c0, 0x00800040,
409 	0x3630, 0xff000fff, 0x00000100,
410 	0x360c, 0x000300c0, 0x00800040,
411 	0x16ec, 0x000000f0, 0x00000070,
412 	0x16f0, 0x00200000, 0x50100000,
413 	0x1c0c, 0x31000311, 0x00000011,
414 	mmMC_SEQ_PMG_PG_HWCNTL, 0x00073ffe, 0x000022a2,
415 	mmMC_XPB_P2P_BAR_CFG, 0x000007ff, 0x00000000,
416 	mmPA_CL_ENHANCE, 0xf000001f, 0x00000007,
417 	mmPA_SC_FORCE_EOV_MAX_CNTS, 0xffffffff, 0x00ffffff,
418 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
419 	mmPA_SC_MODE_CNTL_1, 0x07ffffff, 0x4e000000,
420 	mmPA_SC_RASTER_CONFIG, 0x3f3f3fff, 0x00000000,
421 	0x000c, 0xffffffff, 0x0040,
422 	0x000d, 0x00000040, 0x00004040,
423 	mmSPI_CONFIG_CNTL, 0x03e00000, 0x03600000,
424 	mmSX_DEBUG_1, 0x0000007f, 0x00000020,
425 	mmTA_CNTL_AUX, 0x00010000, 0x00010000,
426 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
427 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
428 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
429 	mmVGT_GS_VERTEX_REUSE, 0x0000001f, 0x00000010,
430 	mmVM_L2_CG, 0x000c0fc0, 0x000c0400,
431 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0xffffffff,
432 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
433 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
434 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
435 };
436 
437 static const u32 hainan_golden_registers2[] =
438 {
439 	mmGB_ADDR_CONFIG, 0xffffffff, 0x2011003,
440 };
441 
442 static const u32 tahiti_mgcg_cgcg_init[] =
443 {
444 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
445 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
446 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
447 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
448 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
449 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
450 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
451 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
452 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
453 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
454 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
455 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
456 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
457 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
458 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
459 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
460 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
461 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
462 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
463 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
464 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
465 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
466 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
467 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
468 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
469 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
470 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
471 	0x2458, 0xffffffff, 0x00010000,
472 	0x2459, 0xffffffff, 0x00030002,
473 	0x245a, 0xffffffff, 0x00040007,
474 	0x245b, 0xffffffff, 0x00060005,
475 	0x245c, 0xffffffff, 0x00090008,
476 	0x245d, 0xffffffff, 0x00020001,
477 	0x245e, 0xffffffff, 0x00040003,
478 	0x245f, 0xffffffff, 0x00000007,
479 	0x2460, 0xffffffff, 0x00060005,
480 	0x2461, 0xffffffff, 0x00090008,
481 	0x2462, 0xffffffff, 0x00030002,
482 	0x2463, 0xffffffff, 0x00050004,
483 	0x2464, 0xffffffff, 0x00000008,
484 	0x2465, 0xffffffff, 0x00070006,
485 	0x2466, 0xffffffff, 0x000a0009,
486 	0x2467, 0xffffffff, 0x00040003,
487 	0x2468, 0xffffffff, 0x00060005,
488 	0x2469, 0xffffffff, 0x00000009,
489 	0x246a, 0xffffffff, 0x00080007,
490 	0x246b, 0xffffffff, 0x000b000a,
491 	0x246c, 0xffffffff, 0x00050004,
492 	0x246d, 0xffffffff, 0x00070006,
493 	0x246e, 0xffffffff, 0x0008000b,
494 	0x246f, 0xffffffff, 0x000a0009,
495 	0x2470, 0xffffffff, 0x000d000c,
496 	0x2471, 0xffffffff, 0x00060005,
497 	0x2472, 0xffffffff, 0x00080007,
498 	0x2473, 0xffffffff, 0x0000000b,
499 	0x2474, 0xffffffff, 0x000a0009,
500 	0x2475, 0xffffffff, 0x000d000c,
501 	0x2476, 0xffffffff, 0x00070006,
502 	0x2477, 0xffffffff, 0x00090008,
503 	0x2478, 0xffffffff, 0x0000000c,
504 	0x2479, 0xffffffff, 0x000b000a,
505 	0x247a, 0xffffffff, 0x000e000d,
506 	0x247b, 0xffffffff, 0x00080007,
507 	0x247c, 0xffffffff, 0x000a0009,
508 	0x247d, 0xffffffff, 0x0000000d,
509 	0x247e, 0xffffffff, 0x000c000b,
510 	0x247f, 0xffffffff, 0x000f000e,
511 	0x2480, 0xffffffff, 0x00090008,
512 	0x2481, 0xffffffff, 0x000b000a,
513 	0x2482, 0xffffffff, 0x000c000f,
514 	0x2483, 0xffffffff, 0x000e000d,
515 	0x2484, 0xffffffff, 0x00110010,
516 	0x2485, 0xffffffff, 0x000a0009,
517 	0x2486, 0xffffffff, 0x000c000b,
518 	0x2487, 0xffffffff, 0x0000000f,
519 	0x2488, 0xffffffff, 0x000e000d,
520 	0x2489, 0xffffffff, 0x00110010,
521 	0x248a, 0xffffffff, 0x000b000a,
522 	0x248b, 0xffffffff, 0x000d000c,
523 	0x248c, 0xffffffff, 0x00000010,
524 	0x248d, 0xffffffff, 0x000f000e,
525 	0x248e, 0xffffffff, 0x00120011,
526 	0x248f, 0xffffffff, 0x000c000b,
527 	0x2490, 0xffffffff, 0x000e000d,
528 	0x2491, 0xffffffff, 0x00000011,
529 	0x2492, 0xffffffff, 0x0010000f,
530 	0x2493, 0xffffffff, 0x00130012,
531 	0x2494, 0xffffffff, 0x000d000c,
532 	0x2495, 0xffffffff, 0x000f000e,
533 	0x2496, 0xffffffff, 0x00100013,
534 	0x2497, 0xffffffff, 0x00120011,
535 	0x2498, 0xffffffff, 0x00150014,
536 	0x2499, 0xffffffff, 0x000e000d,
537 	0x249a, 0xffffffff, 0x0010000f,
538 	0x249b, 0xffffffff, 0x00000013,
539 	0x249c, 0xffffffff, 0x00120011,
540 	0x249d, 0xffffffff, 0x00150014,
541 	0x249e, 0xffffffff, 0x000f000e,
542 	0x249f, 0xffffffff, 0x00110010,
543 	0x24a0, 0xffffffff, 0x00000014,
544 	0x24a1, 0xffffffff, 0x00130012,
545 	0x24a2, 0xffffffff, 0x00160015,
546 	0x24a3, 0xffffffff, 0x0010000f,
547 	0x24a4, 0xffffffff, 0x00120011,
548 	0x24a5, 0xffffffff, 0x00000015,
549 	0x24a6, 0xffffffff, 0x00140013,
550 	0x24a7, 0xffffffff, 0x00170016,
551 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
552 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
553 	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
554 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
555 	0x000c, 0xffffffff, 0x0000001c,
556 	0x000d, 0x000f0000, 0x000f0000,
557 	0x0583, 0xffffffff, 0x00000100,
558 	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
559 	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
560 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
561 	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
562 	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
563 	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
564 	0x157a, 0x00000001, 0x00000001,
565 	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
566 	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
567 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
568 	0x3430, 0xfffffff0, 0x00000100,
569 	0x3630, 0xfffffff0, 0x00000100,
570 };
571 static const u32 pitcairn_mgcg_cgcg_init[] =
572 {
573 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
574 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
575 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
576 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
577 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
578 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
579 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
580 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
581 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
582 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
583 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
584 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
585 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
586 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
587 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
588 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
589 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
590 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
591 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
592 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
593 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
594 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
595 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
596 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
597 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
598 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
599 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
600 	0x2458, 0xffffffff, 0x00010000,
601 	0x2459, 0xffffffff, 0x00030002,
602 	0x245a, 0xffffffff, 0x00040007,
603 	0x245b, 0xffffffff, 0x00060005,
604 	0x245c, 0xffffffff, 0x00090008,
605 	0x245d, 0xffffffff, 0x00020001,
606 	0x245e, 0xffffffff, 0x00040003,
607 	0x245f, 0xffffffff, 0x00000007,
608 	0x2460, 0xffffffff, 0x00060005,
609 	0x2461, 0xffffffff, 0x00090008,
610 	0x2462, 0xffffffff, 0x00030002,
611 	0x2463, 0xffffffff, 0x00050004,
612 	0x2464, 0xffffffff, 0x00000008,
613 	0x2465, 0xffffffff, 0x00070006,
614 	0x2466, 0xffffffff, 0x000a0009,
615 	0x2467, 0xffffffff, 0x00040003,
616 	0x2468, 0xffffffff, 0x00060005,
617 	0x2469, 0xffffffff, 0x00000009,
618 	0x246a, 0xffffffff, 0x00080007,
619 	0x246b, 0xffffffff, 0x000b000a,
620 	0x246c, 0xffffffff, 0x00050004,
621 	0x246d, 0xffffffff, 0x00070006,
622 	0x246e, 0xffffffff, 0x0008000b,
623 	0x246f, 0xffffffff, 0x000a0009,
624 	0x2470, 0xffffffff, 0x000d000c,
625 	0x2480, 0xffffffff, 0x00090008,
626 	0x2481, 0xffffffff, 0x000b000a,
627 	0x2482, 0xffffffff, 0x000c000f,
628 	0x2483, 0xffffffff, 0x000e000d,
629 	0x2484, 0xffffffff, 0x00110010,
630 	0x2485, 0xffffffff, 0x000a0009,
631 	0x2486, 0xffffffff, 0x000c000b,
632 	0x2487, 0xffffffff, 0x0000000f,
633 	0x2488, 0xffffffff, 0x000e000d,
634 	0x2489, 0xffffffff, 0x00110010,
635 	0x248a, 0xffffffff, 0x000b000a,
636 	0x248b, 0xffffffff, 0x000d000c,
637 	0x248c, 0xffffffff, 0x00000010,
638 	0x248d, 0xffffffff, 0x000f000e,
639 	0x248e, 0xffffffff, 0x00120011,
640 	0x248f, 0xffffffff, 0x000c000b,
641 	0x2490, 0xffffffff, 0x000e000d,
642 	0x2491, 0xffffffff, 0x00000011,
643 	0x2492, 0xffffffff, 0x0010000f,
644 	0x2493, 0xffffffff, 0x00130012,
645 	0x2494, 0xffffffff, 0x000d000c,
646 	0x2495, 0xffffffff, 0x000f000e,
647 	0x2496, 0xffffffff, 0x00100013,
648 	0x2497, 0xffffffff, 0x00120011,
649 	0x2498, 0xffffffff, 0x00150014,
650 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
651 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
652 	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
653 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
654 	0x000c, 0xffffffff, 0x0000001c,
655 	0x000d, 0x000f0000, 0x000f0000,
656 	0x0583, 0xffffffff, 0x00000100,
657 	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
658 	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
659 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
660 	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
661 	0x157a, 0x00000001, 0x00000001,
662 	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
663 	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
664 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
665 	0x3430, 0xfffffff0, 0x00000100,
666 	0x3630, 0xfffffff0, 0x00000100,
667 };
668 
669 static const u32 verde_mgcg_cgcg_init[] =
670 {
671 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
672 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
673 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
674 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
675 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
676 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
677 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
678 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
679 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
680 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
681 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
682 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
683 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
684 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
685 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
686 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
687 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
688 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
689 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
690 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
691 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
692 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
693 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
694 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
695 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
696 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
697 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
698 	0x2458, 0xffffffff, 0x00010000,
699 	0x2459, 0xffffffff, 0x00030002,
700 	0x245a, 0xffffffff, 0x00040007,
701 	0x245b, 0xffffffff, 0x00060005,
702 	0x245c, 0xffffffff, 0x00090008,
703 	0x245d, 0xffffffff, 0x00020001,
704 	0x245e, 0xffffffff, 0x00040003,
705 	0x245f, 0xffffffff, 0x00000007,
706 	0x2460, 0xffffffff, 0x00060005,
707 	0x2461, 0xffffffff, 0x00090008,
708 	0x2462, 0xffffffff, 0x00030002,
709 	0x2463, 0xffffffff, 0x00050004,
710 	0x2464, 0xffffffff, 0x00000008,
711 	0x2465, 0xffffffff, 0x00070006,
712 	0x2466, 0xffffffff, 0x000a0009,
713 	0x2467, 0xffffffff, 0x00040003,
714 	0x2468, 0xffffffff, 0x00060005,
715 	0x2469, 0xffffffff, 0x00000009,
716 	0x246a, 0xffffffff, 0x00080007,
717 	0x246b, 0xffffffff, 0x000b000a,
718 	0x246c, 0xffffffff, 0x00050004,
719 	0x246d, 0xffffffff, 0x00070006,
720 	0x246e, 0xffffffff, 0x0008000b,
721 	0x246f, 0xffffffff, 0x000a0009,
722 	0x2470, 0xffffffff, 0x000d000c,
723 	0x2480, 0xffffffff, 0x00090008,
724 	0x2481, 0xffffffff, 0x000b000a,
725 	0x2482, 0xffffffff, 0x000c000f,
726 	0x2483, 0xffffffff, 0x000e000d,
727 	0x2484, 0xffffffff, 0x00110010,
728 	0x2485, 0xffffffff, 0x000a0009,
729 	0x2486, 0xffffffff, 0x000c000b,
730 	0x2487, 0xffffffff, 0x0000000f,
731 	0x2488, 0xffffffff, 0x000e000d,
732 	0x2489, 0xffffffff, 0x00110010,
733 	0x248a, 0xffffffff, 0x000b000a,
734 	0x248b, 0xffffffff, 0x000d000c,
735 	0x248c, 0xffffffff, 0x00000010,
736 	0x248d, 0xffffffff, 0x000f000e,
737 	0x248e, 0xffffffff, 0x00120011,
738 	0x248f, 0xffffffff, 0x000c000b,
739 	0x2490, 0xffffffff, 0x000e000d,
740 	0x2491, 0xffffffff, 0x00000011,
741 	0x2492, 0xffffffff, 0x0010000f,
742 	0x2493, 0xffffffff, 0x00130012,
743 	0x2494, 0xffffffff, 0x000d000c,
744 	0x2495, 0xffffffff, 0x000f000e,
745 	0x2496, 0xffffffff, 0x00100013,
746 	0x2497, 0xffffffff, 0x00120011,
747 	0x2498, 0xffffffff, 0x00150014,
748 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
749 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
750 	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
751 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
752 	0x000c, 0xffffffff, 0x0000001c,
753 	0x000d, 0x000f0000, 0x000f0000,
754 	0x0583, 0xffffffff, 0x00000100,
755 	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
756 	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
757 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
758 	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
759 	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
760 	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
761 	0x157a, 0x00000001, 0x00000001,
762 	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
763 	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
764 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
765 	0x3430, 0xfffffff0, 0x00000100,
766 	0x3630, 0xfffffff0, 0x00000100,
767 };
768 
769 static const u32 oland_mgcg_cgcg_init[] =
770 {
771 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
772 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
773 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
774 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
775 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
776 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
777 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
778 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
779 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
780 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
781 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
782 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
783 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
784 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
785 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
786 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
787 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
788 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
789 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
790 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
791 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
792 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
793 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
794 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
795 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
796 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
797 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
798 	0x2458, 0xffffffff, 0x00010000,
799 	0x2459, 0xffffffff, 0x00030002,
800 	0x245a, 0xffffffff, 0x00040007,
801 	0x245b, 0xffffffff, 0x00060005,
802 	0x245c, 0xffffffff, 0x00090008,
803 	0x245d, 0xffffffff, 0x00020001,
804 	0x245e, 0xffffffff, 0x00040003,
805 	0x245f, 0xffffffff, 0x00000007,
806 	0x2460, 0xffffffff, 0x00060005,
807 	0x2461, 0xffffffff, 0x00090008,
808 	0x2462, 0xffffffff, 0x00030002,
809 	0x2463, 0xffffffff, 0x00050004,
810 	0x2464, 0xffffffff, 0x00000008,
811 	0x2465, 0xffffffff, 0x00070006,
812 	0x2466, 0xffffffff, 0x000a0009,
813 	0x2467, 0xffffffff, 0x00040003,
814 	0x2468, 0xffffffff, 0x00060005,
815 	0x2469, 0xffffffff, 0x00000009,
816 	0x246a, 0xffffffff, 0x00080007,
817 	0x246b, 0xffffffff, 0x000b000a,
818 	0x246c, 0xffffffff, 0x00050004,
819 	0x246d, 0xffffffff, 0x00070006,
820 	0x246e, 0xffffffff, 0x0008000b,
821 	0x246f, 0xffffffff, 0x000a0009,
822 	0x2470, 0xffffffff, 0x000d000c,
823 	0x2471, 0xffffffff, 0x00060005,
824 	0x2472, 0xffffffff, 0x00080007,
825 	0x2473, 0xffffffff, 0x0000000b,
826 	0x2474, 0xffffffff, 0x000a0009,
827 	0x2475, 0xffffffff, 0x000d000c,
828 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
829 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
830 	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
831 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
832 	0x000c, 0xffffffff, 0x0000001c,
833 	0x000d, 0x000f0000, 0x000f0000,
834 	0x0583, 0xffffffff, 0x00000100,
835 	mmXDMA_CLOCK_GATING_CNTL, 0xffffffff, 0x00000100,
836 	mmXDMA_MEM_POWER_CNTL, 0x00000101, 0x00000000,
837 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
838 	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
839 	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
840 	mmCGTT_DRM_CLK_CTRL0, 0xff000fff, 0x00000100,
841 	0x157a, 0x00000001, 0x00000001,
842 	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
843 	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
844 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
845 	0x3430, 0xfffffff0, 0x00000100,
846 	0x3630, 0xfffffff0, 0x00000100,
847 };
848 
849 static const u32 hainan_mgcg_cgcg_init[] =
850 {
851 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xfffffffc,
852 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
853 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
854 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
855 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
856 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
857 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
858 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
859 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
860 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
861 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
862 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
863 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
864 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
865 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
866 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
867 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
868 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
869 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
870 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
871 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
872 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
873 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
874 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
875 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
876 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
877 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
878 	0x2458, 0xffffffff, 0x00010000,
879 	0x2459, 0xffffffff, 0x00030002,
880 	0x245a, 0xffffffff, 0x00040007,
881 	0x245b, 0xffffffff, 0x00060005,
882 	0x245c, 0xffffffff, 0x00090008,
883 	0x245d, 0xffffffff, 0x00020001,
884 	0x245e, 0xffffffff, 0x00040003,
885 	0x245f, 0xffffffff, 0x00000007,
886 	0x2460, 0xffffffff, 0x00060005,
887 	0x2461, 0xffffffff, 0x00090008,
888 	0x2462, 0xffffffff, 0x00030002,
889 	0x2463, 0xffffffff, 0x00050004,
890 	0x2464, 0xffffffff, 0x00000008,
891 	0x2465, 0xffffffff, 0x00070006,
892 	0x2466, 0xffffffff, 0x000a0009,
893 	0x2467, 0xffffffff, 0x00040003,
894 	0x2468, 0xffffffff, 0x00060005,
895 	0x2469, 0xffffffff, 0x00000009,
896 	0x246a, 0xffffffff, 0x00080007,
897 	0x246b, 0xffffffff, 0x000b000a,
898 	0x246c, 0xffffffff, 0x00050004,
899 	0x246d, 0xffffffff, 0x00070006,
900 	0x246e, 0xffffffff, 0x0008000b,
901 	0x246f, 0xffffffff, 0x000a0009,
902 	0x2470, 0xffffffff, 0x000d000c,
903 	0x2471, 0xffffffff, 0x00060005,
904 	0x2472, 0xffffffff, 0x00080007,
905 	0x2473, 0xffffffff, 0x0000000b,
906 	0x2474, 0xffffffff, 0x000a0009,
907 	0x2475, 0xffffffff, 0x000d000c,
908 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
909 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
910 	mmRLC_GCPM_GENERAL_3, 0xffffffff, 0x00000080,
911 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
912 	0x000c, 0xffffffff, 0x0000001c,
913 	0x000d, 0x000f0000, 0x000f0000,
914 	0x0583, 0xffffffff, 0x00000100,
915 	0x0409, 0xffffffff, 0x00000100,
916 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104,
917 	mmMC_CITF_MISC_WR_CG, 0x000c0000, 0x000c0000,
918 	mmMC_CITF_MISC_RD_CG, 0x000c0000, 0x000c0000,
919 	mmHDP_MEM_POWER_LS, 0x00000001, 0x00000001,
920 	mmHDP_XDP_CGTT_BLK_CTRL, 0xc0000fff, 0x00000104,
921 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
922 	0x3430, 0xfffffff0, 0x00000100,
923 	0x3630, 0xfffffff0, 0x00000100,
924 };
925 
926 /* tahiti, pitcairn, verde */
927 static const struct amdgpu_video_codec_info tahiti_video_codecs_encode_array[] =
928 {
929 	{
930 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
931 		.max_width = 2048,
932 		.max_height = 1152,
933 		.max_pixels_per_frame = 2048 * 1152,
934 		.max_level = 0,
935 	},
936 };
937 
938 static const struct amdgpu_video_codecs tahiti_video_codecs_encode =
939 {
940 	.codec_count = ARRAY_SIZE(tahiti_video_codecs_encode_array),
941 	.codec_array = tahiti_video_codecs_encode_array,
942 };
943 
944 /* oland and hainan don't support encode */
945 static const struct amdgpu_video_codecs hainan_video_codecs_encode =
946 {
947 	.codec_count = 0,
948 	.codec_array = NULL,
949 };
950 
951 /* tahiti, pitcairn, verde, oland */
952 static const struct amdgpu_video_codec_info tahiti_video_codecs_decode_array[] =
953 {
954 	{
955 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
956 		.max_width = 2048,
957 		.max_height = 1152,
958 		.max_pixels_per_frame = 2048 * 1152,
959 		.max_level = 3,
960 	},
961 	{
962 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
963 		.max_width = 2048,
964 		.max_height = 1152,
965 		.max_pixels_per_frame = 2048 * 1152,
966 		.max_level = 5,
967 	},
968 	{
969 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
970 		.max_width = 2048,
971 		.max_height = 1152,
972 		.max_pixels_per_frame = 2048 * 1152,
973 		.max_level = 41,
974 	},
975 	{
976 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
977 		.max_width = 2048,
978 		.max_height = 1152,
979 		.max_pixels_per_frame = 2048 * 1152,
980 		.max_level = 4,
981 	},
982 };
983 
984 static const struct amdgpu_video_codecs tahiti_video_codecs_decode =
985 {
986 	.codec_count = ARRAY_SIZE(tahiti_video_codecs_decode_array),
987 	.codec_array = tahiti_video_codecs_decode_array,
988 };
989 
990 /* hainan doesn't support decode */
991 static const struct amdgpu_video_codecs hainan_video_codecs_decode =
992 {
993 	.codec_count = 0,
994 	.codec_array = NULL,
995 };
996 
997 static int si_query_video_codecs(struct amdgpu_device *adev, bool encode,
998 				 const struct amdgpu_video_codecs **codecs)
999 {
1000 	switch (adev->asic_type) {
1001 	case CHIP_VERDE:
1002 	case CHIP_TAHITI:
1003 	case CHIP_PITCAIRN:
1004 		if (encode)
1005 			*codecs = &tahiti_video_codecs_encode;
1006 		else
1007 			*codecs = &tahiti_video_codecs_decode;
1008 		return 0;
1009 	case CHIP_OLAND:
1010 		if (encode)
1011 			*codecs = &hainan_video_codecs_encode;
1012 		else
1013 			*codecs = &tahiti_video_codecs_decode;
1014 		return 0;
1015 	case CHIP_HAINAN:
1016 		if (encode)
1017 			*codecs = &hainan_video_codecs_encode;
1018 		else
1019 			*codecs = &hainan_video_codecs_decode;
1020 		return 0;
1021 	default:
1022 		return -EINVAL;
1023 	}
1024 }
1025 
1026 static u32 si_pcie_rreg(struct amdgpu_device *adev, u32 reg)
1027 {
1028 	unsigned long flags;
1029 	u32 r;
1030 
1031 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
1032 	WREG32(AMDGPU_PCIE_INDEX, reg);
1033 	(void)RREG32(AMDGPU_PCIE_INDEX);
1034 	r = RREG32(AMDGPU_PCIE_DATA);
1035 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
1036 	return r;
1037 }
1038 
1039 static void si_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1040 {
1041 	unsigned long flags;
1042 
1043 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
1044 	WREG32(AMDGPU_PCIE_INDEX, reg);
1045 	(void)RREG32(AMDGPU_PCIE_INDEX);
1046 	WREG32(AMDGPU_PCIE_DATA, v);
1047 	(void)RREG32(AMDGPU_PCIE_DATA);
1048 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
1049 }
1050 
1051 static u32 si_pciep_rreg(struct amdgpu_device *adev, u32 reg)
1052 {
1053 	unsigned long flags;
1054 	u32 r;
1055 
1056 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
1057 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1058 	(void)RREG32(PCIE_PORT_INDEX);
1059 	r = RREG32(PCIE_PORT_DATA);
1060 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
1061 	return r;
1062 }
1063 
1064 static void si_pciep_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1065 {
1066 	unsigned long flags;
1067 
1068 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
1069 	WREG32(PCIE_PORT_INDEX, ((reg) & 0xff));
1070 	(void)RREG32(PCIE_PORT_INDEX);
1071 	WREG32(PCIE_PORT_DATA, (v));
1072 	(void)RREG32(PCIE_PORT_DATA);
1073 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
1074 }
1075 
1076 static u32 si_smc_rreg(struct amdgpu_device *adev, u32 reg)
1077 {
1078 	unsigned long flags;
1079 	u32 r;
1080 
1081 	spin_lock_irqsave(&adev->reg.smc.lock, flags);
1082 	WREG32(mmSMC_IND_INDEX_0, (reg));
1083 	r = RREG32(mmSMC_IND_DATA_0);
1084 	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
1085 	return r;
1086 }
1087 
1088 static void si_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1089 {
1090 	unsigned long flags;
1091 
1092 	spin_lock_irqsave(&adev->reg.smc.lock, flags);
1093 	WREG32(mmSMC_IND_INDEX_0, (reg));
1094 	WREG32(mmSMC_IND_DATA_0, (v));
1095 	spin_unlock_irqrestore(&adev->reg.smc.lock, flags);
1096 }
1097 
1098 static u32 si_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
1099 {
1100 	unsigned long flags;
1101 	u32 r;
1102 
1103 	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
1104 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1105 	r = RREG32(mmUVD_CTX_DATA);
1106 	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
1107 	return r;
1108 }
1109 
1110 static void si_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
1111 {
1112 	unsigned long flags;
1113 
1114 	spin_lock_irqsave(&adev->reg.uvd_ctx.lock, flags);
1115 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
1116 	WREG32(mmUVD_CTX_DATA, (v));
1117 	spin_unlock_irqrestore(&adev->reg.uvd_ctx.lock, flags);
1118 }
1119 
1120 static struct amdgpu_allowed_register_entry si_allowed_read_registers[] = {
1121 	{mmGRBM_STATUS},
1122 	{mmGRBM_STATUS2},
1123 	{mmGRBM_STATUS_SE0},
1124 	{mmGRBM_STATUS_SE1},
1125 	{mmSRBM_STATUS},
1126 	{mmSRBM_STATUS2},
1127 	{mmDMA_STATUS_REG + DMA0_REGISTER_OFFSET},
1128 	{mmDMA_STATUS_REG + DMA1_REGISTER_OFFSET},
1129 	{mmCP_STAT},
1130 	{mmCP_STALLED_STAT1},
1131 	{mmCP_STALLED_STAT2},
1132 	{mmCP_STALLED_STAT3},
1133 	{mmGB_ADDR_CONFIG},
1134 	{mmMC_ARB_RAMCFG},
1135 	{mmGB_TILE_MODE0},
1136 	{mmGB_TILE_MODE1},
1137 	{mmGB_TILE_MODE2},
1138 	{mmGB_TILE_MODE3},
1139 	{mmGB_TILE_MODE4},
1140 	{mmGB_TILE_MODE5},
1141 	{mmGB_TILE_MODE6},
1142 	{mmGB_TILE_MODE7},
1143 	{mmGB_TILE_MODE8},
1144 	{mmGB_TILE_MODE9},
1145 	{mmGB_TILE_MODE10},
1146 	{mmGB_TILE_MODE11},
1147 	{mmGB_TILE_MODE12},
1148 	{mmGB_TILE_MODE13},
1149 	{mmGB_TILE_MODE14},
1150 	{mmGB_TILE_MODE15},
1151 	{mmGB_TILE_MODE16},
1152 	{mmGB_TILE_MODE17},
1153 	{mmGB_TILE_MODE18},
1154 	{mmGB_TILE_MODE19},
1155 	{mmGB_TILE_MODE20},
1156 	{mmGB_TILE_MODE21},
1157 	{mmGB_TILE_MODE22},
1158 	{mmGB_TILE_MODE23},
1159 	{mmGB_TILE_MODE24},
1160 	{mmGB_TILE_MODE25},
1161 	{mmGB_TILE_MODE26},
1162 	{mmGB_TILE_MODE27},
1163 	{mmGB_TILE_MODE28},
1164 	{mmGB_TILE_MODE29},
1165 	{mmGB_TILE_MODE30},
1166 	{mmGB_TILE_MODE31},
1167 	{mmCC_RB_BACKEND_DISABLE, true},
1168 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1169 	{mmPA_SC_RASTER_CONFIG, true},
1170 };
1171 
1172 static uint32_t si_get_register_value(struct amdgpu_device *adev,
1173 				      bool indexed, u32 se_num,
1174 				      u32 sh_num, u32 reg_offset)
1175 {
1176 	if (indexed) {
1177 		uint32_t val;
1178 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1179 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1180 
1181 		switch (reg_offset) {
1182 		case mmCC_RB_BACKEND_DISABLE:
1183 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1184 		case mmGC_USER_RB_BACKEND_DISABLE:
1185 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1186 		case mmPA_SC_RASTER_CONFIG:
1187 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1188 		}
1189 
1190 		mutex_lock(&adev->grbm_idx_mutex);
1191 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1192 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1193 
1194 		val = RREG32(reg_offset);
1195 
1196 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1197 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1198 		mutex_unlock(&adev->grbm_idx_mutex);
1199 		return val;
1200 	} else {
1201 		unsigned idx;
1202 
1203 		switch (reg_offset) {
1204 		case mmGB_ADDR_CONFIG:
1205 			return adev->gfx.config.gb_addr_config;
1206 		case mmMC_ARB_RAMCFG:
1207 			return adev->gfx.config.mc_arb_ramcfg;
1208 		case mmGB_TILE_MODE0:
1209 		case mmGB_TILE_MODE1:
1210 		case mmGB_TILE_MODE2:
1211 		case mmGB_TILE_MODE3:
1212 		case mmGB_TILE_MODE4:
1213 		case mmGB_TILE_MODE5:
1214 		case mmGB_TILE_MODE6:
1215 		case mmGB_TILE_MODE7:
1216 		case mmGB_TILE_MODE8:
1217 		case mmGB_TILE_MODE9:
1218 		case mmGB_TILE_MODE10:
1219 		case mmGB_TILE_MODE11:
1220 		case mmGB_TILE_MODE12:
1221 		case mmGB_TILE_MODE13:
1222 		case mmGB_TILE_MODE14:
1223 		case mmGB_TILE_MODE15:
1224 		case mmGB_TILE_MODE16:
1225 		case mmGB_TILE_MODE17:
1226 		case mmGB_TILE_MODE18:
1227 		case mmGB_TILE_MODE19:
1228 		case mmGB_TILE_MODE20:
1229 		case mmGB_TILE_MODE21:
1230 		case mmGB_TILE_MODE22:
1231 		case mmGB_TILE_MODE23:
1232 		case mmGB_TILE_MODE24:
1233 		case mmGB_TILE_MODE25:
1234 		case mmGB_TILE_MODE26:
1235 		case mmGB_TILE_MODE27:
1236 		case mmGB_TILE_MODE28:
1237 		case mmGB_TILE_MODE29:
1238 		case mmGB_TILE_MODE30:
1239 		case mmGB_TILE_MODE31:
1240 			idx = (reg_offset - mmGB_TILE_MODE0);
1241 			return adev->gfx.config.tile_mode_array[idx];
1242 		default:
1243 			return RREG32(reg_offset);
1244 		}
1245 	}
1246 }
1247 static int si_read_register(struct amdgpu_device *adev, u32 se_num,
1248 			     u32 sh_num, u32 reg_offset, u32 *value)
1249 {
1250 	uint32_t i;
1251 
1252 	*value = 0;
1253 	for (i = 0; i < ARRAY_SIZE(si_allowed_read_registers); i++) {
1254 		bool indexed = si_allowed_read_registers[i].grbm_indexed;
1255 
1256 		if (reg_offset != si_allowed_read_registers[i].reg_offset)
1257 			continue;
1258 
1259 		*value = si_get_register_value(adev, indexed, se_num, sh_num,
1260 					       reg_offset);
1261 		return 0;
1262 	}
1263 	return -EINVAL;
1264 }
1265 
1266 static bool si_read_disabled_bios(struct amdgpu_device *adev)
1267 {
1268 	u32 bus_cntl;
1269 	u32 d1vga_control = 0;
1270 	u32 d2vga_control = 0;
1271 	u32 vga_render_control = 0;
1272 	u32 rom_cntl;
1273 	bool r;
1274 
1275 	bus_cntl = RREG32(mmBUS_CNTL);
1276 	if (adev->mode_info.num_crtc) {
1277 		d1vga_control = RREG32(mmD1VGA_CONTROL);
1278 		d2vga_control = RREG32(mmD2VGA_CONTROL);
1279 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
1280 	}
1281 	rom_cntl = RREG32(R600_ROM_CNTL);
1282 
1283 	/* enable the rom */
1284 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
1285 	if (adev->mode_info.num_crtc) {
1286 		/* Disable VGA mode */
1287 		WREG32(mmD1VGA_CONTROL,
1288 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
1289 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
1290 		WREG32(mmD2VGA_CONTROL,
1291 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
1292 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
1293 		WREG32(mmVGA_RENDER_CONTROL,
1294 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
1295 	}
1296 	WREG32(R600_ROM_CNTL, rom_cntl | R600_SCK_OVERWRITE);
1297 
1298 	r = amdgpu_read_bios(adev);
1299 
1300 	/* restore regs */
1301 	WREG32(mmBUS_CNTL, bus_cntl);
1302 	if (adev->mode_info.num_crtc) {
1303 		WREG32(mmD1VGA_CONTROL, d1vga_control);
1304 		WREG32(mmD2VGA_CONTROL, d2vga_control);
1305 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1306 	}
1307 	WREG32(R600_ROM_CNTL, rom_cntl);
1308 	return r;
1309 }
1310 
1311 #define mmROM_INDEX 0x2A
1312 #define mmROM_DATA  0x2B
1313 
1314 static bool si_read_bios_from_rom(struct amdgpu_device *adev,
1315 				  u8 *bios, u32 length_bytes)
1316 {
1317 	u32 *dw_ptr;
1318 	u32 i, length_dw;
1319 
1320 	if (bios == NULL)
1321 		return false;
1322 	if (length_bytes == 0)
1323 		return false;
1324 	/* APU vbios image is part of sbios image */
1325 	if (adev->flags & AMD_IS_APU)
1326 		return false;
1327 
1328 	dw_ptr = (u32 *)bios;
1329 	length_dw = ALIGN(length_bytes, 4) / 4;
1330 	/* set rom index to 0 */
1331 	WREG32(mmROM_INDEX, 0);
1332 	for (i = 0; i < length_dw; i++)
1333 		dw_ptr[i] = RREG32(mmROM_DATA);
1334 
1335 	return true;
1336 }
1337 
1338 static void si_set_clk_bypass_mode(struct amdgpu_device *adev)
1339 {
1340 	u32 tmp, i;
1341 
1342 	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1343 	tmp |= CG_SPLL_FUNC_CNTL__SPLL_BYPASS_EN_MASK;
1344 	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1345 
1346 	tmp = RREG32(mmCG_SPLL_FUNC_CNTL_2);
1347 	tmp |= CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK;
1348 	WREG32(mmCG_SPLL_FUNC_CNTL_2, tmp);
1349 
1350 	for (i = 0; i < adev->usec_timeout; i++) {
1351 		if (RREG32(mmCG_SPLL_STATUS) & CG_SPLL_STATUS__SPLL_CHG_STATUS_MASK)
1352 			break;
1353 		udelay(1);
1354 	}
1355 
1356 	tmp = RREG32(mmCG_SPLL_FUNC_CNTL_2);
1357 	tmp &= ~(CG_SPLL_FUNC_CNTL_2__SPLL_CTLREQ_CHG_MASK |
1358 		CG_SPLL_FUNC_CNTL_2__SCLK_MUX_UPDATE_MASK);
1359 	WREG32(mmCG_SPLL_FUNC_CNTL_2, tmp);
1360 
1361 	tmp = RREG32(MPLL_CNTL_MODE);
1362 	tmp &= ~MPLL_MCLK_SEL;
1363 	WREG32(MPLL_CNTL_MODE, tmp);
1364 }
1365 
1366 static void si_spll_powerdown(struct amdgpu_device *adev)
1367 {
1368 	u32 tmp;
1369 
1370 	tmp = RREG32(mmSPLL_CNTL_MODE);
1371 	tmp |= SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK;
1372 	WREG32(mmSPLL_CNTL_MODE, tmp);
1373 
1374 	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1375 	tmp |= CG_SPLL_FUNC_CNTL__SPLL_RESET_MASK;
1376 	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1377 
1378 	tmp = RREG32(mmCG_SPLL_FUNC_CNTL);
1379 	tmp |= CG_SPLL_FUNC_CNTL__SPLL_SLEEP_MASK;
1380 	WREG32(mmCG_SPLL_FUNC_CNTL, tmp);
1381 
1382 	tmp = RREG32(mmSPLL_CNTL_MODE);
1383 	tmp &= ~SPLL_CNTL_MODE__SPLL_SW_DIR_CONTROL_MASK;
1384 	WREG32(mmSPLL_CNTL_MODE, tmp);
1385 }
1386 
1387 static int si_gpu_pci_config_reset(struct amdgpu_device *adev)
1388 {
1389 	u32 i;
1390 	int r = -EINVAL;
1391 
1392 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1393 
1394 	/* set mclk/sclk to bypass */
1395 	si_set_clk_bypass_mode(adev);
1396 	/* powerdown spll */
1397 	si_spll_powerdown(adev);
1398 	/* disable BM */
1399 	pci_clear_master(adev->pdev);
1400 	/* reset */
1401 	amdgpu_device_pci_config_reset(adev);
1402 
1403 	udelay(100);
1404 
1405 	/* wait for asic to come out of reset */
1406 	for (i = 0; i < adev->usec_timeout; i++) {
1407 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1408 			/* enable BM */
1409 			pci_set_master(adev->pdev);
1410 			adev->has_hw_reset = true;
1411 			r = 0;
1412 			break;
1413 		}
1414 		udelay(1);
1415 	}
1416 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1417 
1418 	return r;
1419 }
1420 
1421 static int si_asic_supports_baco(struct amdgpu_device *adev)
1422 {
1423 	return 0;
1424 }
1425 
1426 static enum amd_reset_method
1427 si_asic_reset_method(struct amdgpu_device *adev)
1428 {
1429 	if (amdgpu_reset_method == AMD_RESET_METHOD_PCI)
1430 		return amdgpu_reset_method;
1431 	else if (amdgpu_reset_method != AMD_RESET_METHOD_LEGACY &&
1432 		 amdgpu_reset_method != -1)
1433 		dev_warn(adev->dev, "Specified reset method:%d isn't supported, using AUTO instead.\n",
1434 			 amdgpu_reset_method);
1435 
1436 	return AMD_RESET_METHOD_LEGACY;
1437 }
1438 
1439 static int si_asic_reset(struct amdgpu_device *adev)
1440 {
1441 	int r;
1442 
1443 	switch (si_asic_reset_method(adev)) {
1444 	case AMD_RESET_METHOD_PCI:
1445 		dev_info(adev->dev, "PCI reset\n");
1446 		r = amdgpu_device_pci_reset(adev);
1447 		break;
1448 	default:
1449 		dev_info(adev->dev, "PCI CONFIG reset\n");
1450 		r = si_gpu_pci_config_reset(adev);
1451 		break;
1452 	}
1453 
1454 	return r;
1455 }
1456 
1457 static u32 si_get_config_memsize(struct amdgpu_device *adev)
1458 {
1459 	return RREG32(mmCONFIG_MEMSIZE);
1460 }
1461 
1462 static void si_vga_set_state(struct amdgpu_device *adev, bool state)
1463 {
1464 	uint32_t temp;
1465 
1466 	temp = RREG32(mmCONFIG_CNTL);
1467 	if (!state) {
1468 		temp &= ~(1<<0);
1469 		temp |= (1<<1);
1470 	} else {
1471 		temp &= ~(1<<1);
1472 	}
1473 	WREG32(mmCONFIG_CNTL, temp);
1474 }
1475 
1476 static u32 si_get_xclk(struct amdgpu_device *adev)
1477 {
1478 	u32 reference_clock = adev->clock.spll.reference_freq;
1479 	u32 tmp;
1480 
1481 	tmp = RREG32(mmCG_CLKPIN_CNTL_2);
1482 	if (tmp & CG_CLKPIN_CNTL_2__MUX_TCLK_TO_XCLK_MASK)
1483 		return TCLK;
1484 
1485 	tmp = RREG32(mmCG_CLKPIN_CNTL);
1486 	if (tmp & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
1487 		return reference_clock / 4;
1488 
1489 	return reference_clock;
1490 }
1491 
1492 static void si_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1493 {
1494 	if (!ring || !ring->funcs->emit_wreg) {
1495 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1496 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1497 	} else {
1498 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1499 	}
1500 }
1501 
1502 static void si_invalidate_hdp(struct amdgpu_device *adev,
1503 			      struct amdgpu_ring *ring)
1504 {
1505 	if (!ring || !ring->funcs->emit_wreg) {
1506 		WREG32(mmHDP_DEBUG0, 1);
1507 		RREG32(mmHDP_DEBUG0);
1508 	} else {
1509 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1510 	}
1511 }
1512 
1513 static bool si_need_reset_on_init(struct amdgpu_device *adev)
1514 {
1515 	/*
1516 	 * MacBookPros with switchable graphics put VRAM at 0 when the iGPU
1517 	 * is enabled.  Any later post (S3 resume, GPU reset) moves it back to
1518 	 * the vbios default, so reset and post on load to start with the
1519 	 * vbios layout.
1520 	 */
1521 	if (adev->pdev->subsystem_vendor == PCI_VENDOR_ID_APPLE &&
1522 	    !(RREG32(mmMC_VM_FB_LOCATION) & MC_VM_FB_LOCATION__FB_BASE_MASK))
1523 		return true;
1524 
1525 	return false;
1526 }
1527 
1528 static int si_get_pcie_lanes(struct amdgpu_device *adev)
1529 {
1530 	u32 link_width_cntl;
1531 
1532 	if (adev->flags & AMD_IS_APU)
1533 		return 0;
1534 
1535 	link_width_cntl = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
1536 
1537 	switch ((link_width_cntl & PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD_MASK) >> PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_RD__SHIFT) {
1538 	case LC_LINK_WIDTH_X1:
1539 		return 1;
1540 	case LC_LINK_WIDTH_X2:
1541 		return 2;
1542 	case LC_LINK_WIDTH_X4:
1543 		return 4;
1544 	case LC_LINK_WIDTH_X8:
1545 		return 8;
1546 	case LC_LINK_WIDTH_X0:
1547 	case LC_LINK_WIDTH_X16:
1548 	default:
1549 		return 16;
1550 	}
1551 }
1552 
1553 static void si_set_pcie_lanes(struct amdgpu_device *adev, int lanes)
1554 {
1555 	u32 link_width_cntl, mask;
1556 
1557 	if (adev->flags & AMD_IS_APU)
1558 		return;
1559 
1560 	switch (lanes) {
1561 	case 0:
1562 		mask = LC_LINK_WIDTH_X0;
1563 		break;
1564 	case 1:
1565 		mask = LC_LINK_WIDTH_X1;
1566 		break;
1567 	case 2:
1568 		mask = LC_LINK_WIDTH_X2;
1569 		break;
1570 	case 4:
1571 		mask = LC_LINK_WIDTH_X4;
1572 		break;
1573 	case 8:
1574 		mask = LC_LINK_WIDTH_X8;
1575 		break;
1576 	case 16:
1577 		mask = LC_LINK_WIDTH_X16;
1578 		break;
1579 	default:
1580 		DRM_ERROR("invalid pcie lane request: %d\n", lanes);
1581 		return;
1582 	}
1583 
1584 	link_width_cntl = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
1585 	link_width_cntl &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK;
1586 	link_width_cntl |= mask << PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT;
1587 	link_width_cntl |= (PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK |
1588 		PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_ARC_MISSING_ESCAPE_MASK);
1589 
1590 	WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
1591 }
1592 
1593 static void si_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1594 			      uint64_t *count1)
1595 {
1596 	uint32_t perfctr = 0;
1597 	uint64_t cnt0_of, cnt1_of;
1598 	int tmp;
1599 
1600 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1601 	 * that may or may not be different from their GPU counterparts
1602 	 */
1603 	if (adev->flags & AMD_IS_APU)
1604 		return;
1605 
1606 	/* Set the 2 events that we wish to watch, defined above */
1607 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1608 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1609 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1610 
1611 	/* Write to enable desired perf counters */
1612 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1613 	/* Zero out and enable the perf counters
1614 	 * Write 0x5:
1615 	 * Bit 0 = Start all counters(1)
1616 	 * Bit 2 = Global counter reset enable(1)
1617 	 */
1618 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1619 
1620 	msleep(1000);
1621 
1622 	/* Load the shadow and disable the perf counters
1623 	 * Write 0x2:
1624 	 * Bit 0 = Stop counters(0)
1625 	 * Bit 1 = Load the shadow counters(1)
1626 	 */
1627 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1628 
1629 	/* Read register values to get any >32bit overflow */
1630 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1631 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1632 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1633 
1634 	/* Get the values and add the overflow */
1635 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1636 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1637 }
1638 
1639 static uint64_t si_get_pcie_replay_count(struct amdgpu_device *adev)
1640 {
1641 	uint64_t nak_r, nak_g;
1642 
1643 	/* Get the number of NAKs received and generated */
1644 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1645 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1646 
1647 	/* Add the total number of NAKs, i.e the number of replays */
1648 	return (nak_r + nak_g);
1649 }
1650 
1651 static int si_uvd_send_upll_ctlreq(struct amdgpu_device *adev,
1652 				   unsigned cg_upll_func_cntl)
1653 {
1654 	unsigned i;
1655 
1656 	/* Make sure UPLL_CTLREQ is deasserted */
1657 	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1658 
1659 	mdelay(10);
1660 
1661 	/* Assert UPLL_CTLREQ */
1662 	WREG32_P(cg_upll_func_cntl, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1663 
1664 	/* Wait for CTLACK and CTLACK2 to get asserted */
1665 	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1666 		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1667 
1668 		if ((RREG32(cg_upll_func_cntl) & mask) == mask)
1669 			break;
1670 		mdelay(10);
1671 	}
1672 
1673 	/* Deassert UPLL_CTLREQ */
1674 	WREG32_P(cg_upll_func_cntl, 0, ~UPLL_CTLREQ_MASK);
1675 
1676 	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1677 		DRM_ERROR("Timeout setting UVD clocks!\n");
1678 		return -ETIMEDOUT;
1679 	}
1680 
1681 	return 0;
1682 }
1683 
1684 static unsigned si_uvd_calc_upll_post_div(unsigned vco_freq,
1685 					  unsigned target_freq,
1686 					  unsigned pd_min,
1687 					  unsigned pd_even)
1688 {
1689 	unsigned post_div = vco_freq / target_freq;
1690 
1691 	/* Adjust to post divider minimum value */
1692 	if (post_div < pd_min)
1693 		post_div = pd_min;
1694 
1695 	/* We alway need a frequency less than or equal the target */
1696 	if ((vco_freq / post_div) > target_freq)
1697 		post_div += 1;
1698 
1699 	/* Post dividers above a certain value must be even */
1700 	if (post_div > pd_even && post_div % 2)
1701 		post_div += 1;
1702 
1703 	return post_div;
1704 }
1705 
1706 /**
1707  * si_calc_upll_dividers - calc UPLL clock dividers
1708  *
1709  * @adev: amdgpu_device pointer
1710  * @vclk: wanted VCLK
1711  * @dclk: wanted DCLK
1712  * @vco_min: minimum VCO frequency
1713  * @vco_max: maximum VCO frequency
1714  * @fb_factor: factor to multiply vco freq with
1715  * @fb_mask: limit and bitmask for feedback divider
1716  * @pd_min: post divider minimum
1717  * @pd_max: post divider maximum
1718  * @pd_even: post divider must be even above this value
1719  * @optimal_fb_div: resulting feedback divider
1720  * @optimal_vclk_div: resulting vclk post divider
1721  * @optimal_dclk_div: resulting dclk post divider
1722  *
1723  * Calculate dividers for UVDs UPLL (except APUs).
1724  * Returns zero on success; -EINVAL on error.
1725  */
1726 static int si_calc_upll_dividers(struct amdgpu_device *adev,
1727 				 unsigned vclk, unsigned dclk,
1728 				 unsigned vco_min, unsigned vco_max,
1729 				 unsigned fb_factor, unsigned fb_mask,
1730 				 unsigned pd_min, unsigned pd_max,
1731 				 unsigned pd_even,
1732 				 unsigned *optimal_fb_div,
1733 				 unsigned *optimal_vclk_div,
1734 				 unsigned *optimal_dclk_div)
1735 {
1736 	unsigned vco_freq, ref_freq = adev->clock.spll.reference_freq;
1737 
1738 	/* Start off with something large */
1739 	unsigned optimal_score = ~0;
1740 
1741 	/* Loop through vco from low to high */
1742 	vco_min = max(max(vco_min, vclk), dclk);
1743 	for (vco_freq = vco_min; vco_freq <= vco_max; vco_freq += 100) {
1744 		uint64_t fb_div = (uint64_t)vco_freq * fb_factor;
1745 		unsigned vclk_div, dclk_div, score;
1746 
1747 		do_div(fb_div, ref_freq);
1748 
1749 		/* fb div out of range ? */
1750 		if (fb_div > fb_mask)
1751 			break; /* It can oly get worse */
1752 
1753 		fb_div &= fb_mask;
1754 
1755 		/* Calc vclk divider with current vco freq */
1756 		vclk_div = si_uvd_calc_upll_post_div(vco_freq, vclk,
1757 						     pd_min, pd_even);
1758 		if (vclk_div > pd_max)
1759 			break; /* vco is too big, it has to stop */
1760 
1761 		/* Calc dclk divider with current vco freq */
1762 		dclk_div = si_uvd_calc_upll_post_div(vco_freq, dclk,
1763 						     pd_min, pd_even);
1764 		if (dclk_div > pd_max)
1765 			break; /* vco is too big, it has to stop */
1766 
1767 		/* Calc score with current vco freq */
1768 		score = vclk - (vco_freq / vclk_div) + dclk - (vco_freq / dclk_div);
1769 
1770 		/* Determine if this vco setting is better than current optimal settings */
1771 		if (score < optimal_score) {
1772 			*optimal_fb_div = fb_div;
1773 			*optimal_vclk_div = vclk_div;
1774 			*optimal_dclk_div = dclk_div;
1775 			optimal_score = score;
1776 			if (optimal_score == 0)
1777 				break; /* It can't get better than this */
1778 		}
1779 	}
1780 
1781 	/* Did we found a valid setup ? */
1782 	if (optimal_score == ~0)
1783 		return -EINVAL;
1784 
1785 	return 0;
1786 }
1787 
1788 static int si_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1789 {
1790 	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1791 	int r;
1792 
1793 	/* Bypass vclk and dclk with bclk */
1794 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1795 		 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1796 		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1797 
1798 	/* Put PLL in bypass mode */
1799 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1800 
1801 	if (!vclk || !dclk) {
1802 		/* Keep the Bypass mode */
1803 		return 0;
1804 	}
1805 
1806 	r = si_calc_upll_dividers(adev, vclk, dclk, 125000, 250000,
1807 				  16384, 0x03FFFFFF, 0, 128, 5,
1808 				  &fb_div, &vclk_div, &dclk_div);
1809 	if (r)
1810 		return r;
1811 
1812 	/* Set RESET_ANTI_MUX to 0 */
1813 	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1814 
1815 	/* Set VCO_MODE to 1 */
1816 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1817 
1818 	/* Disable sleep mode */
1819 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1820 
1821 	/* Deassert UPLL_RESET */
1822 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1823 
1824 	mdelay(1);
1825 
1826 	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1827 	if (r)
1828 		return r;
1829 
1830 	/* Assert UPLL_RESET again */
1831 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1832 
1833 	/* Disable spread spectrum. */
1834 	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1835 
1836 	/* Set feedback divider */
1837 	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1838 
1839 	/* Set ref divider to 0 */
1840 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1841 
1842 	if (fb_div < 307200)
1843 		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1844 	else
1845 		WREG32_P(CG_UPLL_FUNC_CNTL_4,
1846 			 UPLL_SPARE_ISPARE9,
1847 			 ~UPLL_SPARE_ISPARE9);
1848 
1849 	/* Set PDIV_A and PDIV_B */
1850 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1851 		 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1852 		 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1853 
1854 	/* Give the PLL some time to settle */
1855 	mdelay(15);
1856 
1857 	/* Deassert PLL_RESET */
1858 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1859 
1860 	mdelay(15);
1861 
1862 	/* Switch from bypass mode to normal mode */
1863 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1864 
1865 	r = si_uvd_send_upll_ctlreq(adev, CG_UPLL_FUNC_CNTL);
1866 	if (r)
1867 		return r;
1868 
1869 	/* Switch VCLK and DCLK selection */
1870 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
1871 		 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1872 		 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1873 
1874 	mdelay(100);
1875 
1876 	return 0;
1877 }
1878 
1879 static int si_vce_send_vcepll_ctlreq(struct amdgpu_device *adev)
1880 {
1881 	unsigned i;
1882 
1883 	/* Make sure VCEPLL_CTLREQ is deasserted */
1884 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1885 
1886 	mdelay(10);
1887 
1888 	/* Assert UPLL_CTLREQ */
1889 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
1890 
1891 	/* Wait for CTLACK and CTLACK2 to get asserted */
1892 	for (i = 0; i < SI_MAX_CTLACKS_ASSERTION_WAIT; ++i) {
1893 		uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
1894 
1895 		if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
1896 			break;
1897 		mdelay(10);
1898 	}
1899 
1900 	/* Deassert UPLL_CTLREQ */
1901 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
1902 
1903 	if (i == SI_MAX_CTLACKS_ASSERTION_WAIT) {
1904 		DRM_ERROR("Timeout setting VCE clocks!\n");
1905 		return -ETIMEDOUT;
1906 	}
1907 
1908 	return 0;
1909 }
1910 
1911 static int si_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1912 {
1913 	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
1914 	int r;
1915 
1916 	/* Bypass evclk and ecclk with bclk */
1917 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1918 		     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
1919 		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
1920 
1921 	/* Put PLL in bypass mode */
1922 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
1923 		     ~VCEPLL_BYPASS_EN_MASK);
1924 
1925 	if (!evclk || !ecclk) {
1926 		/*
1927 		 * On some chips, the PLL takes way too long to get out of
1928 		 * sleep mode, causing a timeout waiting on CTLACK/CTLACK2.
1929 		 * Leave the PLL running in bypass mode.
1930 		 */
1931 		if (adev->pdev->device == 0x6780)
1932 			return 0;
1933 
1934 		/* Keep the Bypass mode, put PLL to sleep */
1935 		WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1936 			     ~VCEPLL_SLEEP_MASK);
1937 		return 0;
1938 	}
1939 
1940 	r = si_calc_upll_dividers(adev, evclk, ecclk, 125000, 250000,
1941 				  16384, 0x03FFFFFF, 0, 128, 5,
1942 				  &fb_div, &evclk_div, &ecclk_div);
1943 	if (r)
1944 		return r;
1945 
1946 	/* Set RESET_ANTI_MUX to 0 */
1947 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
1948 
1949 	/* Set VCO_MODE to 1 */
1950 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
1951 		     ~VCEPLL_VCO_MODE_MASK);
1952 
1953 	/* Toggle VCEPLL_SLEEP to 1 then back to 0 */
1954 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
1955 		     ~VCEPLL_SLEEP_MASK);
1956 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
1957 
1958 	/* Deassert VCEPLL_RESET */
1959 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1960 
1961 	mdelay(1);
1962 
1963 	r = si_vce_send_vcepll_ctlreq(adev);
1964 	if (r)
1965 		return r;
1966 
1967 	/* Assert VCEPLL_RESET again */
1968 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
1969 
1970 	/* Disable spread spectrum. */
1971 	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1972 
1973 	/* Set feedback divider */
1974 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3,
1975 		     VCEPLL_FB_DIV(fb_div),
1976 		     ~VCEPLL_FB_DIV_MASK);
1977 
1978 	/* Set ref divider to 0 */
1979 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
1980 
1981 	/* Set PDIV_A and PDIV_B */
1982 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
1983 		     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
1984 		     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
1985 
1986 	/* Give the PLL some time to settle */
1987 	mdelay(15);
1988 
1989 	/* Deassert PLL_RESET */
1990 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
1991 
1992 	mdelay(15);
1993 
1994 	/* Switch from bypass mode to normal mode */
1995 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
1996 
1997 	r = si_vce_send_vcepll_ctlreq(adev);
1998 	if (r)
1999 		return r;
2000 
2001 	/* Switch VCLK and DCLK selection */
2002 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
2003 		     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
2004 		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
2005 
2006 	mdelay(100);
2007 
2008 	return 0;
2009 }
2010 
2011 static const struct amdgpu_asic_funcs si_asic_funcs =
2012 {
2013 	.read_disabled_bios = &si_read_disabled_bios,
2014 	.read_bios_from_rom = &si_read_bios_from_rom,
2015 	.read_register = &si_read_register,
2016 	.reset = &si_asic_reset,
2017 	.reset_method = &si_asic_reset_method,
2018 	.set_vga_state = &si_vga_set_state,
2019 	.get_xclk = &si_get_xclk,
2020 	.set_uvd_clocks = &si_set_uvd_clocks,
2021 	.set_vce_clocks = &si_set_vce_clocks,
2022 	.get_pcie_lanes = &si_get_pcie_lanes,
2023 	.set_pcie_lanes = &si_set_pcie_lanes,
2024 	.get_config_memsize = &si_get_config_memsize,
2025 	.flush_hdp = &si_flush_hdp,
2026 	.invalidate_hdp = &si_invalidate_hdp,
2027 	.get_pcie_usage = &si_get_pcie_usage,
2028 	.need_reset_on_init = &si_need_reset_on_init,
2029 	.get_pcie_replay_count = &si_get_pcie_replay_count,
2030 	.supports_baco = &si_asic_supports_baco,
2031 	.query_video_codecs = &si_query_video_codecs,
2032 };
2033 
2034 static uint32_t si_get_rev_id(struct amdgpu_device *adev)
2035 {
2036 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
2037 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
2038 }
2039 
2040 static int si_common_early_init(struct amdgpu_ip_block *ip_block)
2041 {
2042 	struct amdgpu_device *adev = ip_block->adev;
2043 
2044 	adev->reg.smc.rreg = si_smc_rreg;
2045 	adev->reg.smc.wreg = si_smc_wreg;
2046 	adev->reg.pcie.rreg = &si_pcie_rreg;
2047 	adev->reg.pcie.wreg = &si_pcie_wreg;
2048 	adev->reg.pcie.port_rreg = &si_pciep_rreg;
2049 	adev->reg.pcie.port_wreg = &si_pciep_wreg;
2050 	adev->reg.uvd_ctx.rreg = &si_uvd_ctx_rreg;
2051 	adev->reg.uvd_ctx.wreg = &si_uvd_ctx_wreg;
2052 
2053 	adev->asic_funcs = &si_asic_funcs;
2054 
2055 	adev->rev_id = si_get_rev_id(adev);
2056 	adev->external_rev_id = 0xFF;
2057 	switch (adev->asic_type) {
2058 	case CHIP_TAHITI:
2059 		adev->cg_flags =
2060 			AMD_CG_SUPPORT_GFX_MGCG |
2061 			AMD_CG_SUPPORT_GFX_MGLS |
2062 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2063 			AMD_CG_SUPPORT_GFX_CGLS |
2064 			AMD_CG_SUPPORT_GFX_CGTS |
2065 			AMD_CG_SUPPORT_GFX_CP_LS |
2066 			AMD_CG_SUPPORT_MC_MGCG |
2067 			AMD_CG_SUPPORT_SDMA_MGCG |
2068 			AMD_CG_SUPPORT_BIF_LS |
2069 			AMD_CG_SUPPORT_VCE_MGCG |
2070 			AMD_CG_SUPPORT_UVD_MGCG |
2071 			AMD_CG_SUPPORT_HDP_LS |
2072 			AMD_CG_SUPPORT_HDP_MGCG;
2073 		adev->pg_flags = 0;
2074 		adev->external_rev_id = (adev->rev_id == 0) ? 1 :
2075 					(adev->rev_id == 1) ? 5 : 6;
2076 		break;
2077 	case CHIP_PITCAIRN:
2078 		adev->cg_flags =
2079 			AMD_CG_SUPPORT_GFX_MGCG |
2080 			AMD_CG_SUPPORT_GFX_MGLS |
2081 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2082 			AMD_CG_SUPPORT_GFX_CGLS |
2083 			AMD_CG_SUPPORT_GFX_CGTS |
2084 			AMD_CG_SUPPORT_GFX_CP_LS |
2085 			AMD_CG_SUPPORT_GFX_RLC_LS |
2086 			AMD_CG_SUPPORT_MC_LS |
2087 			AMD_CG_SUPPORT_MC_MGCG |
2088 			AMD_CG_SUPPORT_SDMA_MGCG |
2089 			AMD_CG_SUPPORT_BIF_LS |
2090 			AMD_CG_SUPPORT_VCE_MGCG |
2091 			AMD_CG_SUPPORT_UVD_MGCG |
2092 			AMD_CG_SUPPORT_HDP_LS |
2093 			AMD_CG_SUPPORT_HDP_MGCG;
2094 		adev->pg_flags = 0;
2095 		adev->external_rev_id = adev->rev_id + 20;
2096 		break;
2097 
2098 	case CHIP_VERDE:
2099 		adev->cg_flags =
2100 			AMD_CG_SUPPORT_GFX_MGCG |
2101 			AMD_CG_SUPPORT_GFX_MGLS |
2102 			AMD_CG_SUPPORT_GFX_CGLS |
2103 			AMD_CG_SUPPORT_GFX_CGTS |
2104 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2105 			AMD_CG_SUPPORT_GFX_CP_LS |
2106 			AMD_CG_SUPPORT_MC_LS |
2107 			AMD_CG_SUPPORT_MC_MGCG |
2108 			AMD_CG_SUPPORT_SDMA_MGCG |
2109 			AMD_CG_SUPPORT_SDMA_LS |
2110 			AMD_CG_SUPPORT_BIF_LS |
2111 			AMD_CG_SUPPORT_VCE_MGCG |
2112 			AMD_CG_SUPPORT_UVD_MGCG |
2113 			AMD_CG_SUPPORT_HDP_LS |
2114 			AMD_CG_SUPPORT_HDP_MGCG;
2115 		adev->pg_flags = 0;
2116 		//???
2117 		adev->external_rev_id = adev->rev_id + 40;
2118 		break;
2119 	case CHIP_OLAND:
2120 		adev->cg_flags =
2121 			AMD_CG_SUPPORT_GFX_MGCG |
2122 			AMD_CG_SUPPORT_GFX_MGLS |
2123 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2124 			AMD_CG_SUPPORT_GFX_CGLS |
2125 			AMD_CG_SUPPORT_GFX_CGTS |
2126 			AMD_CG_SUPPORT_GFX_CP_LS |
2127 			AMD_CG_SUPPORT_GFX_RLC_LS |
2128 			AMD_CG_SUPPORT_MC_LS |
2129 			AMD_CG_SUPPORT_MC_MGCG |
2130 			AMD_CG_SUPPORT_SDMA_MGCG |
2131 			AMD_CG_SUPPORT_BIF_LS |
2132 			AMD_CG_SUPPORT_UVD_MGCG |
2133 			AMD_CG_SUPPORT_HDP_LS |
2134 			AMD_CG_SUPPORT_HDP_MGCG;
2135 		adev->pg_flags = 0;
2136 		adev->external_rev_id = 60;
2137 		break;
2138 	case CHIP_HAINAN:
2139 		adev->cg_flags =
2140 			AMD_CG_SUPPORT_GFX_MGCG |
2141 			AMD_CG_SUPPORT_GFX_MGLS |
2142 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2143 			AMD_CG_SUPPORT_GFX_CGLS |
2144 			AMD_CG_SUPPORT_GFX_CGTS |
2145 			AMD_CG_SUPPORT_GFX_CP_LS |
2146 			AMD_CG_SUPPORT_GFX_RLC_LS |
2147 			AMD_CG_SUPPORT_MC_LS |
2148 			AMD_CG_SUPPORT_MC_MGCG |
2149 			AMD_CG_SUPPORT_SDMA_MGCG |
2150 			AMD_CG_SUPPORT_BIF_LS |
2151 			AMD_CG_SUPPORT_HDP_LS |
2152 			AMD_CG_SUPPORT_HDP_MGCG;
2153 		adev->pg_flags = 0;
2154 		adev->external_rev_id = 70;
2155 		break;
2156 
2157 	default:
2158 		return -EINVAL;
2159 	}
2160 
2161 	return 0;
2162 }
2163 
2164 static void si_init_golden_registers(struct amdgpu_device *adev)
2165 {
2166 	switch (adev->asic_type) {
2167 	case CHIP_TAHITI:
2168 		amdgpu_device_program_register_sequence(adev,
2169 							tahiti_golden_registers,
2170 							ARRAY_SIZE(tahiti_golden_registers));
2171 		amdgpu_device_program_register_sequence(adev,
2172 							tahiti_golden_rlc_registers,
2173 							ARRAY_SIZE(tahiti_golden_rlc_registers));
2174 		amdgpu_device_program_register_sequence(adev,
2175 							tahiti_mgcg_cgcg_init,
2176 							ARRAY_SIZE(tahiti_mgcg_cgcg_init));
2177 		amdgpu_device_program_register_sequence(adev,
2178 							tahiti_golden_registers2,
2179 							ARRAY_SIZE(tahiti_golden_registers2));
2180 		break;
2181 	case CHIP_PITCAIRN:
2182 		amdgpu_device_program_register_sequence(adev,
2183 							pitcairn_golden_registers,
2184 							ARRAY_SIZE(pitcairn_golden_registers));
2185 		amdgpu_device_program_register_sequence(adev,
2186 							pitcairn_golden_rlc_registers,
2187 							ARRAY_SIZE(pitcairn_golden_rlc_registers));
2188 		amdgpu_device_program_register_sequence(adev,
2189 							pitcairn_mgcg_cgcg_init,
2190 							ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
2191 		break;
2192 	case CHIP_VERDE:
2193 		amdgpu_device_program_register_sequence(adev,
2194 							verde_golden_registers,
2195 							ARRAY_SIZE(verde_golden_registers));
2196 		amdgpu_device_program_register_sequence(adev,
2197 							verde_golden_rlc_registers,
2198 							ARRAY_SIZE(verde_golden_rlc_registers));
2199 		amdgpu_device_program_register_sequence(adev,
2200 							verde_mgcg_cgcg_init,
2201 							ARRAY_SIZE(verde_mgcg_cgcg_init));
2202 		amdgpu_device_program_register_sequence(adev,
2203 							verde_pg_init,
2204 							ARRAY_SIZE(verde_pg_init));
2205 		break;
2206 	case CHIP_OLAND:
2207 		amdgpu_device_program_register_sequence(adev,
2208 							oland_golden_registers,
2209 							ARRAY_SIZE(oland_golden_registers));
2210 		amdgpu_device_program_register_sequence(adev,
2211 							oland_golden_rlc_registers,
2212 							ARRAY_SIZE(oland_golden_rlc_registers));
2213 		amdgpu_device_program_register_sequence(adev,
2214 							oland_mgcg_cgcg_init,
2215 							ARRAY_SIZE(oland_mgcg_cgcg_init));
2216 		break;
2217 	case CHIP_HAINAN:
2218 		amdgpu_device_program_register_sequence(adev,
2219 							hainan_golden_registers,
2220 							ARRAY_SIZE(hainan_golden_registers));
2221 		amdgpu_device_program_register_sequence(adev,
2222 							hainan_golden_registers2,
2223 							ARRAY_SIZE(hainan_golden_registers2));
2224 		amdgpu_device_program_register_sequence(adev,
2225 							hainan_mgcg_cgcg_init,
2226 							ARRAY_SIZE(hainan_mgcg_cgcg_init));
2227 		break;
2228 
2229 
2230 	default:
2231 		break;
2232 	}
2233 }
2234 
2235 static void si_pcie_gen3_enable(struct amdgpu_device *adev)
2236 {
2237 	struct pci_dev *root = adev->pdev->bus->self;
2238 	u32 speed_cntl, current_data_rate;
2239 	int i;
2240 	u16 tmp16;
2241 
2242 	if (pci_is_root_bus(adev->pdev->bus))
2243 		return;
2244 
2245 	if (amdgpu_pcie_gen2 == 0)
2246 		return;
2247 
2248 	if (adev->flags & AMD_IS_APU)
2249 		return;
2250 
2251 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
2252 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
2253 		return;
2254 
2255 	speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2256 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
2257 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
2258 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2259 		if (current_data_rate == 2) {
2260 			drm_info(adev_to_drm(adev), "PCIE gen 3 link speeds already enabled\n");
2261 			return;
2262 		}
2263 		drm_info(adev_to_drm(adev), "enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
2264 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
2265 		if (current_data_rate == 1) {
2266 			drm_info(adev_to_drm(adev), "PCIE gen 2 link speeds already enabled\n");
2267 			return;
2268 		}
2269 		drm_info(adev_to_drm(adev), "enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
2270 	}
2271 
2272 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
2273 		return;
2274 
2275 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
2276 		if (current_data_rate != 2) {
2277 			u16 bridge_cfg, gpu_cfg;
2278 			u16 bridge_cfg2, gpu_cfg2;
2279 			u32 max_lw, current_lw, tmp;
2280 
2281 			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
2282 			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
2283 
2284 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
2285 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >> PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
2286 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK) >> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
2287 
2288 			if (current_lw < max_lw) {
2289 				tmp = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
2290 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
2291 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
2292 					tmp |= (max_lw << PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
2293 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK | PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
2294 					WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
2295 				}
2296 			}
2297 
2298 			for (i = 0; i < 10; i++) {
2299 				pcie_capability_read_word(adev->pdev,
2300 							  PCI_EXP_DEVSTA,
2301 							  &tmp16);
2302 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
2303 					break;
2304 
2305 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
2306 							  &bridge_cfg);
2307 				pcie_capability_read_word(adev->pdev,
2308 							  PCI_EXP_LNKCTL,
2309 							  &gpu_cfg);
2310 
2311 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
2312 							  &bridge_cfg2);
2313 				pcie_capability_read_word(adev->pdev,
2314 							  PCI_EXP_LNKCTL2,
2315 							  &gpu_cfg2);
2316 
2317 				tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2318 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
2319 				WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2320 
2321 				tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2322 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
2323 				WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2324 
2325 				mdelay(100);
2326 
2327 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
2328 								   PCI_EXP_LNKCTL_HAWD,
2329 								   bridge_cfg &
2330 								   PCI_EXP_LNKCTL_HAWD);
2331 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
2332 								   PCI_EXP_LNKCTL_HAWD,
2333 								   gpu_cfg &
2334 								   PCI_EXP_LNKCTL_HAWD);
2335 
2336 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
2337 								   PCI_EXP_LNKCTL2_ENTER_COMP |
2338 								   PCI_EXP_LNKCTL2_TX_MARGIN,
2339 								   bridge_cfg2 &
2340 								   (PCI_EXP_LNKCTL2_ENTER_COMP |
2341 								    PCI_EXP_LNKCTL2_TX_MARGIN));
2342 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
2343 								   PCI_EXP_LNKCTL2_ENTER_COMP |
2344 								   PCI_EXP_LNKCTL2_TX_MARGIN,
2345 								   gpu_cfg2 &
2346 								   (PCI_EXP_LNKCTL2_ENTER_COMP |
2347 								    PCI_EXP_LNKCTL2_TX_MARGIN));
2348 
2349 				tmp = RREG32_PCIE_PORT(ixPCIE_LC_CNTL4);
2350 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
2351 				WREG32_PCIE_PORT(ixPCIE_LC_CNTL4, tmp);
2352 			}
2353 		}
2354 	}
2355 
2356 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK | PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
2357 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
2358 	WREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL, speed_cntl);
2359 
2360 	tmp16 = 0;
2361 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
2362 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
2363 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
2364 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
2365 	else
2366 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
2367 	pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL2,
2368 					   PCI_EXP_LNKCTL2_TLS, tmp16);
2369 
2370 	speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2371 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
2372 	WREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL, speed_cntl);
2373 
2374 	for (i = 0; i < adev->usec_timeout; i++) {
2375 		speed_cntl = RREG32_PCIE_PORT(ixPCIE_LC_SPEED_CNTL);
2376 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
2377 			break;
2378 		udelay(1);
2379 	}
2380 }
2381 
2382 static inline u32 si_pif_phy0_rreg(struct amdgpu_device *adev, u32 reg)
2383 {
2384 	unsigned long flags;
2385 	u32 r;
2386 
2387 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
2388 	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2389 	r = RREG32(EVERGREEN_PIF_PHY0_DATA);
2390 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
2391 	return r;
2392 }
2393 
2394 static inline void si_pif_phy0_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2395 {
2396 	unsigned long flags;
2397 
2398 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
2399 	WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
2400 	WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
2401 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
2402 }
2403 
2404 static inline u32 si_pif_phy1_rreg(struct amdgpu_device *adev, u32 reg)
2405 {
2406 	unsigned long flags;
2407 	u32 r;
2408 
2409 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
2410 	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2411 	r = RREG32(EVERGREEN_PIF_PHY1_DATA);
2412 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
2413 	return r;
2414 }
2415 
2416 static inline void si_pif_phy1_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
2417 {
2418 	unsigned long flags;
2419 
2420 	spin_lock_irqsave(&adev->reg.pcie.lock, flags);
2421 	WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
2422 	WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
2423 	spin_unlock_irqrestore(&adev->reg.pcie.lock, flags);
2424 }
2425 static void si_program_aspm(struct amdgpu_device *adev)
2426 {
2427 	u32 data, orig;
2428 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
2429 	bool disable_clkreq = false;
2430 
2431 	if (!amdgpu_device_should_use_aspm(adev))
2432 		return;
2433 
2434 	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL);
2435 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
2436 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) | PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
2437 	if (orig != data)
2438 		WREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL, data);
2439 
2440 	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL3);
2441 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
2442 	if (orig != data)
2443 		WREG32_PCIE_PORT(ixPCIE_LC_CNTL3, data);
2444 
2445 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
2446 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
2447 	if (orig != data)
2448 		WREG32_PCIE(ixPCIE_P_CNTL, data);
2449 
2450 	orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL);
2451 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK | PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
2452 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
2453 	if (!disable_l0s)
2454 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
2455 
2456 	if (!disable_l1) {
2457 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
2458 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
2459 		if (orig != data)
2460 			WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2461 
2462 		if (!disable_plloff_in_l1) {
2463 			bool clk_req_support;
2464 
2465 			orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_0);
2466 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
2467 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
2468 			if (orig != data)
2469 				si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_0, data);
2470 
2471 			orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_1);
2472 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
2473 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
2474 			if (orig != data)
2475 				si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_1, data);
2476 
2477 			orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_0);
2478 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK | PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
2479 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) | (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
2480 			if (orig != data)
2481 				si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_0, data);
2482 
2483 			orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_1);
2484 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK | PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
2485 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) | (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
2486 			if (orig != data)
2487 				si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_1, data);
2488 
2489 			if ((adev->asic_type != CHIP_OLAND) && (adev->asic_type != CHIP_HAINAN)) {
2490 				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_0);
2491 				data &= ~PB0_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK;
2492 				if (orig != data)
2493 					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_0, data);
2494 
2495 				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_1);
2496 				data &= ~PB0_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK;
2497 				if (orig != data)
2498 					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_1, data);
2499 
2500 				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_2);
2501 				data &= ~PB0_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK;
2502 				if (orig != data)
2503 					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_2, data);
2504 
2505 				orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_PWRDOWN_3);
2506 				data &= ~PB0_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK;
2507 				if (orig != data)
2508 					si_pif_phy0_wreg(adev,ixPB0_PIF_PWRDOWN_3, data);
2509 
2510 				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_0);
2511 				data &= ~PB1_PIF_PWRDOWN_0__PLL_RAMP_UP_TIME_0_MASK;
2512 				if (orig != data)
2513 					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_0, data);
2514 
2515 				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_1);
2516 				data &= ~PB1_PIF_PWRDOWN_1__PLL_RAMP_UP_TIME_1_MASK;
2517 				if (orig != data)
2518 					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_1, data);
2519 
2520 				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_2);
2521 				data &= ~PB1_PIF_PWRDOWN_2__PLL_RAMP_UP_TIME_2_MASK;
2522 				if (orig != data)
2523 					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_2, data);
2524 
2525 				orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_PWRDOWN_3);
2526 				data &= ~PB1_PIF_PWRDOWN_3__PLL_RAMP_UP_TIME_3_MASK;
2527 				if (orig != data)
2528 					si_pif_phy1_wreg(adev,ixPB1_PIF_PWRDOWN_3, data);
2529 			}
2530 			orig = data = RREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL);
2531 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
2532 			data |= (3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
2533 			if (orig != data)
2534 				WREG32_PCIE_PORT(ixPCIE_LC_LINK_WIDTH_CNTL, data);
2535 
2536 			orig = data = si_pif_phy0_rreg(adev,ixPB0_PIF_CNTL);
2537 			data &= ~PB0_PIF_CNTL__LS2_EXIT_TIME_MASK;
2538 			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2539 				data |= (5 << PB0_PIF_CNTL__LS2_EXIT_TIME__SHIFT);
2540 			if (orig != data)
2541 				si_pif_phy0_wreg(adev,ixPB0_PIF_CNTL, data);
2542 
2543 			orig = data = si_pif_phy1_rreg(adev,ixPB1_PIF_CNTL);
2544 			data &= ~PB1_PIF_CNTL__LS2_EXIT_TIME_MASK;
2545 			if ((adev->asic_type == CHIP_OLAND) || (adev->asic_type == CHIP_HAINAN))
2546 				data |= (5 << PB1_PIF_CNTL__LS2_EXIT_TIME__SHIFT);
2547 			if (orig != data)
2548 				si_pif_phy1_wreg(adev,ixPB1_PIF_CNTL, data);
2549 
2550 			if (!disable_clkreq &&
2551 			    !pci_is_root_bus(adev->pdev->bus)) {
2552 				struct pci_dev *root = adev->pdev->bus->self;
2553 				u32 lnkcap;
2554 
2555 				clk_req_support = false;
2556 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
2557 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
2558 					clk_req_support = true;
2559 			} else {
2560 				clk_req_support = false;
2561 			}
2562 
2563 			if (clk_req_support) {
2564 				orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL2);
2565 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK | PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
2566 				if (orig != data)
2567 					WREG32_PCIE_PORT(ixPCIE_LC_CNTL2, data);
2568 
2569 				orig = data = RREG32(mmTHM_CLK_CNTL);
2570 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK | THM_CLK_CNTL__TMON_CLK_SEL_MASK);
2571 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) | (1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
2572 				if (orig != data)
2573 					WREG32(mmTHM_CLK_CNTL, data);
2574 
2575 				orig = data = RREG32(mmMISC_CLK_CNTL);
2576 				data &= ~(MISC_CLK_CNTL__DEEP_SLEEP_CLK_SEL_MASK | MISC_CLK_CNTL__ZCLK_SEL_MASK);
2577 				data |= (1 << MISC_CLK_CNTL__DEEP_SLEEP_CLK_SEL__SHIFT) | (1 << MISC_CLK_CNTL__ZCLK_SEL__SHIFT);
2578 				if (orig != data)
2579 					WREG32(mmMISC_CLK_CNTL, data);
2580 
2581 				orig = data = RREG32(mmCG_CLKPIN_CNTL);
2582 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
2583 				if (orig != data)
2584 					WREG32(mmCG_CLKPIN_CNTL, data);
2585 
2586 				orig = data = RREG32(mmCG_CLKPIN_CNTL_2);
2587 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
2588 				if (orig != data)
2589 					WREG32(mmCG_CLKPIN_CNTL_2, data);
2590 
2591 				orig = data = RREG32(mmMPLL_BYPASSCLK_SEL);
2592 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
2593 				data |= 4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT;
2594 				if (orig != data)
2595 					WREG32(mmMPLL_BYPASSCLK_SEL, data);
2596 
2597 				orig = data = RREG32(mmSPLL_CNTL_MODE);
2598 				data &= ~SPLL_CNTL_MODE__SPLL_REFCLK_SEL_MASK;
2599 				if (orig != data)
2600 					WREG32(mmSPLL_CNTL_MODE, data);
2601 			}
2602 		}
2603 	} else {
2604 		if (orig != data)
2605 			WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2606 	}
2607 
2608 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
2609 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK | PCIE_CNTL2__MST_MEM_LS_EN_MASK | PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
2610 	if (orig != data)
2611 		WREG32_PCIE(ixPCIE_CNTL2, data);
2612 
2613 	if (!disable_l0s) {
2614 		data = RREG32_PCIE_PORT(ixPCIE_LC_N_FTS_CNTL);
2615 		if((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) == PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
2616 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
2617 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) && (data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
2618 				orig = data = RREG32_PCIE_PORT(ixPCIE_LC_CNTL);
2619 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
2620 				if (orig != data)
2621 					WREG32_PCIE_PORT(ixPCIE_LC_CNTL, data);
2622 			}
2623 		}
2624 	}
2625 }
2626 
2627 static void si_fix_pci_max_read_req_size(struct amdgpu_device *adev)
2628 {
2629 	int readrq;
2630 	u16 v;
2631 
2632 	readrq = pcie_get_readrq(adev->pdev);
2633 	v = ffs(readrq) - 8;
2634 	if ((v == 0) || (v == 6) || (v == 7))
2635 		pcie_set_readrq(adev->pdev, 512);
2636 }
2637 
2638 static int si_common_hw_init(struct amdgpu_ip_block *ip_block)
2639 {
2640 	struct amdgpu_device *adev = ip_block->adev;
2641 
2642 	si_fix_pci_max_read_req_size(adev);
2643 	si_init_golden_registers(adev);
2644 	si_pcie_gen3_enable(adev);
2645 	si_program_aspm(adev);
2646 
2647 	return 0;
2648 }
2649 
2650 static int si_common_hw_fini(struct amdgpu_ip_block *ip_block)
2651 {
2652 	return 0;
2653 }
2654 
2655 static int si_common_resume(struct amdgpu_ip_block *ip_block)
2656 {
2657 	return si_common_hw_init(ip_block);
2658 }
2659 
2660 static int si_common_set_clockgating_state(struct amdgpu_ip_block *ip_block,
2661 					    enum amd_clockgating_state state)
2662 {
2663 	return 0;
2664 }
2665 
2666 static int si_common_set_powergating_state(struct amdgpu_ip_block *ip_block,
2667 					    enum amd_powergating_state state)
2668 {
2669 	return 0;
2670 }
2671 
2672 static const struct amd_ip_funcs si_common_ip_funcs = {
2673 	.name = "si_common",
2674 	.early_init = si_common_early_init,
2675 	.hw_init = si_common_hw_init,
2676 	.hw_fini = si_common_hw_fini,
2677 	.resume = si_common_resume,
2678 	.set_clockgating_state = si_common_set_clockgating_state,
2679 	.set_powergating_state = si_common_set_powergating_state,
2680 };
2681 
2682 static const struct amdgpu_ip_block_version si_common_ip_block =
2683 {
2684 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2685 	.major = 1,
2686 	.minor = 0,
2687 	.rev = 0,
2688 	.funcs = &si_common_ip_funcs,
2689 };
2690 
2691 int si_set_ip_blocks(struct amdgpu_device *adev)
2692 {
2693 	switch (adev->asic_type) {
2694 	case CHIP_VERDE:
2695 	case CHIP_TAHITI:
2696 	case CHIP_PITCAIRN:
2697 		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2698 		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2699 		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2700 		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2701 		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2702 		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2703 		if (adev->enable_virtual_display)
2704 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2705 #if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2706 		else if (amdgpu_device_has_dc_support(adev))
2707 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2708 #endif
2709 		else
2710 			amdgpu_device_ip_block_add(adev, &dce_v6_0_ip_block);
2711 		amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2712 		amdgpu_device_ip_block_add(adev, &vce_v1_0_ip_block);
2713 		break;
2714 	case CHIP_OLAND:
2715 		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2716 		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2717 		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2718 		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2719 		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2720 		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2721 		if (adev->enable_virtual_display)
2722 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2723 #if defined(CONFIG_DRM_AMD_DC) && defined(CONFIG_DRM_AMD_DC_SI)
2724 		else if (amdgpu_device_has_dc_support(adev))
2725 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2726 #endif
2727 		else
2728 			amdgpu_device_ip_block_add(adev, &dce_v6_4_ip_block);
2729 		amdgpu_device_ip_block_add(adev, &uvd_v3_1_ip_block);
2730 		break;
2731 	case CHIP_HAINAN:
2732 		amdgpu_device_ip_block_add(adev, &si_common_ip_block);
2733 		amdgpu_device_ip_block_add(adev, &gmc_v6_0_ip_block);
2734 		amdgpu_device_ip_block_add(adev, &si_ih_ip_block);
2735 		amdgpu_device_ip_block_add(adev, &gfx_v6_0_ip_block);
2736 		amdgpu_device_ip_block_add(adev, &si_dma_ip_block);
2737 		amdgpu_device_ip_block_add(adev, &si_smu_ip_block);
2738 		if (adev->enable_virtual_display)
2739 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2740 		break;
2741 	default:
2742 		break;
2743 	}
2744 	return 0;
2745 }
2746 
2747