xref: /linux/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c (revision 4ab5a5d2a4a2289c2af07accbec7170ca5671f41)
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/kernel.h>
24 #include <linux/firmware.h>
25 #include <drm/drmP.h>
26 #include "amdgpu.h"
27 #include "amdgpu_gfx.h"
28 #include "vi.h"
29 #include "vi_structs.h"
30 #include "vid.h"
31 #include "amdgpu_ucode.h"
32 #include "amdgpu_atombios.h"
33 #include "atombios_i2c.h"
34 #include "clearstate_vi.h"
35 
36 #include "gmc/gmc_8_2_d.h"
37 #include "gmc/gmc_8_2_sh_mask.h"
38 
39 #include "oss/oss_3_0_d.h"
40 #include "oss/oss_3_0_sh_mask.h"
41 
42 #include "bif/bif_5_0_d.h"
43 #include "bif/bif_5_0_sh_mask.h"
44 #include "gca/gfx_8_0_d.h"
45 #include "gca/gfx_8_0_enum.h"
46 #include "gca/gfx_8_0_sh_mask.h"
47 #include "gca/gfx_8_0_enum.h"
48 
49 #include "dce/dce_10_0_d.h"
50 #include "dce/dce_10_0_sh_mask.h"
51 
52 #include "smu/smu_7_1_3_d.h"
53 
54 #include "ivsrcid/ivsrcid_vislands30.h"
55 
56 #define GFX8_NUM_GFX_RINGS     1
57 #define GFX8_MEC_HPD_SIZE 2048
58 
59 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001
60 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001
61 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002
62 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003
63 
64 #define ARRAY_MODE(x)					((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
65 #define PIPE_CONFIG(x)					((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
66 #define TILE_SPLIT(x)					((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
67 #define MICRO_TILE_MODE_NEW(x)				((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT)
68 #define SAMPLE_SPLIT(x)					((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
69 #define BANK_WIDTH(x)					((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT)
70 #define BANK_HEIGHT(x)					((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT)
71 #define MACRO_TILE_ASPECT(x)				((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)
72 #define NUM_BANKS(x)					((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)
73 
74 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK            0x00000001L
75 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK            0x00000002L
76 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK           0x00000004L
77 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK           0x00000008L
78 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK           0x00000010L
79 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK           0x00000020L
80 
81 /* BPM SERDES CMD */
82 #define SET_BPM_SERDES_CMD    1
83 #define CLE_BPM_SERDES_CMD    0
84 
85 /* BPM Register Address*/
86 enum {
87 	BPM_REG_CGLS_EN = 0,        /* Enable/Disable CGLS */
88 	BPM_REG_CGLS_ON,            /* ON/OFF CGLS: shall be controlled by RLC FW */
89 	BPM_REG_CGCG_OVERRIDE,      /* Set/Clear CGCG Override */
90 	BPM_REG_MGCG_OVERRIDE,      /* Set/Clear MGCG Override */
91 	BPM_REG_FGCG_OVERRIDE,      /* Set/Clear FGCG Override */
92 	BPM_REG_FGCG_MAX
93 };
94 
95 #define RLC_FormatDirectRegListLength        14
96 
97 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");
98 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");
99 MODULE_FIRMWARE("amdgpu/carrizo_me.bin");
100 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin");
101 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");
102 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");
103 
104 MODULE_FIRMWARE("amdgpu/stoney_ce.bin");
105 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");
106 MODULE_FIRMWARE("amdgpu/stoney_me.bin");
107 MODULE_FIRMWARE("amdgpu/stoney_mec.bin");
108 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");
109 
110 MODULE_FIRMWARE("amdgpu/tonga_ce.bin");
111 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");
112 MODULE_FIRMWARE("amdgpu/tonga_me.bin");
113 MODULE_FIRMWARE("amdgpu/tonga_mec.bin");
114 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin");
115 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin");
116 
117 MODULE_FIRMWARE("amdgpu/topaz_ce.bin");
118 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin");
119 MODULE_FIRMWARE("amdgpu/topaz_me.bin");
120 MODULE_FIRMWARE("amdgpu/topaz_mec.bin");
121 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");
122 
123 MODULE_FIRMWARE("amdgpu/fiji_ce.bin");
124 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");
125 MODULE_FIRMWARE("amdgpu/fiji_me.bin");
126 MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
127 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
128 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
129 
130 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
131 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
132 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
133 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");
134 MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
135 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");
136 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
137 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");
138 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
139 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
140 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
141 
142 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
143 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
144 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
145 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
146 MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
147 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
148 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
149 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
150 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
151 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
152 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
153 
154 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
155 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
156 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
157 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");
158 MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
159 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");
160 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
161 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");
162 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
163 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
164 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
165 
166 MODULE_FIRMWARE("amdgpu/vegam_ce.bin");
167 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin");
168 MODULE_FIRMWARE("amdgpu/vegam_me.bin");
169 MODULE_FIRMWARE("amdgpu/vegam_mec.bin");
170 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin");
171 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin");
172 
173 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
174 {
175 	{mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
176 	{mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
177 	{mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
178 	{mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
179 	{mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
180 	{mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
181 	{mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
182 	{mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
183 	{mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
184 	{mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
185 	{mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
186 	{mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
187 	{mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
188 	{mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
189 	{mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
190 	{mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
191 };
192 
193 static const u32 golden_settings_tonga_a11[] =
194 {
195 	mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
196 	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
197 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
198 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
199 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
200 	mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
201 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
202 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
203 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
204 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
205 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
206 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
207 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
208 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
209 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
210 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
211 };
212 
213 static const u32 tonga_golden_common_all[] =
214 {
215 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
216 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
217 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
218 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
219 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
220 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
221 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
222 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
223 };
224 
225 static const u32 tonga_mgcg_cgcg_init[] =
226 {
227 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
228 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
229 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
230 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
231 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
232 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
233 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
234 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
235 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
236 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
237 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
238 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
239 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
240 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
241 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
242 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
243 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
244 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
245 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
246 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
247 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
248 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
249 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
250 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
251 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
252 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
253 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
254 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
255 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
256 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
257 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
258 	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
259 	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
260 	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
261 	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
262 	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
263 	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
264 	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
265 	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
266 	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
267 	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
268 	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
269 	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
270 	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
271 	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
272 	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
273 	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
274 	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
275 	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
276 	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
277 	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
278 	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
279 	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
280 	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
281 	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
282 	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
283 	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
284 	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
285 	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
286 	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
287 	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
288 	mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
289 	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
290 	mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
291 	mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
292 	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
293 	mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
294 	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
295 	mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
296 	mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
297 	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
298 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
299 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
300 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
301 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
302 };
303 
304 static const u32 golden_settings_vegam_a11[] =
305 {
306 	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
307 	mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000,
308 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
309 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
310 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
311 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
312 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a,
313 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e,
314 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
315 	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
316 	mmSQ_CONFIG, 0x07f80000, 0x01180000,
317 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
318 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
319 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
320 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
321 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
322 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
323 };
324 
325 static const u32 vegam_golden_common_all[] =
326 {
327 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
328 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
329 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
330 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
331 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
332 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
333 };
334 
335 static const u32 golden_settings_polaris11_a11[] =
336 {
337 	mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
338 	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
339 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
340 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
341 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
342 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
343 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
344 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
345 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
346 	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
347 	mmSQ_CONFIG, 0x07f80000, 0x01180000,
348 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
349 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
350 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
351 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
352 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
353 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
354 };
355 
356 static const u32 polaris11_golden_common_all[] =
357 {
358 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
359 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,
360 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
361 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
362 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
363 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
364 };
365 
366 static const u32 golden_settings_polaris10_a11[] =
367 {
368 	mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,
369 	mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
370 	mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
371 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
372 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
373 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
374 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
375 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
376 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a,
377 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
378 	mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
379 	mmSQ_CONFIG, 0x07f80000, 0x07180000,
380 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
381 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
382 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
383 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
384 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
385 };
386 
387 static const u32 polaris10_golden_common_all[] =
388 {
389 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
390 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
391 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
392 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
393 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
394 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
395 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
396 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
397 };
398 
399 static const u32 fiji_golden_common_all[] =
400 {
401 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
402 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
403 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
404 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
405 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
406 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
407 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
408 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
409 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
410 	mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
411 };
412 
413 static const u32 golden_settings_fiji_a10[] =
414 {
415 	mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
416 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
417 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
418 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
419 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
420 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
421 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
422 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
423 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
424 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
425 	mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
426 };
427 
428 static const u32 fiji_mgcg_cgcg_init[] =
429 {
430 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
431 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
432 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
433 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
434 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
435 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
436 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
437 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
438 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
439 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
440 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
441 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
442 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
443 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
444 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
445 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
446 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
447 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
448 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
449 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
450 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
451 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
452 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
453 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
454 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
455 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
456 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
457 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
458 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
459 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
460 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
461 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
462 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
463 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
464 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
465 };
466 
467 static const u32 golden_settings_iceland_a11[] =
468 {
469 	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
470 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
471 	mmDB_DEBUG3, 0xc0000000, 0xc0000000,
472 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
473 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
474 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
475 	mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,
476 	mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
477 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
478 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
479 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
480 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
481 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
482 	mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
483 	mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
484 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010,
485 };
486 
487 static const u32 iceland_golden_common_all[] =
488 {
489 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
490 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
491 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
492 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
493 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
494 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
495 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
496 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
497 };
498 
499 static const u32 iceland_mgcg_cgcg_init[] =
500 {
501 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
502 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
503 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
504 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
505 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100,
506 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100,
507 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100,
508 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
509 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
510 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
511 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
512 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
513 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
514 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
515 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
516 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
517 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
518 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
519 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
520 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
521 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
522 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
523 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100,
524 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
525 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
526 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
527 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
528 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
529 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
530 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
531 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
532 	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
533 	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
534 	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
535 	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
536 	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
537 	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
538 	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
539 	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
540 	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
541 	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
542 	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
543 	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
544 	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
545 	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
546 	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
547 	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
548 	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
549 	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
550 	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
551 	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
552 	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
553 	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
554 	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
555 	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
556 	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
557 	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
558 	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
559 	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
560 	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
561 	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
562 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
563 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
564 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
565 };
566 
567 static const u32 cz_golden_settings_a11[] =
568 {
569 	mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
570 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
571 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
572 	mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,
573 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
574 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
575 	mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
576 	mmTA_CNTL_AUX, 0x000f000f, 0x00010000,
577 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
578 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
579 	mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,
580 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302
581 };
582 
583 static const u32 cz_golden_common_all[] =
584 {
585 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
586 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
587 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
588 	mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
589 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
590 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
591 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
592 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
593 };
594 
595 static const u32 cz_mgcg_cgcg_init[] =
596 {
597 	mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
598 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
599 	mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
600 	mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
601 	mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
602 	mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
603 	mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100,
604 	mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
605 	mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
606 	mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
607 	mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
608 	mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
609 	mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
610 	mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
611 	mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
612 	mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
613 	mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
614 	mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
615 	mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
616 	mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
617 	mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
618 	mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
619 	mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
620 	mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
621 	mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
622 	mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
623 	mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
624 	mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
625 	mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
626 	mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
627 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
628 	mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
629 	mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
630 	mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
631 	mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
632 	mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
633 	mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
634 	mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
635 	mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
636 	mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
637 	mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
638 	mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
639 	mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
640 	mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
641 	mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
642 	mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
643 	mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
644 	mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
645 	mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
646 	mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
647 	mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
648 	mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
649 	mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
650 	mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
651 	mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
652 	mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
653 	mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
654 	mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
655 	mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
656 	mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
657 	mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
658 	mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
659 	mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
660 	mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
661 	mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
662 	mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
663 	mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
664 	mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
665 	mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
666 	mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
667 	mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
668 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
669 	mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
670 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
671 	mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
672 };
673 
674 static const u32 stoney_golden_settings_a11[] =
675 {
676 	mmDB_DEBUG2, 0xf00fffff, 0x00000400,
677 	mmGB_GPU_ID, 0x0000000f, 0x00000000,
678 	mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
679 	mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
680 	mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
681 	mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
682 	mmTCC_CTRL, 0x00100000, 0xf31fff7f,
683 	mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
684 	mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,
685 	mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,
686 };
687 
688 static const u32 stoney_golden_common_all[] =
689 {
690 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
691 	mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,
692 	mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
693 	mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
694 	mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
695 	mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
696 	mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
697 	mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
698 };
699 
700 static const u32 stoney_mgcg_cgcg_init[] =
701 {
702 	mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
703 	mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
704 	mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
705 	mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
706 	mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
707 };
708 
709 
710 static const char * const sq_edc_source_names[] = {
711 	"SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred",
712 	"SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch",
713 	"SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return",
714 	"SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR",
715 	"SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS",
716 	"SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS",
717 	"SQ_EDC_INFO_SOURCE_TA: EDC source is TA",
718 };
719 
720 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
721 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
722 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
723 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
724 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
725 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
726 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
727 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);
728 
729 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
730 {
731 	switch (adev->asic_type) {
732 	case CHIP_TOPAZ:
733 		amdgpu_device_program_register_sequence(adev,
734 							iceland_mgcg_cgcg_init,
735 							ARRAY_SIZE(iceland_mgcg_cgcg_init));
736 		amdgpu_device_program_register_sequence(adev,
737 							golden_settings_iceland_a11,
738 							ARRAY_SIZE(golden_settings_iceland_a11));
739 		amdgpu_device_program_register_sequence(adev,
740 							iceland_golden_common_all,
741 							ARRAY_SIZE(iceland_golden_common_all));
742 		break;
743 	case CHIP_FIJI:
744 		amdgpu_device_program_register_sequence(adev,
745 							fiji_mgcg_cgcg_init,
746 							ARRAY_SIZE(fiji_mgcg_cgcg_init));
747 		amdgpu_device_program_register_sequence(adev,
748 							golden_settings_fiji_a10,
749 							ARRAY_SIZE(golden_settings_fiji_a10));
750 		amdgpu_device_program_register_sequence(adev,
751 							fiji_golden_common_all,
752 							ARRAY_SIZE(fiji_golden_common_all));
753 		break;
754 
755 	case CHIP_TONGA:
756 		amdgpu_device_program_register_sequence(adev,
757 							tonga_mgcg_cgcg_init,
758 							ARRAY_SIZE(tonga_mgcg_cgcg_init));
759 		amdgpu_device_program_register_sequence(adev,
760 							golden_settings_tonga_a11,
761 							ARRAY_SIZE(golden_settings_tonga_a11));
762 		amdgpu_device_program_register_sequence(adev,
763 							tonga_golden_common_all,
764 							ARRAY_SIZE(tonga_golden_common_all));
765 		break;
766 	case CHIP_VEGAM:
767 		amdgpu_device_program_register_sequence(adev,
768 							golden_settings_vegam_a11,
769 							ARRAY_SIZE(golden_settings_vegam_a11));
770 		amdgpu_device_program_register_sequence(adev,
771 							vegam_golden_common_all,
772 							ARRAY_SIZE(vegam_golden_common_all));
773 		break;
774 	case CHIP_POLARIS11:
775 	case CHIP_POLARIS12:
776 		amdgpu_device_program_register_sequence(adev,
777 							golden_settings_polaris11_a11,
778 							ARRAY_SIZE(golden_settings_polaris11_a11));
779 		amdgpu_device_program_register_sequence(adev,
780 							polaris11_golden_common_all,
781 							ARRAY_SIZE(polaris11_golden_common_all));
782 		break;
783 	case CHIP_POLARIS10:
784 		amdgpu_device_program_register_sequence(adev,
785 							golden_settings_polaris10_a11,
786 							ARRAY_SIZE(golden_settings_polaris10_a11));
787 		amdgpu_device_program_register_sequence(adev,
788 							polaris10_golden_common_all,
789 							ARRAY_SIZE(polaris10_golden_common_all));
790 		WREG32_SMC(ixCG_ACLK_CNTL, 0x0000001C);
791 		if (adev->pdev->revision == 0xc7 &&
792 		    ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
793 		     (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||
794 		     (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1682))) {
795 			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);
796 			amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);
797 		}
798 		break;
799 	case CHIP_CARRIZO:
800 		amdgpu_device_program_register_sequence(adev,
801 							cz_mgcg_cgcg_init,
802 							ARRAY_SIZE(cz_mgcg_cgcg_init));
803 		amdgpu_device_program_register_sequence(adev,
804 							cz_golden_settings_a11,
805 							ARRAY_SIZE(cz_golden_settings_a11));
806 		amdgpu_device_program_register_sequence(adev,
807 							cz_golden_common_all,
808 							ARRAY_SIZE(cz_golden_common_all));
809 		break;
810 	case CHIP_STONEY:
811 		amdgpu_device_program_register_sequence(adev,
812 							stoney_mgcg_cgcg_init,
813 							ARRAY_SIZE(stoney_mgcg_cgcg_init));
814 		amdgpu_device_program_register_sequence(adev,
815 							stoney_golden_settings_a11,
816 							ARRAY_SIZE(stoney_golden_settings_a11));
817 		amdgpu_device_program_register_sequence(adev,
818 							stoney_golden_common_all,
819 							ARRAY_SIZE(stoney_golden_common_all));
820 		break;
821 	default:
822 		break;
823 	}
824 }
825 
826 static void gfx_v8_0_scratch_init(struct amdgpu_device *adev)
827 {
828 	adev->gfx.scratch.num_reg = 8;
829 	adev->gfx.scratch.reg_base = mmSCRATCH_REG0;
830 	adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
831 }
832 
833 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
834 {
835 	struct amdgpu_device *adev = ring->adev;
836 	uint32_t scratch;
837 	uint32_t tmp = 0;
838 	unsigned i;
839 	int r;
840 
841 	r = amdgpu_gfx_scratch_get(adev, &scratch);
842 	if (r) {
843 		DRM_ERROR("amdgpu: cp failed to get scratch reg (%d).\n", r);
844 		return r;
845 	}
846 	WREG32(scratch, 0xCAFEDEAD);
847 	r = amdgpu_ring_alloc(ring, 3);
848 	if (r) {
849 		DRM_ERROR("amdgpu: cp failed to lock ring %d (%d).\n",
850 			  ring->idx, r);
851 		amdgpu_gfx_scratch_free(adev, scratch);
852 		return r;
853 	}
854 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
855 	amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
856 	amdgpu_ring_write(ring, 0xDEADBEEF);
857 	amdgpu_ring_commit(ring);
858 
859 	for (i = 0; i < adev->usec_timeout; i++) {
860 		tmp = RREG32(scratch);
861 		if (tmp == 0xDEADBEEF)
862 			break;
863 		DRM_UDELAY(1);
864 	}
865 	if (i < adev->usec_timeout) {
866 		DRM_DEBUG("ring test on %d succeeded in %d usecs\n",
867 			 ring->idx, i);
868 	} else {
869 		DRM_ERROR("amdgpu: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
870 			  ring->idx, scratch, tmp);
871 		r = -EINVAL;
872 	}
873 	amdgpu_gfx_scratch_free(adev, scratch);
874 	return r;
875 }
876 
877 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
878 {
879 	struct amdgpu_device *adev = ring->adev;
880 	struct amdgpu_ib ib;
881 	struct dma_fence *f = NULL;
882 
883 	unsigned int index;
884 	uint64_t gpu_addr;
885 	uint32_t tmp;
886 	long r;
887 
888 	r = amdgpu_device_wb_get(adev, &index);
889 	if (r) {
890 		dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
891 		return r;
892 	}
893 
894 	gpu_addr = adev->wb.gpu_addr + (index * 4);
895 	adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
896 	memset(&ib, 0, sizeof(ib));
897 	r = amdgpu_ib_get(adev, NULL, 16, &ib);
898 	if (r) {
899 		DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
900 		goto err1;
901 	}
902 	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
903 	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
904 	ib.ptr[2] = lower_32_bits(gpu_addr);
905 	ib.ptr[3] = upper_32_bits(gpu_addr);
906 	ib.ptr[4] = 0xDEADBEEF;
907 	ib.length_dw = 5;
908 
909 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
910 	if (r)
911 		goto err2;
912 
913 	r = dma_fence_wait_timeout(f, false, timeout);
914 	if (r == 0) {
915 		DRM_ERROR("amdgpu: IB test timed out.\n");
916 		r = -ETIMEDOUT;
917 		goto err2;
918 	} else if (r < 0) {
919 		DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
920 		goto err2;
921 	}
922 
923 	tmp = adev->wb.wb[index];
924 	if (tmp == 0xDEADBEEF) {
925 		DRM_DEBUG("ib test on ring %d succeeded\n", ring->idx);
926 		r = 0;
927 	} else {
928 		DRM_ERROR("ib test on ring %d failed\n", ring->idx);
929 		r = -EINVAL;
930 	}
931 
932 err2:
933 	amdgpu_ib_free(adev, &ib, NULL);
934 	dma_fence_put(f);
935 err1:
936 	amdgpu_device_wb_free(adev, index);
937 	return r;
938 }
939 
940 
941 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev)
942 {
943 	release_firmware(adev->gfx.pfp_fw);
944 	adev->gfx.pfp_fw = NULL;
945 	release_firmware(adev->gfx.me_fw);
946 	adev->gfx.me_fw = NULL;
947 	release_firmware(adev->gfx.ce_fw);
948 	adev->gfx.ce_fw = NULL;
949 	release_firmware(adev->gfx.rlc_fw);
950 	adev->gfx.rlc_fw = NULL;
951 	release_firmware(adev->gfx.mec_fw);
952 	adev->gfx.mec_fw = NULL;
953 	if ((adev->asic_type != CHIP_STONEY) &&
954 	    (adev->asic_type != CHIP_TOPAZ))
955 		release_firmware(adev->gfx.mec2_fw);
956 	adev->gfx.mec2_fw = NULL;
957 
958 	kfree(adev->gfx.rlc.register_list_format);
959 }
960 
961 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
962 {
963 	const char *chip_name;
964 	char fw_name[30];
965 	int err;
966 	struct amdgpu_firmware_info *info = NULL;
967 	const struct common_firmware_header *header = NULL;
968 	const struct gfx_firmware_header_v1_0 *cp_hdr;
969 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
970 	unsigned int *tmp = NULL, i;
971 
972 	DRM_DEBUG("\n");
973 
974 	switch (adev->asic_type) {
975 	case CHIP_TOPAZ:
976 		chip_name = "topaz";
977 		break;
978 	case CHIP_TONGA:
979 		chip_name = "tonga";
980 		break;
981 	case CHIP_CARRIZO:
982 		chip_name = "carrizo";
983 		break;
984 	case CHIP_FIJI:
985 		chip_name = "fiji";
986 		break;
987 	case CHIP_STONEY:
988 		chip_name = "stoney";
989 		break;
990 	case CHIP_POLARIS10:
991 		chip_name = "polaris10";
992 		break;
993 	case CHIP_POLARIS11:
994 		chip_name = "polaris11";
995 		break;
996 	case CHIP_POLARIS12:
997 		chip_name = "polaris12";
998 		break;
999 	case CHIP_VEGAM:
1000 		chip_name = "vegam";
1001 		break;
1002 	default:
1003 		BUG();
1004 	}
1005 
1006 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1007 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp_2.bin", chip_name);
1008 		err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1009 		if (err == -ENOENT) {
1010 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1011 			err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1012 		}
1013 	} else {
1014 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1015 		err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1016 	}
1017 	if (err)
1018 		goto out;
1019 	err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1020 	if (err)
1021 		goto out;
1022 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1023 	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1024 	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1025 
1026 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1027 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me_2.bin", chip_name);
1028 		err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1029 		if (err == -ENOENT) {
1030 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1031 			err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1032 		}
1033 	} else {
1034 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1035 		err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1036 	}
1037 	if (err)
1038 		goto out;
1039 	err = amdgpu_ucode_validate(adev->gfx.me_fw);
1040 	if (err)
1041 		goto out;
1042 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1043 	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1044 
1045 	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1046 
1047 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1048 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce_2.bin", chip_name);
1049 		err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1050 		if (err == -ENOENT) {
1051 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1052 			err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1053 		}
1054 	} else {
1055 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1056 		err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1057 	}
1058 	if (err)
1059 		goto out;
1060 	err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1061 	if (err)
1062 		goto out;
1063 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1064 	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1065 	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1066 
1067 	/*
1068 	 * Support for MCBP/Virtualization in combination with chained IBs is
1069 	 * formal released on feature version #46
1070 	 */
1071 	if (adev->gfx.ce_feature_version >= 46 &&
1072 	    adev->gfx.pfp_feature_version >= 46) {
1073 		adev->virt.chained_ib_support = true;
1074 		DRM_INFO("Chained IB support enabled!\n");
1075 	} else
1076 		adev->virt.chained_ib_support = false;
1077 
1078 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1079 	err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1080 	if (err)
1081 		goto out;
1082 	err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1083 	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1084 	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1085 	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1086 
1087 	adev->gfx.rlc.save_and_restore_offset =
1088 			le32_to_cpu(rlc_hdr->save_and_restore_offset);
1089 	adev->gfx.rlc.clear_state_descriptor_offset =
1090 			le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1091 	adev->gfx.rlc.avail_scratch_ram_locations =
1092 			le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1093 	adev->gfx.rlc.reg_restore_list_size =
1094 			le32_to_cpu(rlc_hdr->reg_restore_list_size);
1095 	adev->gfx.rlc.reg_list_format_start =
1096 			le32_to_cpu(rlc_hdr->reg_list_format_start);
1097 	adev->gfx.rlc.reg_list_format_separate_start =
1098 			le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1099 	adev->gfx.rlc.starting_offsets_start =
1100 			le32_to_cpu(rlc_hdr->starting_offsets_start);
1101 	adev->gfx.rlc.reg_list_format_size_bytes =
1102 			le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1103 	adev->gfx.rlc.reg_list_size_bytes =
1104 			le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1105 
1106 	adev->gfx.rlc.register_list_format =
1107 			kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1108 					adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1109 
1110 	if (!adev->gfx.rlc.register_list_format) {
1111 		err = -ENOMEM;
1112 		goto out;
1113 	}
1114 
1115 	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1116 			le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1117 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1118 		adev->gfx.rlc.register_list_format[i] =	le32_to_cpu(tmp[i]);
1119 
1120 	adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1121 
1122 	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1123 			le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1124 	for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1125 		adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1126 
1127 	if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1128 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec_2.bin", chip_name);
1129 		err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1130 		if (err == -ENOENT) {
1131 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1132 			err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1133 		}
1134 	} else {
1135 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1136 		err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1137 	}
1138 	if (err)
1139 		goto out;
1140 	err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1141 	if (err)
1142 		goto out;
1143 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1144 	adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1145 	adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1146 
1147 	if ((adev->asic_type != CHIP_STONEY) &&
1148 	    (adev->asic_type != CHIP_TOPAZ)) {
1149 		if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1150 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2_2.bin", chip_name);
1151 			err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1152 			if (err == -ENOENT) {
1153 				snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1154 				err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1155 			}
1156 		} else {
1157 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1158 			err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1159 		}
1160 		if (!err) {
1161 			err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1162 			if (err)
1163 				goto out;
1164 			cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1165 				adev->gfx.mec2_fw->data;
1166 			adev->gfx.mec2_fw_version =
1167 				le32_to_cpu(cp_hdr->header.ucode_version);
1168 			adev->gfx.mec2_feature_version =
1169 				le32_to_cpu(cp_hdr->ucode_feature_version);
1170 		} else {
1171 			err = 0;
1172 			adev->gfx.mec2_fw = NULL;
1173 		}
1174 	}
1175 
1176 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1177 	info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1178 	info->fw = adev->gfx.pfp_fw;
1179 	header = (const struct common_firmware_header *)info->fw->data;
1180 	adev->firmware.fw_size +=
1181 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1182 
1183 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1184 	info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1185 	info->fw = adev->gfx.me_fw;
1186 	header = (const struct common_firmware_header *)info->fw->data;
1187 	adev->firmware.fw_size +=
1188 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1189 
1190 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1191 	info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1192 	info->fw = adev->gfx.ce_fw;
1193 	header = (const struct common_firmware_header *)info->fw->data;
1194 	adev->firmware.fw_size +=
1195 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1196 
1197 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1198 	info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1199 	info->fw = adev->gfx.rlc_fw;
1200 	header = (const struct common_firmware_header *)info->fw->data;
1201 	adev->firmware.fw_size +=
1202 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1203 
1204 	info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1205 	info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1206 	info->fw = adev->gfx.mec_fw;
1207 	header = (const struct common_firmware_header *)info->fw->data;
1208 	adev->firmware.fw_size +=
1209 		ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1210 
1211 	/* we need account JT in */
1212 	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1213 	adev->firmware.fw_size +=
1214 		ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
1215 
1216 	if (amdgpu_sriov_vf(adev)) {
1217 		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];
1218 		info->ucode_id = AMDGPU_UCODE_ID_STORAGE;
1219 		info->fw = adev->gfx.mec_fw;
1220 		adev->firmware.fw_size +=
1221 			ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);
1222 	}
1223 
1224 	if (adev->gfx.mec2_fw) {
1225 		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1226 		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1227 		info->fw = adev->gfx.mec2_fw;
1228 		header = (const struct common_firmware_header *)info->fw->data;
1229 		adev->firmware.fw_size +=
1230 			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1231 	}
1232 
1233 out:
1234 	if (err) {
1235 		dev_err(adev->dev,
1236 			"gfx8: Failed to load firmware \"%s\"\n",
1237 			fw_name);
1238 		release_firmware(adev->gfx.pfp_fw);
1239 		adev->gfx.pfp_fw = NULL;
1240 		release_firmware(adev->gfx.me_fw);
1241 		adev->gfx.me_fw = NULL;
1242 		release_firmware(adev->gfx.ce_fw);
1243 		adev->gfx.ce_fw = NULL;
1244 		release_firmware(adev->gfx.rlc_fw);
1245 		adev->gfx.rlc_fw = NULL;
1246 		release_firmware(adev->gfx.mec_fw);
1247 		adev->gfx.mec_fw = NULL;
1248 		release_firmware(adev->gfx.mec2_fw);
1249 		adev->gfx.mec2_fw = NULL;
1250 	}
1251 	return err;
1252 }
1253 
1254 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
1255 				    volatile u32 *buffer)
1256 {
1257 	u32 count = 0, i;
1258 	const struct cs_section_def *sect = NULL;
1259 	const struct cs_extent_def *ext = NULL;
1260 
1261 	if (adev->gfx.rlc.cs_data == NULL)
1262 		return;
1263 	if (buffer == NULL)
1264 		return;
1265 
1266 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1267 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1268 
1269 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1270 	buffer[count++] = cpu_to_le32(0x80000000);
1271 	buffer[count++] = cpu_to_le32(0x80000000);
1272 
1273 	for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1274 		for (ext = sect->section; ext->extent != NULL; ++ext) {
1275 			if (sect->id == SECT_CONTEXT) {
1276 				buffer[count++] =
1277 					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1278 				buffer[count++] = cpu_to_le32(ext->reg_index -
1279 						PACKET3_SET_CONTEXT_REG_START);
1280 				for (i = 0; i < ext->reg_count; i++)
1281 					buffer[count++] = cpu_to_le32(ext->extent[i]);
1282 			} else {
1283 				return;
1284 			}
1285 		}
1286 	}
1287 
1288 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
1289 	buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG -
1290 			PACKET3_SET_CONTEXT_REG_START);
1291 	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
1292 	buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);
1293 
1294 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1295 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1296 
1297 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1298 	buffer[count++] = cpu_to_le32(0);
1299 }
1300 
1301 static void cz_init_cp_jump_table(struct amdgpu_device *adev)
1302 {
1303 	const __le32 *fw_data;
1304 	volatile u32 *dst_ptr;
1305 	int me, i, max_me = 4;
1306 	u32 bo_offset = 0;
1307 	u32 table_offset, table_size;
1308 
1309 	if (adev->asic_type == CHIP_CARRIZO)
1310 		max_me = 5;
1311 
1312 	/* write the cp table buffer */
1313 	dst_ptr = adev->gfx.rlc.cp_table_ptr;
1314 	for (me = 0; me < max_me; me++) {
1315 		if (me == 0) {
1316 			const struct gfx_firmware_header_v1_0 *hdr =
1317 				(const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1318 			fw_data = (const __le32 *)
1319 				(adev->gfx.ce_fw->data +
1320 				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1321 			table_offset = le32_to_cpu(hdr->jt_offset);
1322 			table_size = le32_to_cpu(hdr->jt_size);
1323 		} else if (me == 1) {
1324 			const struct gfx_firmware_header_v1_0 *hdr =
1325 				(const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1326 			fw_data = (const __le32 *)
1327 				(adev->gfx.pfp_fw->data +
1328 				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1329 			table_offset = le32_to_cpu(hdr->jt_offset);
1330 			table_size = le32_to_cpu(hdr->jt_size);
1331 		} else if (me == 2) {
1332 			const struct gfx_firmware_header_v1_0 *hdr =
1333 				(const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1334 			fw_data = (const __le32 *)
1335 				(adev->gfx.me_fw->data +
1336 				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1337 			table_offset = le32_to_cpu(hdr->jt_offset);
1338 			table_size = le32_to_cpu(hdr->jt_size);
1339 		} else if (me == 3) {
1340 			const struct gfx_firmware_header_v1_0 *hdr =
1341 				(const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1342 			fw_data = (const __le32 *)
1343 				(adev->gfx.mec_fw->data +
1344 				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1345 			table_offset = le32_to_cpu(hdr->jt_offset);
1346 			table_size = le32_to_cpu(hdr->jt_size);
1347 		} else  if (me == 4) {
1348 			const struct gfx_firmware_header_v1_0 *hdr =
1349 				(const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
1350 			fw_data = (const __le32 *)
1351 				(adev->gfx.mec2_fw->data +
1352 				 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1353 			table_offset = le32_to_cpu(hdr->jt_offset);
1354 			table_size = le32_to_cpu(hdr->jt_size);
1355 		}
1356 
1357 		for (i = 0; i < table_size; i ++) {
1358 			dst_ptr[bo_offset + i] =
1359 				cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
1360 		}
1361 
1362 		bo_offset += table_size;
1363 	}
1364 }
1365 
1366 static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
1367 {
1368 	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, NULL, NULL);
1369 	amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, NULL, NULL);
1370 }
1371 
1372 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
1373 {
1374 	volatile u32 *dst_ptr;
1375 	u32 dws;
1376 	const struct cs_section_def *cs_data;
1377 	int r;
1378 
1379 	adev->gfx.rlc.cs_data = vi_cs_data;
1380 
1381 	cs_data = adev->gfx.rlc.cs_data;
1382 
1383 	if (cs_data) {
1384 		/* clear state block */
1385 		adev->gfx.rlc.clear_state_size = dws = gfx_v8_0_get_csb_size(adev);
1386 
1387 		r = amdgpu_bo_create_reserved(adev, dws * 4, PAGE_SIZE,
1388 					      AMDGPU_GEM_DOMAIN_VRAM,
1389 					      &adev->gfx.rlc.clear_state_obj,
1390 					      &adev->gfx.rlc.clear_state_gpu_addr,
1391 					      (void **)&adev->gfx.rlc.cs_ptr);
1392 		if (r) {
1393 			dev_warn(adev->dev, "(%d) create RLC c bo failed\n", r);
1394 			gfx_v8_0_rlc_fini(adev);
1395 			return r;
1396 		}
1397 
1398 		/* set up the cs buffer */
1399 		dst_ptr = adev->gfx.rlc.cs_ptr;
1400 		gfx_v8_0_get_csb_buffer(adev, dst_ptr);
1401 		amdgpu_bo_kunmap(adev->gfx.rlc.clear_state_obj);
1402 		amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
1403 	}
1404 
1405 	if ((adev->asic_type == CHIP_CARRIZO) ||
1406 	    (adev->asic_type == CHIP_STONEY)) {
1407 		adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1408 		r = amdgpu_bo_create_reserved(adev, adev->gfx.rlc.cp_table_size,
1409 					      PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
1410 					      &adev->gfx.rlc.cp_table_obj,
1411 					      &adev->gfx.rlc.cp_table_gpu_addr,
1412 					      (void **)&adev->gfx.rlc.cp_table_ptr);
1413 		if (r) {
1414 			dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
1415 			return r;
1416 		}
1417 
1418 		cz_init_cp_jump_table(adev);
1419 
1420 		amdgpu_bo_kunmap(adev->gfx.rlc.cp_table_obj);
1421 		amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
1422 	}
1423 
1424 	return 0;
1425 }
1426 
1427 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
1428 {
1429 	amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1430 }
1431 
1432 static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
1433 {
1434 	int r;
1435 	u32 *hpd;
1436 	size_t mec_hpd_size;
1437 
1438 	bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1439 
1440 	/* take ownership of the relevant compute queues */
1441 	amdgpu_gfx_compute_queue_acquire(adev);
1442 
1443 	mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE;
1444 
1445 	r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1446 				      AMDGPU_GEM_DOMAIN_GTT,
1447 				      &adev->gfx.mec.hpd_eop_obj,
1448 				      &adev->gfx.mec.hpd_eop_gpu_addr,
1449 				      (void **)&hpd);
1450 	if (r) {
1451 		dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1452 		return r;
1453 	}
1454 
1455 	memset(hpd, 0, mec_hpd_size);
1456 
1457 	amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1458 	amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1459 
1460 	return 0;
1461 }
1462 
1463 static const u32 vgpr_init_compute_shader[] =
1464 {
1465 	0x7e000209, 0x7e020208,
1466 	0x7e040207, 0x7e060206,
1467 	0x7e080205, 0x7e0a0204,
1468 	0x7e0c0203, 0x7e0e0202,
1469 	0x7e100201, 0x7e120200,
1470 	0x7e140209, 0x7e160208,
1471 	0x7e180207, 0x7e1a0206,
1472 	0x7e1c0205, 0x7e1e0204,
1473 	0x7e200203, 0x7e220202,
1474 	0x7e240201, 0x7e260200,
1475 	0x7e280209, 0x7e2a0208,
1476 	0x7e2c0207, 0x7e2e0206,
1477 	0x7e300205, 0x7e320204,
1478 	0x7e340203, 0x7e360202,
1479 	0x7e380201, 0x7e3a0200,
1480 	0x7e3c0209, 0x7e3e0208,
1481 	0x7e400207, 0x7e420206,
1482 	0x7e440205, 0x7e460204,
1483 	0x7e480203, 0x7e4a0202,
1484 	0x7e4c0201, 0x7e4e0200,
1485 	0x7e500209, 0x7e520208,
1486 	0x7e540207, 0x7e560206,
1487 	0x7e580205, 0x7e5a0204,
1488 	0x7e5c0203, 0x7e5e0202,
1489 	0x7e600201, 0x7e620200,
1490 	0x7e640209, 0x7e660208,
1491 	0x7e680207, 0x7e6a0206,
1492 	0x7e6c0205, 0x7e6e0204,
1493 	0x7e700203, 0x7e720202,
1494 	0x7e740201, 0x7e760200,
1495 	0x7e780209, 0x7e7a0208,
1496 	0x7e7c0207, 0x7e7e0206,
1497 	0xbf8a0000, 0xbf810000,
1498 };
1499 
1500 static const u32 sgpr_init_compute_shader[] =
1501 {
1502 	0xbe8a0100, 0xbe8c0102,
1503 	0xbe8e0104, 0xbe900106,
1504 	0xbe920108, 0xbe940100,
1505 	0xbe960102, 0xbe980104,
1506 	0xbe9a0106, 0xbe9c0108,
1507 	0xbe9e0100, 0xbea00102,
1508 	0xbea20104, 0xbea40106,
1509 	0xbea60108, 0xbea80100,
1510 	0xbeaa0102, 0xbeac0104,
1511 	0xbeae0106, 0xbeb00108,
1512 	0xbeb20100, 0xbeb40102,
1513 	0xbeb60104, 0xbeb80106,
1514 	0xbeba0108, 0xbebc0100,
1515 	0xbebe0102, 0xbec00104,
1516 	0xbec20106, 0xbec40108,
1517 	0xbec60100, 0xbec80102,
1518 	0xbee60004, 0xbee70005,
1519 	0xbeea0006, 0xbeeb0007,
1520 	0xbee80008, 0xbee90009,
1521 	0xbefc0000, 0xbf8a0000,
1522 	0xbf810000, 0x00000000,
1523 };
1524 
1525 static const u32 vgpr_init_regs[] =
1526 {
1527 	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,
1528 	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1529 	mmCOMPUTE_NUM_THREAD_X, 256*4,
1530 	mmCOMPUTE_NUM_THREAD_Y, 1,
1531 	mmCOMPUTE_NUM_THREAD_Z, 1,
1532 	mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */
1533 	mmCOMPUTE_PGM_RSRC2, 20,
1534 	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1535 	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1536 	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1537 	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1538 	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1539 	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1540 	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1541 	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1542 	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1543 	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1544 };
1545 
1546 static const u32 sgpr1_init_regs[] =
1547 {
1548 	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,
1549 	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1550 	mmCOMPUTE_NUM_THREAD_X, 256*5,
1551 	mmCOMPUTE_NUM_THREAD_Y, 1,
1552 	mmCOMPUTE_NUM_THREAD_Z, 1,
1553 	mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1554 	mmCOMPUTE_PGM_RSRC2, 20,
1555 	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1556 	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1557 	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1558 	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1559 	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1560 	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1561 	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1562 	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1563 	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1564 	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1565 };
1566 
1567 static const u32 sgpr2_init_regs[] =
1568 {
1569 	mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,
1570 	mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
1571 	mmCOMPUTE_NUM_THREAD_X, 256*5,
1572 	mmCOMPUTE_NUM_THREAD_Y, 1,
1573 	mmCOMPUTE_NUM_THREAD_Z, 1,
1574 	mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1575 	mmCOMPUTE_PGM_RSRC2, 20,
1576 	mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1577 	mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1578 	mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1579 	mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1580 	mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1581 	mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1582 	mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1583 	mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1584 	mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1585 	mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1586 };
1587 
1588 static const u32 sec_ded_counter_registers[] =
1589 {
1590 	mmCPC_EDC_ATC_CNT,
1591 	mmCPC_EDC_SCRATCH_CNT,
1592 	mmCPC_EDC_UCODE_CNT,
1593 	mmCPF_EDC_ATC_CNT,
1594 	mmCPF_EDC_ROQ_CNT,
1595 	mmCPF_EDC_TAG_CNT,
1596 	mmCPG_EDC_ATC_CNT,
1597 	mmCPG_EDC_DMA_CNT,
1598 	mmCPG_EDC_TAG_CNT,
1599 	mmDC_EDC_CSINVOC_CNT,
1600 	mmDC_EDC_RESTORE_CNT,
1601 	mmDC_EDC_STATE_CNT,
1602 	mmGDS_EDC_CNT,
1603 	mmGDS_EDC_GRBM_CNT,
1604 	mmGDS_EDC_OA_DED,
1605 	mmSPI_EDC_CNT,
1606 	mmSQC_ATC_EDC_GATCL1_CNT,
1607 	mmSQC_EDC_CNT,
1608 	mmSQ_EDC_DED_CNT,
1609 	mmSQ_EDC_INFO,
1610 	mmSQ_EDC_SEC_CNT,
1611 	mmTCC_EDC_CNT,
1612 	mmTCP_ATC_EDC_GATCL1_CNT,
1613 	mmTCP_EDC_CNT,
1614 	mmTD_EDC_CNT
1615 };
1616 
1617 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
1618 {
1619 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
1620 	struct amdgpu_ib ib;
1621 	struct dma_fence *f = NULL;
1622 	int r, i;
1623 	u32 tmp;
1624 	unsigned total_size, vgpr_offset, sgpr_offset;
1625 	u64 gpu_addr;
1626 
1627 	/* only supported on CZ */
1628 	if (adev->asic_type != CHIP_CARRIZO)
1629 		return 0;
1630 
1631 	/* bail if the compute ring is not ready */
1632 	if (!ring->ready)
1633 		return 0;
1634 
1635 	tmp = RREG32(mmGB_EDC_MODE);
1636 	WREG32(mmGB_EDC_MODE, 0);
1637 
1638 	total_size =
1639 		(((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1640 	total_size +=
1641 		(((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1642 	total_size +=
1643 		(((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1644 	total_size = ALIGN(total_size, 256);
1645 	vgpr_offset = total_size;
1646 	total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
1647 	sgpr_offset = total_size;
1648 	total_size += sizeof(sgpr_init_compute_shader);
1649 
1650 	/* allocate an indirect buffer to put the commands in */
1651 	memset(&ib, 0, sizeof(ib));
1652 	r = amdgpu_ib_get(adev, NULL, total_size, &ib);
1653 	if (r) {
1654 		DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
1655 		return r;
1656 	}
1657 
1658 	/* load the compute shaders */
1659 	for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
1660 		ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
1661 
1662 	for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
1663 		ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
1664 
1665 	/* init the ib length to 0 */
1666 	ib.length_dw = 0;
1667 
1668 	/* VGPR */
1669 	/* write the register state for the compute dispatch */
1670 	for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
1671 		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1672 		ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
1673 		ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
1674 	}
1675 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1676 	gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
1677 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1678 	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1679 	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1680 	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1681 
1682 	/* write dispatch packet */
1683 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1684 	ib.ptr[ib.length_dw++] = 8; /* x */
1685 	ib.ptr[ib.length_dw++] = 1; /* y */
1686 	ib.ptr[ib.length_dw++] = 1; /* z */
1687 	ib.ptr[ib.length_dw++] =
1688 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1689 
1690 	/* write CS partial flush packet */
1691 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1692 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1693 
1694 	/* SGPR1 */
1695 	/* write the register state for the compute dispatch */
1696 	for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
1697 		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1698 		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
1699 		ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
1700 	}
1701 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1702 	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1703 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1704 	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1705 	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1706 	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1707 
1708 	/* write dispatch packet */
1709 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1710 	ib.ptr[ib.length_dw++] = 8; /* x */
1711 	ib.ptr[ib.length_dw++] = 1; /* y */
1712 	ib.ptr[ib.length_dw++] = 1; /* z */
1713 	ib.ptr[ib.length_dw++] =
1714 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1715 
1716 	/* write CS partial flush packet */
1717 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1718 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1719 
1720 	/* SGPR2 */
1721 	/* write the register state for the compute dispatch */
1722 	for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
1723 		ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1724 		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
1725 		ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
1726 	}
1727 	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1728 	gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1729 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1730 	ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1731 	ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1732 	ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1733 
1734 	/* write dispatch packet */
1735 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1736 	ib.ptr[ib.length_dw++] = 8; /* x */
1737 	ib.ptr[ib.length_dw++] = 1; /* y */
1738 	ib.ptr[ib.length_dw++] = 1; /* z */
1739 	ib.ptr[ib.length_dw++] =
1740 		REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1741 
1742 	/* write CS partial flush packet */
1743 	ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1744 	ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1745 
1746 	/* shedule the ib on the ring */
1747 	r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1748 	if (r) {
1749 		DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
1750 		goto fail;
1751 	}
1752 
1753 	/* wait for the GPU to finish processing the IB */
1754 	r = dma_fence_wait(f, false);
1755 	if (r) {
1756 		DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
1757 		goto fail;
1758 	}
1759 
1760 	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);
1761 	tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);
1762 	WREG32(mmGB_EDC_MODE, tmp);
1763 
1764 	tmp = RREG32(mmCC_GC_EDC_CONFIG);
1765 	tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;
1766 	WREG32(mmCC_GC_EDC_CONFIG, tmp);
1767 
1768 
1769 	/* read back registers to clear the counters */
1770 	for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)
1771 		RREG32(sec_ded_counter_registers[i]);
1772 
1773 fail:
1774 	amdgpu_ib_free(adev, &ib, NULL);
1775 	dma_fence_put(f);
1776 
1777 	return r;
1778 }
1779 
1780 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
1781 {
1782 	u32 gb_addr_config;
1783 	u32 mc_shared_chmap, mc_arb_ramcfg;
1784 	u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
1785 	u32 tmp;
1786 	int ret;
1787 
1788 	switch (adev->asic_type) {
1789 	case CHIP_TOPAZ:
1790 		adev->gfx.config.max_shader_engines = 1;
1791 		adev->gfx.config.max_tile_pipes = 2;
1792 		adev->gfx.config.max_cu_per_sh = 6;
1793 		adev->gfx.config.max_sh_per_se = 1;
1794 		adev->gfx.config.max_backends_per_se = 2;
1795 		adev->gfx.config.max_texture_channel_caches = 2;
1796 		adev->gfx.config.max_gprs = 256;
1797 		adev->gfx.config.max_gs_threads = 32;
1798 		adev->gfx.config.max_hw_contexts = 8;
1799 
1800 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1801 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1802 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1803 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1804 		gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
1805 		break;
1806 	case CHIP_FIJI:
1807 		adev->gfx.config.max_shader_engines = 4;
1808 		adev->gfx.config.max_tile_pipes = 16;
1809 		adev->gfx.config.max_cu_per_sh = 16;
1810 		adev->gfx.config.max_sh_per_se = 1;
1811 		adev->gfx.config.max_backends_per_se = 4;
1812 		adev->gfx.config.max_texture_channel_caches = 16;
1813 		adev->gfx.config.max_gprs = 256;
1814 		adev->gfx.config.max_gs_threads = 32;
1815 		adev->gfx.config.max_hw_contexts = 8;
1816 
1817 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1818 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1819 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1820 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1821 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1822 		break;
1823 	case CHIP_POLARIS11:
1824 	case CHIP_POLARIS12:
1825 		ret = amdgpu_atombios_get_gfx_info(adev);
1826 		if (ret)
1827 			return ret;
1828 		adev->gfx.config.max_gprs = 256;
1829 		adev->gfx.config.max_gs_threads = 32;
1830 		adev->gfx.config.max_hw_contexts = 8;
1831 
1832 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1833 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1834 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1835 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1836 		gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;
1837 		break;
1838 	case CHIP_POLARIS10:
1839 	case CHIP_VEGAM:
1840 		ret = amdgpu_atombios_get_gfx_info(adev);
1841 		if (ret)
1842 			return ret;
1843 		adev->gfx.config.max_gprs = 256;
1844 		adev->gfx.config.max_gs_threads = 32;
1845 		adev->gfx.config.max_hw_contexts = 8;
1846 
1847 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1848 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1849 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1850 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1851 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1852 		break;
1853 	case CHIP_TONGA:
1854 		adev->gfx.config.max_shader_engines = 4;
1855 		adev->gfx.config.max_tile_pipes = 8;
1856 		adev->gfx.config.max_cu_per_sh = 8;
1857 		adev->gfx.config.max_sh_per_se = 1;
1858 		adev->gfx.config.max_backends_per_se = 2;
1859 		adev->gfx.config.max_texture_channel_caches = 8;
1860 		adev->gfx.config.max_gprs = 256;
1861 		adev->gfx.config.max_gs_threads = 32;
1862 		adev->gfx.config.max_hw_contexts = 8;
1863 
1864 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1865 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1866 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1867 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1868 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1869 		break;
1870 	case CHIP_CARRIZO:
1871 		adev->gfx.config.max_shader_engines = 1;
1872 		adev->gfx.config.max_tile_pipes = 2;
1873 		adev->gfx.config.max_sh_per_se = 1;
1874 		adev->gfx.config.max_backends_per_se = 2;
1875 		adev->gfx.config.max_cu_per_sh = 8;
1876 		adev->gfx.config.max_texture_channel_caches = 2;
1877 		adev->gfx.config.max_gprs = 256;
1878 		adev->gfx.config.max_gs_threads = 32;
1879 		adev->gfx.config.max_hw_contexts = 8;
1880 
1881 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1882 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1883 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1884 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1885 		gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1886 		break;
1887 	case CHIP_STONEY:
1888 		adev->gfx.config.max_shader_engines = 1;
1889 		adev->gfx.config.max_tile_pipes = 2;
1890 		adev->gfx.config.max_sh_per_se = 1;
1891 		adev->gfx.config.max_backends_per_se = 1;
1892 		adev->gfx.config.max_cu_per_sh = 3;
1893 		adev->gfx.config.max_texture_channel_caches = 2;
1894 		adev->gfx.config.max_gprs = 256;
1895 		adev->gfx.config.max_gs_threads = 16;
1896 		adev->gfx.config.max_hw_contexts = 8;
1897 
1898 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1899 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1900 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1901 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1902 		gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1903 		break;
1904 	default:
1905 		adev->gfx.config.max_shader_engines = 2;
1906 		adev->gfx.config.max_tile_pipes = 4;
1907 		adev->gfx.config.max_cu_per_sh = 2;
1908 		adev->gfx.config.max_sh_per_se = 1;
1909 		adev->gfx.config.max_backends_per_se = 2;
1910 		adev->gfx.config.max_texture_channel_caches = 4;
1911 		adev->gfx.config.max_gprs = 256;
1912 		adev->gfx.config.max_gs_threads = 32;
1913 		adev->gfx.config.max_hw_contexts = 8;
1914 
1915 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1916 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1917 		adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1918 		adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1919 		gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1920 		break;
1921 	}
1922 
1923 	mc_shared_chmap = RREG32(mmMC_SHARED_CHMAP);
1924 	adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
1925 	mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
1926 
1927 	adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
1928 	adev->gfx.config.mem_max_burst_length_bytes = 256;
1929 	if (adev->flags & AMD_IS_APU) {
1930 		/* Get memory bank mapping mode. */
1931 		tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
1932 		dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1933 		dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1934 
1935 		tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
1936 		dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1937 		dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1938 
1939 		/* Validate settings in case only one DIMM installed. */
1940 		if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
1941 			dimm00_addr_map = 0;
1942 		if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
1943 			dimm01_addr_map = 0;
1944 		if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
1945 			dimm10_addr_map = 0;
1946 		if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
1947 			dimm11_addr_map = 0;
1948 
1949 		/* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
1950 		/* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
1951 		if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
1952 			adev->gfx.config.mem_row_size_in_kb = 2;
1953 		else
1954 			adev->gfx.config.mem_row_size_in_kb = 1;
1955 	} else {
1956 		tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
1957 		adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1958 		if (adev->gfx.config.mem_row_size_in_kb > 4)
1959 			adev->gfx.config.mem_row_size_in_kb = 4;
1960 	}
1961 
1962 	adev->gfx.config.shader_engine_tile_size = 32;
1963 	adev->gfx.config.num_gpus = 1;
1964 	adev->gfx.config.multi_gpu_tile_size = 64;
1965 
1966 	/* fix up row size */
1967 	switch (adev->gfx.config.mem_row_size_in_kb) {
1968 	case 1:
1969 	default:
1970 		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
1971 		break;
1972 	case 2:
1973 		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
1974 		break;
1975 	case 4:
1976 		gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
1977 		break;
1978 	}
1979 	adev->gfx.config.gb_addr_config = gb_addr_config;
1980 
1981 	return 0;
1982 }
1983 
1984 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1985 					int mec, int pipe, int queue)
1986 {
1987 	int r;
1988 	unsigned irq_type;
1989 	struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1990 
1991 	ring = &adev->gfx.compute_ring[ring_id];
1992 
1993 	/* mec0 is me1 */
1994 	ring->me = mec + 1;
1995 	ring->pipe = pipe;
1996 	ring->queue = queue;
1997 
1998 	ring->ring_obj = NULL;
1999 	ring->use_doorbell = true;
2000 	ring->doorbell_index = AMDGPU_DOORBELL_MEC_RING0 + ring_id;
2001 	ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2002 				+ (ring_id * GFX8_MEC_HPD_SIZE);
2003 	sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2004 
2005 	irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2006 		+ ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2007 		+ ring->pipe;
2008 
2009 	/* type-2 packets are deprecated on MEC, use type-3 instead */
2010 	r = amdgpu_ring_init(adev, ring, 1024,
2011 			&adev->gfx.eop_irq, irq_type);
2012 	if (r)
2013 		return r;
2014 
2015 
2016 	return 0;
2017 }
2018 
2019 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);
2020 
2021 static int gfx_v8_0_sw_init(void *handle)
2022 {
2023 	int i, j, k, r, ring_id;
2024 	struct amdgpu_ring *ring;
2025 	struct amdgpu_kiq *kiq;
2026 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2027 
2028 	switch (adev->asic_type) {
2029 	case CHIP_TONGA:
2030 	case CHIP_CARRIZO:
2031 	case CHIP_FIJI:
2032 	case CHIP_POLARIS10:
2033 	case CHIP_POLARIS11:
2034 	case CHIP_POLARIS12:
2035 	case CHIP_VEGAM:
2036 		adev->gfx.mec.num_mec = 2;
2037 		break;
2038 	case CHIP_TOPAZ:
2039 	case CHIP_STONEY:
2040 	default:
2041 		adev->gfx.mec.num_mec = 1;
2042 		break;
2043 	}
2044 
2045 	adev->gfx.mec.num_pipe_per_mec = 4;
2046 	adev->gfx.mec.num_queue_per_pipe = 8;
2047 
2048 	/* EOP Event */
2049 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq);
2050 	if (r)
2051 		return r;
2052 
2053 	/* Privileged reg */
2054 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT,
2055 			      &adev->gfx.priv_reg_irq);
2056 	if (r)
2057 		return r;
2058 
2059 	/* Privileged inst */
2060 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT,
2061 			      &adev->gfx.priv_inst_irq);
2062 	if (r)
2063 		return r;
2064 
2065 	/* Add CP EDC/ECC irq  */
2066 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR,
2067 			      &adev->gfx.cp_ecc_error_irq);
2068 	if (r)
2069 		return r;
2070 
2071 	/* SQ interrupts. */
2072 	r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG,
2073 			      &adev->gfx.sq_irq);
2074 	if (r) {
2075 		DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r);
2076 		return r;
2077 	}
2078 
2079 	INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func);
2080 
2081 	adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2082 
2083 	gfx_v8_0_scratch_init(adev);
2084 
2085 	r = gfx_v8_0_init_microcode(adev);
2086 	if (r) {
2087 		DRM_ERROR("Failed to load gfx firmware!\n");
2088 		return r;
2089 	}
2090 
2091 	r = gfx_v8_0_rlc_init(adev);
2092 	if (r) {
2093 		DRM_ERROR("Failed to init rlc BOs!\n");
2094 		return r;
2095 	}
2096 
2097 	r = gfx_v8_0_mec_init(adev);
2098 	if (r) {
2099 		DRM_ERROR("Failed to init MEC BOs!\n");
2100 		return r;
2101 	}
2102 
2103 	/* set up the gfx ring */
2104 	for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2105 		ring = &adev->gfx.gfx_ring[i];
2106 		ring->ring_obj = NULL;
2107 		sprintf(ring->name, "gfx");
2108 		/* no gfx doorbells on iceland */
2109 		if (adev->asic_type != CHIP_TOPAZ) {
2110 			ring->use_doorbell = true;
2111 			ring->doorbell_index = AMDGPU_DOORBELL_GFX_RING0;
2112 		}
2113 
2114 		r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
2115 				     AMDGPU_CP_IRQ_GFX_EOP);
2116 		if (r)
2117 			return r;
2118 	}
2119 
2120 
2121 	/* set up the compute queues - allocate horizontally across pipes */
2122 	ring_id = 0;
2123 	for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2124 		for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2125 			for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2126 				if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2127 					continue;
2128 
2129 				r = gfx_v8_0_compute_ring_init(adev,
2130 								ring_id,
2131 								i, k, j);
2132 				if (r)
2133 					return r;
2134 
2135 				ring_id++;
2136 			}
2137 		}
2138 	}
2139 
2140 	r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE);
2141 	if (r) {
2142 		DRM_ERROR("Failed to init KIQ BOs!\n");
2143 		return r;
2144 	}
2145 
2146 	kiq = &adev->gfx.kiq;
2147 	r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2148 	if (r)
2149 		return r;
2150 
2151 	/* create MQD for all compute queues as well as KIQ for SRIOV case */
2152 	r = amdgpu_gfx_compute_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation));
2153 	if (r)
2154 		return r;
2155 
2156 	adev->gfx.ce_ram_size = 0x8000;
2157 
2158 	r = gfx_v8_0_gpu_early_init(adev);
2159 	if (r)
2160 		return r;
2161 
2162 	return 0;
2163 }
2164 
2165 static int gfx_v8_0_sw_fini(void *handle)
2166 {
2167 	int i;
2168 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2169 
2170 	amdgpu_bo_free_kernel(&adev->gds.oa_gfx_bo, NULL, NULL);
2171 	amdgpu_bo_free_kernel(&adev->gds.gws_gfx_bo, NULL, NULL);
2172 	amdgpu_bo_free_kernel(&adev->gds.gds_gfx_bo, NULL, NULL);
2173 
2174 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2175 		amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2176 	for (i = 0; i < adev->gfx.num_compute_rings; i++)
2177 		amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2178 
2179 	amdgpu_gfx_compute_mqd_sw_fini(adev);
2180 	amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring, &adev->gfx.kiq.irq);
2181 	amdgpu_gfx_kiq_fini(adev);
2182 
2183 	gfx_v8_0_mec_fini(adev);
2184 	gfx_v8_0_rlc_fini(adev);
2185 	amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2186 				&adev->gfx.rlc.clear_state_gpu_addr,
2187 				(void **)&adev->gfx.rlc.cs_ptr);
2188 	if ((adev->asic_type == CHIP_CARRIZO) ||
2189 	    (adev->asic_type == CHIP_STONEY)) {
2190 		amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2191 				&adev->gfx.rlc.cp_table_gpu_addr,
2192 				(void **)&adev->gfx.rlc.cp_table_ptr);
2193 	}
2194 	gfx_v8_0_free_microcode(adev);
2195 
2196 	return 0;
2197 }
2198 
2199 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
2200 {
2201 	uint32_t *modearray, *mod2array;
2202 	const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
2203 	const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
2204 	u32 reg_offset;
2205 
2206 	modearray = adev->gfx.config.tile_mode_array;
2207 	mod2array = adev->gfx.config.macrotile_mode_array;
2208 
2209 	for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2210 		modearray[reg_offset] = 0;
2211 
2212 	for (reg_offset = 0; reg_offset <  num_secondary_tile_mode_states; reg_offset++)
2213 		mod2array[reg_offset] = 0;
2214 
2215 	switch (adev->asic_type) {
2216 	case CHIP_TOPAZ:
2217 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2218 				PIPE_CONFIG(ADDR_SURF_P2) |
2219 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2220 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2221 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2222 				PIPE_CONFIG(ADDR_SURF_P2) |
2223 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2224 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2225 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2226 				PIPE_CONFIG(ADDR_SURF_P2) |
2227 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2228 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2229 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2230 				PIPE_CONFIG(ADDR_SURF_P2) |
2231 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2232 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2233 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2234 				PIPE_CONFIG(ADDR_SURF_P2) |
2235 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2236 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2237 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2238 				PIPE_CONFIG(ADDR_SURF_P2) |
2239 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2240 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2241 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2242 				PIPE_CONFIG(ADDR_SURF_P2) |
2243 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2244 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2245 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2246 				PIPE_CONFIG(ADDR_SURF_P2));
2247 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2248 				PIPE_CONFIG(ADDR_SURF_P2) |
2249 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2250 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2251 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2252 				 PIPE_CONFIG(ADDR_SURF_P2) |
2253 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2254 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2255 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2256 				 PIPE_CONFIG(ADDR_SURF_P2) |
2257 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2258 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2259 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2260 				 PIPE_CONFIG(ADDR_SURF_P2) |
2261 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2262 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2263 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2264 				 PIPE_CONFIG(ADDR_SURF_P2) |
2265 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2266 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2267 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2268 				 PIPE_CONFIG(ADDR_SURF_P2) |
2269 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2270 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2271 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2272 				 PIPE_CONFIG(ADDR_SURF_P2) |
2273 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2274 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2275 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2276 				 PIPE_CONFIG(ADDR_SURF_P2) |
2277 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2278 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2279 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2280 				 PIPE_CONFIG(ADDR_SURF_P2) |
2281 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2282 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2283 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2284 				 PIPE_CONFIG(ADDR_SURF_P2) |
2285 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2286 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2287 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2288 				 PIPE_CONFIG(ADDR_SURF_P2) |
2289 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2290 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2291 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2292 				 PIPE_CONFIG(ADDR_SURF_P2) |
2293 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2294 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2295 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2296 				 PIPE_CONFIG(ADDR_SURF_P2) |
2297 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2298 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2299 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2300 				 PIPE_CONFIG(ADDR_SURF_P2) |
2301 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2302 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2303 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2304 				 PIPE_CONFIG(ADDR_SURF_P2) |
2305 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2306 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2307 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2308 				 PIPE_CONFIG(ADDR_SURF_P2) |
2309 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2310 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2311 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2312 				 PIPE_CONFIG(ADDR_SURF_P2) |
2313 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2314 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2315 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2316 				 PIPE_CONFIG(ADDR_SURF_P2) |
2317 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2318 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2319 
2320 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2321 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2322 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2323 				NUM_BANKS(ADDR_SURF_8_BANK));
2324 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2325 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2326 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2327 				NUM_BANKS(ADDR_SURF_8_BANK));
2328 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2329 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2330 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2331 				NUM_BANKS(ADDR_SURF_8_BANK));
2332 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2333 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2334 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2335 				NUM_BANKS(ADDR_SURF_8_BANK));
2336 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2337 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2338 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2339 				NUM_BANKS(ADDR_SURF_8_BANK));
2340 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2341 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2342 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2343 				NUM_BANKS(ADDR_SURF_8_BANK));
2344 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2345 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2346 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2347 				NUM_BANKS(ADDR_SURF_8_BANK));
2348 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2349 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2350 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2351 				NUM_BANKS(ADDR_SURF_16_BANK));
2352 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2353 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2354 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2355 				NUM_BANKS(ADDR_SURF_16_BANK));
2356 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2357 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2358 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2359 				 NUM_BANKS(ADDR_SURF_16_BANK));
2360 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2361 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2362 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2363 				 NUM_BANKS(ADDR_SURF_16_BANK));
2364 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2365 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2366 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2367 				 NUM_BANKS(ADDR_SURF_16_BANK));
2368 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2369 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2370 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2371 				 NUM_BANKS(ADDR_SURF_16_BANK));
2372 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2373 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2374 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2375 				 NUM_BANKS(ADDR_SURF_8_BANK));
2376 
2377 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2378 			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
2379 			    reg_offset != 23)
2380 				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2381 
2382 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2383 			if (reg_offset != 7)
2384 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2385 
2386 		break;
2387 	case CHIP_FIJI:
2388 	case CHIP_VEGAM:
2389 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2390 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2391 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2392 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2393 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2394 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2395 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2396 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2397 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2398 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2399 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2400 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2401 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2402 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2403 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2404 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2405 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2406 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2407 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2408 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2409 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2410 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2411 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2412 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2413 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2414 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2415 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2416 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2417 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2418 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2419 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2420 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2421 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2422 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2423 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2424 				PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2425 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2426 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2427 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2428 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2429 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2430 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2431 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2432 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2433 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2434 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2435 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2436 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2437 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2438 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2439 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2440 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2441 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2442 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2443 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2445 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2446 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2447 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2448 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2449 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2450 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2451 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2452 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2453 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2454 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2455 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2456 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2457 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2458 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2459 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2460 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2461 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2462 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2463 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2464 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2465 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2466 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2467 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2468 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2469 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2470 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2471 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2472 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2473 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2474 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2475 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2476 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2477 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2478 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2479 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2480 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2481 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2482 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2483 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2484 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2485 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2486 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2487 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2488 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2489 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2490 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2491 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2492 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2493 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2494 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2495 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2496 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2497 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2498 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2499 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2500 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2501 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2502 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2503 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2504 				 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2505 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2506 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2507 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2508 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2509 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2510 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2511 
2512 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2513 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2514 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2515 				NUM_BANKS(ADDR_SURF_8_BANK));
2516 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2517 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2518 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2519 				NUM_BANKS(ADDR_SURF_8_BANK));
2520 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2521 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2522 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2523 				NUM_BANKS(ADDR_SURF_8_BANK));
2524 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2525 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2526 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2527 				NUM_BANKS(ADDR_SURF_8_BANK));
2528 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2530 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2531 				NUM_BANKS(ADDR_SURF_8_BANK));
2532 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2533 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2534 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2535 				NUM_BANKS(ADDR_SURF_8_BANK));
2536 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2537 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2538 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2539 				NUM_BANKS(ADDR_SURF_8_BANK));
2540 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2541 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2542 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2543 				NUM_BANKS(ADDR_SURF_8_BANK));
2544 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2545 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2546 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2547 				NUM_BANKS(ADDR_SURF_8_BANK));
2548 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2549 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2550 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2551 				 NUM_BANKS(ADDR_SURF_8_BANK));
2552 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2553 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2554 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2555 				 NUM_BANKS(ADDR_SURF_8_BANK));
2556 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2557 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2558 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2559 				 NUM_BANKS(ADDR_SURF_8_BANK));
2560 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2561 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2562 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2563 				 NUM_BANKS(ADDR_SURF_8_BANK));
2564 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2566 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2567 				 NUM_BANKS(ADDR_SURF_4_BANK));
2568 
2569 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2570 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2571 
2572 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2573 			if (reg_offset != 7)
2574 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2575 
2576 		break;
2577 	case CHIP_TONGA:
2578 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2579 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2580 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2581 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2582 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2583 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2584 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2585 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2586 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2587 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2588 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2589 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2590 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2591 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2592 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2593 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2594 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2595 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2596 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2597 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2598 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2599 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2600 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2601 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2602 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2603 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2604 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2605 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2606 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2607 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2608 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2609 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2610 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2611 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2612 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2613 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2614 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2615 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2616 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2617 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2618 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2619 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2620 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2621 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2622 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2623 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2624 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2625 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2626 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2627 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2628 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2629 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2630 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2631 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2632 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2633 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2634 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2635 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2636 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2637 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2638 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2639 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2640 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2641 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2642 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2643 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2644 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2645 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2646 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2647 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2648 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2649 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2650 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2651 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2652 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2653 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2654 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2655 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2656 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2657 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2658 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2659 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2660 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2661 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2662 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2663 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2664 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2665 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2666 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2667 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2668 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2669 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2670 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2671 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2672 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2673 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2674 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2675 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2676 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2677 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2678 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2679 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2680 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2681 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2682 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2683 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2684 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2685 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2686 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2687 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2688 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2689 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2690 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2691 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2692 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2693 				 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2694 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2695 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2696 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2697 				 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2698 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2699 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2700 
2701 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2703 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2704 				NUM_BANKS(ADDR_SURF_16_BANK));
2705 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2706 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2707 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2708 				NUM_BANKS(ADDR_SURF_16_BANK));
2709 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2710 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2711 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2712 				NUM_BANKS(ADDR_SURF_16_BANK));
2713 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2715 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2716 				NUM_BANKS(ADDR_SURF_16_BANK));
2717 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2718 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2719 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2720 				NUM_BANKS(ADDR_SURF_16_BANK));
2721 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2722 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2723 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2724 				NUM_BANKS(ADDR_SURF_16_BANK));
2725 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2728 				NUM_BANKS(ADDR_SURF_16_BANK));
2729 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2730 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2731 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2732 				NUM_BANKS(ADDR_SURF_16_BANK));
2733 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2734 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2735 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2736 				NUM_BANKS(ADDR_SURF_16_BANK));
2737 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2738 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2739 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2740 				 NUM_BANKS(ADDR_SURF_16_BANK));
2741 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2742 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2743 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2744 				 NUM_BANKS(ADDR_SURF_16_BANK));
2745 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2746 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2747 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2748 				 NUM_BANKS(ADDR_SURF_8_BANK));
2749 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2750 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2751 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2752 				 NUM_BANKS(ADDR_SURF_4_BANK));
2753 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2754 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2755 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2756 				 NUM_BANKS(ADDR_SURF_4_BANK));
2757 
2758 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2759 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2760 
2761 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2762 			if (reg_offset != 7)
2763 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2764 
2765 		break;
2766 	case CHIP_POLARIS11:
2767 	case CHIP_POLARIS12:
2768 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2769 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2770 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2771 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2772 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2773 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2774 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2775 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2776 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2777 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2778 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2779 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2780 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2781 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2782 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2783 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2784 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2785 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2786 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2787 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2788 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2789 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2790 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2791 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2792 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2793 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2794 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2795 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2796 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2797 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2798 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2799 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2800 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2801 				PIPE_CONFIG(ADDR_SURF_P4_16x16));
2802 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2803 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2804 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2805 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2806 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2807 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2808 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2809 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2810 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2811 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2812 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2813 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2814 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2815 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2816 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2817 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2818 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2819 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2820 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2821 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2822 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2823 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2824 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2825 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2826 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2827 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2828 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2829 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2830 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2831 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2832 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2833 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2834 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2835 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2836 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2837 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2838 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2839 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2840 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2841 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2842 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2843 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2844 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2845 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2846 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2847 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2848 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2849 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2850 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2851 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2852 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2853 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2854 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2855 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2856 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2857 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2858 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2859 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2860 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2861 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2862 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2863 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2864 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2865 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2866 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2867 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2868 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2869 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2870 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2871 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2872 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2873 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2874 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2875 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2876 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2877 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2878 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2879 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2880 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2881 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2882 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2883 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2884 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2885 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2886 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2887 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2888 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2889 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2890 
2891 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2892 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2893 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2894 				NUM_BANKS(ADDR_SURF_16_BANK));
2895 
2896 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2897 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2898 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2899 				NUM_BANKS(ADDR_SURF_16_BANK));
2900 
2901 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2902 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2903 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2904 				NUM_BANKS(ADDR_SURF_16_BANK));
2905 
2906 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2907 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2908 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2909 				NUM_BANKS(ADDR_SURF_16_BANK));
2910 
2911 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2912 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2913 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2914 				NUM_BANKS(ADDR_SURF_16_BANK));
2915 
2916 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2917 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2918 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2919 				NUM_BANKS(ADDR_SURF_16_BANK));
2920 
2921 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2922 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2923 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2924 				NUM_BANKS(ADDR_SURF_16_BANK));
2925 
2926 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2927 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2928 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2929 				NUM_BANKS(ADDR_SURF_16_BANK));
2930 
2931 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2932 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2933 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2934 				NUM_BANKS(ADDR_SURF_16_BANK));
2935 
2936 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2937 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2938 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2939 				NUM_BANKS(ADDR_SURF_16_BANK));
2940 
2941 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2942 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2943 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2944 				NUM_BANKS(ADDR_SURF_16_BANK));
2945 
2946 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2947 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2948 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2949 				NUM_BANKS(ADDR_SURF_16_BANK));
2950 
2951 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2952 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2953 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2954 				NUM_BANKS(ADDR_SURF_8_BANK));
2955 
2956 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2957 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2958 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2959 				NUM_BANKS(ADDR_SURF_4_BANK));
2960 
2961 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2962 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2963 
2964 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2965 			if (reg_offset != 7)
2966 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2967 
2968 		break;
2969 	case CHIP_POLARIS10:
2970 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2971 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2972 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2973 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2974 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2975 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2976 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2977 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2978 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2979 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2980 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2981 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2982 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2983 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2984 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2985 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2986 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2987 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2988 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2989 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2990 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2991 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2992 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2993 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2994 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2995 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2996 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2997 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2998 		modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2999 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3000 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3001 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3002 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3003 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
3004 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3005 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3006 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3007 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3008 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3009 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3010 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3011 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3012 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3013 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3014 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3015 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3016 		modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3017 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3018 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3019 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3020 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3021 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3022 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3023 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3024 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3025 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3026 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3027 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3028 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3029 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3030 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3031 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3032 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3033 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3034 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3035 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3036 		modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3037 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3038 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3039 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3040 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3041 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3042 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3043 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3044 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3045 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3046 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3047 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3048 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3049 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3050 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3051 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3052 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3053 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3054 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3055 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3056 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3057 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3058 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3059 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3060 		modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3061 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3062 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3063 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3064 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3065 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3066 				MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3067 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3068 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3069 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3070 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3071 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3072 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3073 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3074 				MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3075 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3076 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3077 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3078 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3079 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3080 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3081 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3082 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3083 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3084 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3085 				PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
3086 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3087 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3088 		modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3089 				PIPE_CONFIG(ADDR_SURF_P4_16x16) |
3090 				MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3091 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3092 
3093 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3094 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3095 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3096 				NUM_BANKS(ADDR_SURF_16_BANK));
3097 
3098 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3099 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3100 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3101 				NUM_BANKS(ADDR_SURF_16_BANK));
3102 
3103 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3104 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3105 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3106 				NUM_BANKS(ADDR_SURF_16_BANK));
3107 
3108 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3109 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3110 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3111 				NUM_BANKS(ADDR_SURF_16_BANK));
3112 
3113 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3114 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3115 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3116 				NUM_BANKS(ADDR_SURF_16_BANK));
3117 
3118 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3119 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3120 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3121 				NUM_BANKS(ADDR_SURF_16_BANK));
3122 
3123 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3124 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3125 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3126 				NUM_BANKS(ADDR_SURF_16_BANK));
3127 
3128 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3129 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3130 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3131 				NUM_BANKS(ADDR_SURF_16_BANK));
3132 
3133 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3134 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3135 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3136 				NUM_BANKS(ADDR_SURF_16_BANK));
3137 
3138 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3139 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3140 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3141 				NUM_BANKS(ADDR_SURF_16_BANK));
3142 
3143 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3144 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3145 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3146 				NUM_BANKS(ADDR_SURF_16_BANK));
3147 
3148 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3149 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3150 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3151 				NUM_BANKS(ADDR_SURF_8_BANK));
3152 
3153 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3154 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3155 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3156 				NUM_BANKS(ADDR_SURF_4_BANK));
3157 
3158 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3159 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3160 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3161 				NUM_BANKS(ADDR_SURF_4_BANK));
3162 
3163 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3164 			WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3165 
3166 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3167 			if (reg_offset != 7)
3168 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3169 
3170 		break;
3171 	case CHIP_STONEY:
3172 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3173 				PIPE_CONFIG(ADDR_SURF_P2) |
3174 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3175 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3176 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3177 				PIPE_CONFIG(ADDR_SURF_P2) |
3178 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3179 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3180 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3181 				PIPE_CONFIG(ADDR_SURF_P2) |
3182 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3183 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3184 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3185 				PIPE_CONFIG(ADDR_SURF_P2) |
3186 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3187 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3188 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3189 				PIPE_CONFIG(ADDR_SURF_P2) |
3190 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3191 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3192 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3193 				PIPE_CONFIG(ADDR_SURF_P2) |
3194 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3195 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3196 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3197 				PIPE_CONFIG(ADDR_SURF_P2) |
3198 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3199 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3200 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3201 				PIPE_CONFIG(ADDR_SURF_P2));
3202 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3203 				PIPE_CONFIG(ADDR_SURF_P2) |
3204 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3205 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3206 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3207 				 PIPE_CONFIG(ADDR_SURF_P2) |
3208 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3209 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3210 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3211 				 PIPE_CONFIG(ADDR_SURF_P2) |
3212 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3213 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3214 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3215 				 PIPE_CONFIG(ADDR_SURF_P2) |
3216 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3217 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3218 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3219 				 PIPE_CONFIG(ADDR_SURF_P2) |
3220 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3221 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3222 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3223 				 PIPE_CONFIG(ADDR_SURF_P2) |
3224 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3225 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3226 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3227 				 PIPE_CONFIG(ADDR_SURF_P2) |
3228 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3229 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3230 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3231 				 PIPE_CONFIG(ADDR_SURF_P2) |
3232 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3233 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3234 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3235 				 PIPE_CONFIG(ADDR_SURF_P2) |
3236 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3237 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3238 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3239 				 PIPE_CONFIG(ADDR_SURF_P2) |
3240 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3241 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3242 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3243 				 PIPE_CONFIG(ADDR_SURF_P2) |
3244 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3245 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3246 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3247 				 PIPE_CONFIG(ADDR_SURF_P2) |
3248 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3249 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3250 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3251 				 PIPE_CONFIG(ADDR_SURF_P2) |
3252 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3253 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3254 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3255 				 PIPE_CONFIG(ADDR_SURF_P2) |
3256 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3257 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3258 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3259 				 PIPE_CONFIG(ADDR_SURF_P2) |
3260 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3261 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3262 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3263 				 PIPE_CONFIG(ADDR_SURF_P2) |
3264 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3265 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3266 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3267 				 PIPE_CONFIG(ADDR_SURF_P2) |
3268 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3269 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3270 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3271 				 PIPE_CONFIG(ADDR_SURF_P2) |
3272 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3273 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3274 
3275 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3276 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3277 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3278 				NUM_BANKS(ADDR_SURF_8_BANK));
3279 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3280 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3281 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3282 				NUM_BANKS(ADDR_SURF_8_BANK));
3283 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3284 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3285 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3286 				NUM_BANKS(ADDR_SURF_8_BANK));
3287 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3288 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3289 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3290 				NUM_BANKS(ADDR_SURF_8_BANK));
3291 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3292 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3293 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3294 				NUM_BANKS(ADDR_SURF_8_BANK));
3295 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3296 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3297 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3298 				NUM_BANKS(ADDR_SURF_8_BANK));
3299 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3300 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3301 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3302 				NUM_BANKS(ADDR_SURF_8_BANK));
3303 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3304 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3305 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3306 				NUM_BANKS(ADDR_SURF_16_BANK));
3307 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3308 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3309 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3310 				NUM_BANKS(ADDR_SURF_16_BANK));
3311 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3312 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3313 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3314 				 NUM_BANKS(ADDR_SURF_16_BANK));
3315 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3316 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3317 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3318 				 NUM_BANKS(ADDR_SURF_16_BANK));
3319 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3320 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3321 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3322 				 NUM_BANKS(ADDR_SURF_16_BANK));
3323 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3324 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3325 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3326 				 NUM_BANKS(ADDR_SURF_16_BANK));
3327 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3328 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3329 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3330 				 NUM_BANKS(ADDR_SURF_8_BANK));
3331 
3332 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3333 			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3334 			    reg_offset != 23)
3335 				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3336 
3337 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3338 			if (reg_offset != 7)
3339 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3340 
3341 		break;
3342 	default:
3343 		dev_warn(adev->dev,
3344 			 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
3345 			 adev->asic_type);
3346 
3347 	case CHIP_CARRIZO:
3348 		modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3349 				PIPE_CONFIG(ADDR_SURF_P2) |
3350 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3351 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3352 		modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3353 				PIPE_CONFIG(ADDR_SURF_P2) |
3354 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3355 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3356 		modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3357 				PIPE_CONFIG(ADDR_SURF_P2) |
3358 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3359 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3360 		modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3361 				PIPE_CONFIG(ADDR_SURF_P2) |
3362 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3363 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3364 		modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3365 				PIPE_CONFIG(ADDR_SURF_P2) |
3366 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3367 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3368 		modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3369 				PIPE_CONFIG(ADDR_SURF_P2) |
3370 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3371 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3372 		modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3373 				PIPE_CONFIG(ADDR_SURF_P2) |
3374 				TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3375 				MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3376 		modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3377 				PIPE_CONFIG(ADDR_SURF_P2));
3378 		modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3379 				PIPE_CONFIG(ADDR_SURF_P2) |
3380 				MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3381 				SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3382 		modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3383 				 PIPE_CONFIG(ADDR_SURF_P2) |
3384 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3385 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3386 		modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3387 				 PIPE_CONFIG(ADDR_SURF_P2) |
3388 				 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3389 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3390 		modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3391 				 PIPE_CONFIG(ADDR_SURF_P2) |
3392 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3393 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3394 		modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3395 				 PIPE_CONFIG(ADDR_SURF_P2) |
3396 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3397 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3398 		modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3399 				 PIPE_CONFIG(ADDR_SURF_P2) |
3400 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3401 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3402 		modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3403 				 PIPE_CONFIG(ADDR_SURF_P2) |
3404 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3405 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3406 		modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3407 				 PIPE_CONFIG(ADDR_SURF_P2) |
3408 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3409 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3410 		modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3411 				 PIPE_CONFIG(ADDR_SURF_P2) |
3412 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3413 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3414 		modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3415 				 PIPE_CONFIG(ADDR_SURF_P2) |
3416 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3417 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3418 		modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3419 				 PIPE_CONFIG(ADDR_SURF_P2) |
3420 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3421 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3422 		modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3423 				 PIPE_CONFIG(ADDR_SURF_P2) |
3424 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3425 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3426 		modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3427 				 PIPE_CONFIG(ADDR_SURF_P2) |
3428 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3429 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3430 		modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3431 				 PIPE_CONFIG(ADDR_SURF_P2) |
3432 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3433 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3434 		modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3435 				 PIPE_CONFIG(ADDR_SURF_P2) |
3436 				 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3437 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3438 		modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3439 				 PIPE_CONFIG(ADDR_SURF_P2) |
3440 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3441 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3442 		modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3443 				 PIPE_CONFIG(ADDR_SURF_P2) |
3444 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3445 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3446 		modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3447 				 PIPE_CONFIG(ADDR_SURF_P2) |
3448 				 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3449 				 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3450 
3451 		mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3452 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3453 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3454 				NUM_BANKS(ADDR_SURF_8_BANK));
3455 		mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3456 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3457 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3458 				NUM_BANKS(ADDR_SURF_8_BANK));
3459 		mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3460 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3461 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3462 				NUM_BANKS(ADDR_SURF_8_BANK));
3463 		mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3464 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3465 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3466 				NUM_BANKS(ADDR_SURF_8_BANK));
3467 		mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3468 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3469 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3470 				NUM_BANKS(ADDR_SURF_8_BANK));
3471 		mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3472 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3473 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3474 				NUM_BANKS(ADDR_SURF_8_BANK));
3475 		mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3476 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3477 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3478 				NUM_BANKS(ADDR_SURF_8_BANK));
3479 		mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3480 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3481 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3482 				NUM_BANKS(ADDR_SURF_16_BANK));
3483 		mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3484 				BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3485 				MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3486 				NUM_BANKS(ADDR_SURF_16_BANK));
3487 		mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3488 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3489 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3490 				 NUM_BANKS(ADDR_SURF_16_BANK));
3491 		mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3492 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3493 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3494 				 NUM_BANKS(ADDR_SURF_16_BANK));
3495 		mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3496 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3497 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3498 				 NUM_BANKS(ADDR_SURF_16_BANK));
3499 		mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3500 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3501 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3502 				 NUM_BANKS(ADDR_SURF_16_BANK));
3503 		mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3504 				 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3505 				 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3506 				 NUM_BANKS(ADDR_SURF_8_BANK));
3507 
3508 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3509 			if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3510 			    reg_offset != 23)
3511 				WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3512 
3513 		for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3514 			if (reg_offset != 7)
3515 				WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3516 
3517 		break;
3518 	}
3519 }
3520 
3521 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
3522 				  u32 se_num, u32 sh_num, u32 instance)
3523 {
3524 	u32 data;
3525 
3526 	if (instance == 0xffffffff)
3527 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
3528 	else
3529 		data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
3530 
3531 	if (se_num == 0xffffffff)
3532 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
3533 	else
3534 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
3535 
3536 	if (sh_num == 0xffffffff)
3537 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
3538 	else
3539 		data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
3540 
3541 	WREG32(mmGRBM_GFX_INDEX, data);
3542 }
3543 
3544 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,
3545 				  u32 me, u32 pipe, u32 q)
3546 {
3547 	vi_srbm_select(adev, me, pipe, q, 0);
3548 }
3549 
3550 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
3551 {
3552 	u32 data, mask;
3553 
3554 	data =  RREG32(mmCC_RB_BACKEND_DISABLE) |
3555 		RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3556 
3557 	data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
3558 
3559 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
3560 					 adev->gfx.config.max_sh_per_se);
3561 
3562 	return (~data) & mask;
3563 }
3564 
3565 static void
3566 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
3567 {
3568 	switch (adev->asic_type) {
3569 	case CHIP_FIJI:
3570 	case CHIP_VEGAM:
3571 		*rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
3572 			  RB_XSEL2(1) | PKR_MAP(2) |
3573 			  PKR_XSEL(1) | PKR_YSEL(1) |
3574 			  SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);
3575 		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
3576 			   SE_PAIR_YSEL(2);
3577 		break;
3578 	case CHIP_TONGA:
3579 	case CHIP_POLARIS10:
3580 		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3581 			  SE_XSEL(1) | SE_YSEL(1);
3582 		*rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |
3583 			   SE_PAIR_YSEL(2);
3584 		break;
3585 	case CHIP_TOPAZ:
3586 	case CHIP_CARRIZO:
3587 		*rconf |= RB_MAP_PKR0(2);
3588 		*rconf1 |= 0x0;
3589 		break;
3590 	case CHIP_POLARIS11:
3591 	case CHIP_POLARIS12:
3592 		*rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3593 			  SE_XSEL(1) | SE_YSEL(1);
3594 		*rconf1 |= 0x0;
3595 		break;
3596 	case CHIP_STONEY:
3597 		*rconf |= 0x0;
3598 		*rconf1 |= 0x0;
3599 		break;
3600 	default:
3601 		DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
3602 		break;
3603 	}
3604 }
3605 
3606 static void
3607 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
3608 					u32 raster_config, u32 raster_config_1,
3609 					unsigned rb_mask, unsigned num_rb)
3610 {
3611 	unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
3612 	unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
3613 	unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
3614 	unsigned rb_per_se = num_rb / num_se;
3615 	unsigned se_mask[4];
3616 	unsigned se;
3617 
3618 	se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
3619 	se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
3620 	se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
3621 	se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
3622 
3623 	WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
3624 	WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
3625 	WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
3626 
3627 	if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
3628 			     (!se_mask[2] && !se_mask[3]))) {
3629 		raster_config_1 &= ~SE_PAIR_MAP_MASK;
3630 
3631 		if (!se_mask[0] && !se_mask[1]) {
3632 			raster_config_1 |=
3633 				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
3634 		} else {
3635 			raster_config_1 |=
3636 				SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
3637 		}
3638 	}
3639 
3640 	for (se = 0; se < num_se; se++) {
3641 		unsigned raster_config_se = raster_config;
3642 		unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
3643 		unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
3644 		int idx = (se / 2) * 2;
3645 
3646 		if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
3647 			raster_config_se &= ~SE_MAP_MASK;
3648 
3649 			if (!se_mask[idx]) {
3650 				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
3651 			} else {
3652 				raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
3653 			}
3654 		}
3655 
3656 		pkr0_mask &= rb_mask;
3657 		pkr1_mask &= rb_mask;
3658 		if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
3659 			raster_config_se &= ~PKR_MAP_MASK;
3660 
3661 			if (!pkr0_mask) {
3662 				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
3663 			} else {
3664 				raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
3665 			}
3666 		}
3667 
3668 		if (rb_per_se >= 2) {
3669 			unsigned rb0_mask = 1 << (se * rb_per_se);
3670 			unsigned rb1_mask = rb0_mask << 1;
3671 
3672 			rb0_mask &= rb_mask;
3673 			rb1_mask &= rb_mask;
3674 			if (!rb0_mask || !rb1_mask) {
3675 				raster_config_se &= ~RB_MAP_PKR0_MASK;
3676 
3677 				if (!rb0_mask) {
3678 					raster_config_se |=
3679 						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
3680 				} else {
3681 					raster_config_se |=
3682 						RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
3683 				}
3684 			}
3685 
3686 			if (rb_per_se > 2) {
3687 				rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
3688 				rb1_mask = rb0_mask << 1;
3689 				rb0_mask &= rb_mask;
3690 				rb1_mask &= rb_mask;
3691 				if (!rb0_mask || !rb1_mask) {
3692 					raster_config_se &= ~RB_MAP_PKR1_MASK;
3693 
3694 					if (!rb0_mask) {
3695 						raster_config_se |=
3696 							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
3697 					} else {
3698 						raster_config_se |=
3699 							RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
3700 					}
3701 				}
3702 			}
3703 		}
3704 
3705 		/* GRBM_GFX_INDEX has a different offset on VI */
3706 		gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff);
3707 		WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
3708 		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3709 	}
3710 
3711 	/* GRBM_GFX_INDEX has a different offset on VI */
3712 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3713 }
3714 
3715 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
3716 {
3717 	int i, j;
3718 	u32 data;
3719 	u32 raster_config = 0, raster_config_1 = 0;
3720 	u32 active_rbs = 0;
3721 	u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
3722 					adev->gfx.config.max_sh_per_se;
3723 	unsigned num_rb_pipes;
3724 
3725 	mutex_lock(&adev->grbm_idx_mutex);
3726 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3727 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3728 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3729 			data = gfx_v8_0_get_rb_active_bitmap(adev);
3730 			active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
3731 					       rb_bitmap_width_per_sh);
3732 		}
3733 	}
3734 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3735 
3736 	adev->gfx.config.backend_enable_mask = active_rbs;
3737 	adev->gfx.config.num_rbs = hweight32(active_rbs);
3738 
3739 	num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
3740 			     adev->gfx.config.max_shader_engines, 16);
3741 
3742 	gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);
3743 
3744 	if (!adev->gfx.config.backend_enable_mask ||
3745 			adev->gfx.config.num_rbs >= num_rb_pipes) {
3746 		WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
3747 		WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3748 	} else {
3749 		gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
3750 							adev->gfx.config.backend_enable_mask,
3751 							num_rb_pipes);
3752 	}
3753 
3754 	/* cache the values for userspace */
3755 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3756 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3757 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3758 			adev->gfx.config.rb_config[i][j].rb_backend_disable =
3759 				RREG32(mmCC_RB_BACKEND_DISABLE);
3760 			adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
3761 				RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3762 			adev->gfx.config.rb_config[i][j].raster_config =
3763 				RREG32(mmPA_SC_RASTER_CONFIG);
3764 			adev->gfx.config.rb_config[i][j].raster_config_1 =
3765 				RREG32(mmPA_SC_RASTER_CONFIG_1);
3766 		}
3767 	}
3768 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3769 	mutex_unlock(&adev->grbm_idx_mutex);
3770 }
3771 
3772 /**
3773  * gfx_v8_0_init_compute_vmid - gart enable
3774  *
3775  * @adev: amdgpu_device pointer
3776  *
3777  * Initialize compute vmid sh_mem registers
3778  *
3779  */
3780 #define DEFAULT_SH_MEM_BASES	(0x6000)
3781 #define FIRST_COMPUTE_VMID	(8)
3782 #define LAST_COMPUTE_VMID	(16)
3783 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
3784 {
3785 	int i;
3786 	uint32_t sh_mem_config;
3787 	uint32_t sh_mem_bases;
3788 
3789 	/*
3790 	 * Configure apertures:
3791 	 * LDS:         0x60000000'00000000 - 0x60000001'00000000 (4GB)
3792 	 * Scratch:     0x60000001'00000000 - 0x60000002'00000000 (4GB)
3793 	 * GPUVM:       0x60010000'00000000 - 0x60020000'00000000 (1TB)
3794 	 */
3795 	sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
3796 
3797 	sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 <<
3798 			SH_MEM_CONFIG__ADDRESS_MODE__SHIFT |
3799 			SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
3800 			SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
3801 			MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
3802 			SH_MEM_CONFIG__PRIVATE_ATC_MASK;
3803 
3804 	mutex_lock(&adev->srbm_mutex);
3805 	for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
3806 		vi_srbm_select(adev, 0, 0, 0, i);
3807 		/* CP and shaders */
3808 		WREG32(mmSH_MEM_CONFIG, sh_mem_config);
3809 		WREG32(mmSH_MEM_APE1_BASE, 1);
3810 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
3811 		WREG32(mmSH_MEM_BASES, sh_mem_bases);
3812 	}
3813 	vi_srbm_select(adev, 0, 0, 0, 0);
3814 	mutex_unlock(&adev->srbm_mutex);
3815 }
3816 
3817 static void gfx_v8_0_config_init(struct amdgpu_device *adev)
3818 {
3819 	switch (adev->asic_type) {
3820 	default:
3821 		adev->gfx.config.double_offchip_lds_buf = 1;
3822 		break;
3823 	case CHIP_CARRIZO:
3824 	case CHIP_STONEY:
3825 		adev->gfx.config.double_offchip_lds_buf = 0;
3826 		break;
3827 	}
3828 }
3829 
3830 static void gfx_v8_0_constants_init(struct amdgpu_device *adev)
3831 {
3832 	u32 tmp, sh_static_mem_cfg;
3833 	int i;
3834 
3835 	WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);
3836 	WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3837 	WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3838 	WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
3839 
3840 	gfx_v8_0_tiling_mode_table_init(adev);
3841 	gfx_v8_0_setup_rb(adev);
3842 	gfx_v8_0_get_cu_info(adev);
3843 	gfx_v8_0_config_init(adev);
3844 
3845 	/* XXX SH_MEM regs */
3846 	/* where to put LDS, scratch, GPUVM in FSA64 space */
3847 	sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
3848 				   SWIZZLE_ENABLE, 1);
3849 	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3850 				   ELEMENT_SIZE, 1);
3851 	sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3852 				   INDEX_STRIDE, 3);
3853 	WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
3854 
3855 	mutex_lock(&adev->srbm_mutex);
3856 	for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
3857 		vi_srbm_select(adev, 0, 0, 0, i);
3858 		/* CP and shaders */
3859 		if (i == 0) {
3860 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);
3861 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3862 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3863 					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3864 			WREG32(mmSH_MEM_CONFIG, tmp);
3865 			WREG32(mmSH_MEM_BASES, 0);
3866 		} else {
3867 			tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);
3868 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3869 			tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3870 					    SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3871 			WREG32(mmSH_MEM_CONFIG, tmp);
3872 			tmp = adev->gmc.shared_aperture_start >> 48;
3873 			WREG32(mmSH_MEM_BASES, tmp);
3874 		}
3875 
3876 		WREG32(mmSH_MEM_APE1_BASE, 1);
3877 		WREG32(mmSH_MEM_APE1_LIMIT, 0);
3878 	}
3879 	vi_srbm_select(adev, 0, 0, 0, 0);
3880 	mutex_unlock(&adev->srbm_mutex);
3881 
3882 	gfx_v8_0_init_compute_vmid(adev);
3883 
3884 	mutex_lock(&adev->grbm_idx_mutex);
3885 	/*
3886 	 * making sure that the following register writes will be broadcasted
3887 	 * to all the shaders
3888 	 */
3889 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3890 
3891 	WREG32(mmPA_SC_FIFO_SIZE,
3892 		   (adev->gfx.config.sc_prim_fifo_size_frontend <<
3893 			PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
3894 		   (adev->gfx.config.sc_prim_fifo_size_backend <<
3895 			PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
3896 		   (adev->gfx.config.sc_hiz_tile_fifo_size <<
3897 			PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
3898 		   (adev->gfx.config.sc_earlyz_tile_fifo_size <<
3899 			PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
3900 
3901 	tmp = RREG32(mmSPI_ARB_PRIORITY);
3902 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
3903 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
3904 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
3905 	tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
3906 	WREG32(mmSPI_ARB_PRIORITY, tmp);
3907 
3908 	mutex_unlock(&adev->grbm_idx_mutex);
3909 
3910 }
3911 
3912 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3913 {
3914 	u32 i, j, k;
3915 	u32 mask;
3916 
3917 	mutex_lock(&adev->grbm_idx_mutex);
3918 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3919 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3920 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
3921 			for (k = 0; k < adev->usec_timeout; k++) {
3922 				if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3923 					break;
3924 				udelay(1);
3925 			}
3926 			if (k == adev->usec_timeout) {
3927 				gfx_v8_0_select_se_sh(adev, 0xffffffff,
3928 						      0xffffffff, 0xffffffff);
3929 				mutex_unlock(&adev->grbm_idx_mutex);
3930 				DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
3931 					 i, j);
3932 				return;
3933 			}
3934 		}
3935 	}
3936 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
3937 	mutex_unlock(&adev->grbm_idx_mutex);
3938 
3939 	mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3940 		RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3941 		RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3942 		RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3943 	for (k = 0; k < adev->usec_timeout; k++) {
3944 		if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3945 			break;
3946 		udelay(1);
3947 	}
3948 }
3949 
3950 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3951 					       bool enable)
3952 {
3953 	u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3954 
3955 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
3956 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
3957 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
3958 	tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
3959 
3960 	WREG32(mmCP_INT_CNTL_RING0, tmp);
3961 }
3962 
3963 static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
3964 {
3965 	/* csib */
3966 	WREG32(mmRLC_CSIB_ADDR_HI,
3967 			adev->gfx.rlc.clear_state_gpu_addr >> 32);
3968 	WREG32(mmRLC_CSIB_ADDR_LO,
3969 			adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
3970 	WREG32(mmRLC_CSIB_LENGTH,
3971 			adev->gfx.rlc.clear_state_size);
3972 }
3973 
3974 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,
3975 				int ind_offset,
3976 				int list_size,
3977 				int *unique_indices,
3978 				int *indices_count,
3979 				int max_indices,
3980 				int *ind_start_offsets,
3981 				int *offset_count,
3982 				int max_offset)
3983 {
3984 	int indices;
3985 	bool new_entry = true;
3986 
3987 	for (; ind_offset < list_size; ind_offset++) {
3988 
3989 		if (new_entry) {
3990 			new_entry = false;
3991 			ind_start_offsets[*offset_count] = ind_offset;
3992 			*offset_count = *offset_count + 1;
3993 			BUG_ON(*offset_count >= max_offset);
3994 		}
3995 
3996 		if (register_list_format[ind_offset] == 0xFFFFFFFF) {
3997 			new_entry = true;
3998 			continue;
3999 		}
4000 
4001 		ind_offset += 2;
4002 
4003 		/* look for the matching indice */
4004 		for (indices = 0;
4005 			indices < *indices_count;
4006 			indices++) {
4007 			if (unique_indices[indices] ==
4008 				register_list_format[ind_offset])
4009 				break;
4010 		}
4011 
4012 		if (indices >= *indices_count) {
4013 			unique_indices[*indices_count] =
4014 				register_list_format[ind_offset];
4015 			indices = *indices_count;
4016 			*indices_count = *indices_count + 1;
4017 			BUG_ON(*indices_count >= max_indices);
4018 		}
4019 
4020 		register_list_format[ind_offset] = indices;
4021 	}
4022 }
4023 
4024 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
4025 {
4026 	int i, temp, data;
4027 	int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0};
4028 	int indices_count = 0;
4029 	int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
4030 	int offset_count = 0;
4031 
4032 	int list_size;
4033 	unsigned int *register_list_format =
4034 		kmalloc(adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
4035 	if (!register_list_format)
4036 		return -ENOMEM;
4037 	memcpy(register_list_format, adev->gfx.rlc.register_list_format,
4038 			adev->gfx.rlc.reg_list_format_size_bytes);
4039 
4040 	gfx_v8_0_parse_ind_reg_list(register_list_format,
4041 				RLC_FormatDirectRegListLength,
4042 				adev->gfx.rlc.reg_list_format_size_bytes >> 2,
4043 				unique_indices,
4044 				&indices_count,
4045 				ARRAY_SIZE(unique_indices),
4046 				indirect_start_offsets,
4047 				&offset_count,
4048 				ARRAY_SIZE(indirect_start_offsets));
4049 
4050 	/* save and restore list */
4051 	WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
4052 
4053 	WREG32(mmRLC_SRM_ARAM_ADDR, 0);
4054 	for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
4055 		WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]);
4056 
4057 	/* indirect list */
4058 	WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start);
4059 	for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)
4060 		WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]);
4061 
4062 	list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
4063 	list_size = list_size >> 1;
4064 	WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size);
4065 	WREG32(mmRLC_GPM_SCRATCH_DATA, list_size);
4066 
4067 	/* starting offsets starts */
4068 	WREG32(mmRLC_GPM_SCRATCH_ADDR,
4069 		adev->gfx.rlc.starting_offsets_start);
4070 	for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
4071 		WREG32(mmRLC_GPM_SCRATCH_DATA,
4072 				indirect_start_offsets[i]);
4073 
4074 	/* unique indices */
4075 	temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
4076 	data = mmRLC_SRM_INDEX_CNTL_DATA_0;
4077 	for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {
4078 		if (unique_indices[i] != 0) {
4079 			WREG32(temp + i, unique_indices[i] & 0x3FFFF);
4080 			WREG32(data + i, unique_indices[i] >> 20);
4081 		}
4082 	}
4083 	kfree(register_list_format);
4084 
4085 	return 0;
4086 }
4087 
4088 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
4089 {
4090 	WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);
4091 }
4092 
4093 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
4094 {
4095 	uint32_t data;
4096 
4097 	WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);
4098 
4099 	data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);
4100 	data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);
4101 	data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);
4102 	data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);
4103 	WREG32(mmRLC_PG_DELAY, data);
4104 
4105 	WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);
4106 	WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);
4107 
4108 }
4109 
4110 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
4111 						bool enable)
4112 {
4113 	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);
4114 }
4115 
4116 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
4117 						  bool enable)
4118 {
4119 	WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);
4120 }
4121 
4122 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
4123 {
4124 	WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1);
4125 }
4126 
4127 static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
4128 {
4129 	if ((adev->asic_type == CHIP_CARRIZO) ||
4130 	    (adev->asic_type == CHIP_STONEY)) {
4131 		gfx_v8_0_init_csb(adev);
4132 		gfx_v8_0_init_save_restore_list(adev);
4133 		gfx_v8_0_enable_save_restore_machine(adev);
4134 		WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
4135 		gfx_v8_0_init_power_gating(adev);
4136 		WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
4137 	} else if ((adev->asic_type == CHIP_POLARIS11) ||
4138 		   (adev->asic_type == CHIP_POLARIS12) ||
4139 		   (adev->asic_type == CHIP_VEGAM)) {
4140 		gfx_v8_0_init_csb(adev);
4141 		gfx_v8_0_init_save_restore_list(adev);
4142 		gfx_v8_0_enable_save_restore_machine(adev);
4143 		gfx_v8_0_init_power_gating(adev);
4144 	}
4145 
4146 }
4147 
4148 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
4149 {
4150 	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);
4151 
4152 	gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4153 	gfx_v8_0_wait_for_rlc_serdes(adev);
4154 }
4155 
4156 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
4157 {
4158 	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4159 	udelay(50);
4160 
4161 	WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
4162 	udelay(50);
4163 }
4164 
4165 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
4166 {
4167 	WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);
4168 
4169 	/* carrizo do enable cp interrupt after cp inited */
4170 	if (!(adev->flags & AMD_IS_APU))
4171 		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4172 
4173 	udelay(50);
4174 }
4175 
4176 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
4177 {
4178 	gfx_v8_0_rlc_stop(adev);
4179 	gfx_v8_0_rlc_reset(adev);
4180 	gfx_v8_0_init_pg(adev);
4181 	gfx_v8_0_rlc_start(adev);
4182 
4183 	return 0;
4184 }
4185 
4186 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
4187 {
4188 	int i;
4189 	u32 tmp = RREG32(mmCP_ME_CNTL);
4190 
4191 	if (enable) {
4192 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);
4193 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);
4194 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);
4195 	} else {
4196 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
4197 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
4198 		tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
4199 		for (i = 0; i < adev->gfx.num_gfx_rings; i++)
4200 			adev->gfx.gfx_ring[i].ready = false;
4201 	}
4202 	WREG32(mmCP_ME_CNTL, tmp);
4203 	udelay(50);
4204 }
4205 
4206 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)
4207 {
4208 	u32 count = 0;
4209 	const struct cs_section_def *sect = NULL;
4210 	const struct cs_extent_def *ext = NULL;
4211 
4212 	/* begin clear state */
4213 	count += 2;
4214 	/* context control state */
4215 	count += 3;
4216 
4217 	for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4218 		for (ext = sect->section; ext->extent != NULL; ++ext) {
4219 			if (sect->id == SECT_CONTEXT)
4220 				count += 2 + ext->reg_count;
4221 			else
4222 				return 0;
4223 		}
4224 	}
4225 	/* pa_sc_raster_config/pa_sc_raster_config1 */
4226 	count += 4;
4227 	/* end clear state */
4228 	count += 2;
4229 	/* clear state */
4230 	count += 2;
4231 
4232 	return count;
4233 }
4234 
4235 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
4236 {
4237 	struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
4238 	const struct cs_section_def *sect = NULL;
4239 	const struct cs_extent_def *ext = NULL;
4240 	int r, i;
4241 
4242 	/* init the CP */
4243 	WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
4244 	WREG32(mmCP_ENDIAN_SWAP, 0);
4245 	WREG32(mmCP_DEVICE_ID, 1);
4246 
4247 	gfx_v8_0_cp_gfx_enable(adev, true);
4248 
4249 	r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4);
4250 	if (r) {
4251 		DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
4252 		return r;
4253 	}
4254 
4255 	/* clear state buffer */
4256 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4257 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4258 
4259 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4260 	amdgpu_ring_write(ring, 0x80000000);
4261 	amdgpu_ring_write(ring, 0x80000000);
4262 
4263 	for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4264 		for (ext = sect->section; ext->extent != NULL; ++ext) {
4265 			if (sect->id == SECT_CONTEXT) {
4266 				amdgpu_ring_write(ring,
4267 				       PACKET3(PACKET3_SET_CONTEXT_REG,
4268 					       ext->reg_count));
4269 				amdgpu_ring_write(ring,
4270 				       ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
4271 				for (i = 0; i < ext->reg_count; i++)
4272 					amdgpu_ring_write(ring, ext->extent[i]);
4273 			}
4274 		}
4275 	}
4276 
4277 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4278 	amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
4279 	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
4280 	amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
4281 
4282 	amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4283 	amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4284 
4285 	amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4286 	amdgpu_ring_write(ring, 0);
4287 
4288 	/* init the CE partitions */
4289 	amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4290 	amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4291 	amdgpu_ring_write(ring, 0x8000);
4292 	amdgpu_ring_write(ring, 0x8000);
4293 
4294 	amdgpu_ring_commit(ring);
4295 
4296 	return 0;
4297 }
4298 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring)
4299 {
4300 	u32 tmp;
4301 	/* no gfx doorbells on iceland */
4302 	if (adev->asic_type == CHIP_TOPAZ)
4303 		return;
4304 
4305 	tmp = RREG32(mmCP_RB_DOORBELL_CONTROL);
4306 
4307 	if (ring->use_doorbell) {
4308 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4309 				DOORBELL_OFFSET, ring->doorbell_index);
4310 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4311 						DOORBELL_HIT, 0);
4312 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4313 					    DOORBELL_EN, 1);
4314 	} else {
4315 		tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
4316 	}
4317 
4318 	WREG32(mmCP_RB_DOORBELL_CONTROL, tmp);
4319 
4320 	if (adev->flags & AMD_IS_APU)
4321 		return;
4322 
4323 	tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
4324 					DOORBELL_RANGE_LOWER,
4325 					AMDGPU_DOORBELL_GFX_RING0);
4326 	WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
4327 
4328 	WREG32(mmCP_RB_DOORBELL_RANGE_UPPER,
4329 		CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
4330 }
4331 
4332 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
4333 {
4334 	struct amdgpu_ring *ring;
4335 	u32 tmp;
4336 	u32 rb_bufsz;
4337 	u64 rb_addr, rptr_addr, wptr_gpu_addr;
4338 	int r;
4339 
4340 	/* Set the write pointer delay */
4341 	WREG32(mmCP_RB_WPTR_DELAY, 0);
4342 
4343 	/* set the RB to use vmid 0 */
4344 	WREG32(mmCP_RB_VMID, 0);
4345 
4346 	/* Set ring buffer size */
4347 	ring = &adev->gfx.gfx_ring[0];
4348 	rb_bufsz = order_base_2(ring->ring_size / 8);
4349 	tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
4350 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
4351 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3);
4352 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1);
4353 #ifdef __BIG_ENDIAN
4354 	tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
4355 #endif
4356 	WREG32(mmCP_RB0_CNTL, tmp);
4357 
4358 	/* Initialize the ring buffer's read and write pointers */
4359 	WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
4360 	ring->wptr = 0;
4361 	WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
4362 
4363 	/* set the wb address wether it's enabled or not */
4364 	rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
4365 	WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
4366 	WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
4367 
4368 	wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4369 	WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
4370 	WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
4371 	mdelay(1);
4372 	WREG32(mmCP_RB0_CNTL, tmp);
4373 
4374 	rb_addr = ring->gpu_addr >> 8;
4375 	WREG32(mmCP_RB0_BASE, rb_addr);
4376 	WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
4377 
4378 	gfx_v8_0_set_cpg_door_bell(adev, ring);
4379 	/* start the ring */
4380 	amdgpu_ring_clear_ring(ring);
4381 	gfx_v8_0_cp_gfx_start(adev);
4382 	ring->ready = true;
4383 	r = amdgpu_ring_test_ring(ring);
4384 	if (r)
4385 		ring->ready = false;
4386 
4387 	return r;
4388 }
4389 
4390 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
4391 {
4392 	int i;
4393 
4394 	if (enable) {
4395 		WREG32(mmCP_MEC_CNTL, 0);
4396 	} else {
4397 		WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
4398 		for (i = 0; i < adev->gfx.num_compute_rings; i++)
4399 			adev->gfx.compute_ring[i].ready = false;
4400 		adev->gfx.kiq.ring.ready = false;
4401 	}
4402 	udelay(50);
4403 }
4404 
4405 /* KIQ functions */
4406 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
4407 {
4408 	uint32_t tmp;
4409 	struct amdgpu_device *adev = ring->adev;
4410 
4411 	/* tell RLC which is KIQ queue */
4412 	tmp = RREG32(mmRLC_CP_SCHEDULERS);
4413 	tmp &= 0xffffff00;
4414 	tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4415 	WREG32(mmRLC_CP_SCHEDULERS, tmp);
4416 	tmp |= 0x80;
4417 	WREG32(mmRLC_CP_SCHEDULERS, tmp);
4418 }
4419 
4420 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)
4421 {
4422 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4423 	uint64_t queue_mask = 0;
4424 	int r, i;
4425 
4426 	for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
4427 		if (!test_bit(i, adev->gfx.mec.queue_bitmap))
4428 			continue;
4429 
4430 		/* This situation may be hit in the future if a new HW
4431 		 * generation exposes more than 64 queues. If so, the
4432 		 * definition of queue_mask needs updating */
4433 		if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
4434 			DRM_ERROR("Invalid KCQ enabled: %d\n", i);
4435 			break;
4436 		}
4437 
4438 		queue_mask |= (1ull << i);
4439 	}
4440 
4441 	r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8);
4442 	if (r) {
4443 		DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4444 		return r;
4445 	}
4446 	/* set resources */
4447 	amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
4448 	amdgpu_ring_write(kiq_ring, 0);	/* vmid_mask:0 queue_type:0 (KIQ) */
4449 	amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask));	/* queue mask lo */
4450 	amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask));	/* queue mask hi */
4451 	amdgpu_ring_write(kiq_ring, 0);	/* gws mask lo */
4452 	amdgpu_ring_write(kiq_ring, 0);	/* gws mask hi */
4453 	amdgpu_ring_write(kiq_ring, 0);	/* oac mask */
4454 	amdgpu_ring_write(kiq_ring, 0);	/* gds heap base:0, gds heap size:0 */
4455 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4456 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4457 		uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
4458 		uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4459 
4460 		/* map queues */
4461 		amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
4462 		/* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
4463 		amdgpu_ring_write(kiq_ring,
4464 				  PACKET3_MAP_QUEUES_NUM_QUEUES(1));
4465 		amdgpu_ring_write(kiq_ring,
4466 				  PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) |
4467 				  PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
4468 				  PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
4469 				  PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */
4470 		amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
4471 		amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
4472 		amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
4473 		amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
4474 	}
4475 
4476 	r = amdgpu_ring_test_ring(kiq_ring);
4477 	if (r) {
4478 		DRM_ERROR("KCQ enable failed\n");
4479 		kiq_ring->ready = false;
4480 	}
4481 	return r;
4482 }
4483 
4484 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req)
4485 {
4486 	int i, r = 0;
4487 
4488 	if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
4489 		WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req);
4490 		for (i = 0; i < adev->usec_timeout; i++) {
4491 			if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
4492 				break;
4493 			udelay(1);
4494 		}
4495 		if (i == adev->usec_timeout)
4496 			r = -ETIMEDOUT;
4497 	}
4498 	WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
4499 	WREG32(mmCP_HQD_PQ_RPTR, 0);
4500 	WREG32(mmCP_HQD_PQ_WPTR, 0);
4501 
4502 	return r;
4503 }
4504 
4505 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
4506 {
4507 	struct amdgpu_device *adev = ring->adev;
4508 	struct vi_mqd *mqd = ring->mqd_ptr;
4509 	uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4510 	uint32_t tmp;
4511 
4512 	mqd->header = 0xC0310800;
4513 	mqd->compute_pipelinestat_enable = 0x00000001;
4514 	mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4515 	mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4516 	mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4517 	mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4518 	mqd->compute_misc_reserved = 0x00000003;
4519 	mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
4520 						     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4521 	mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
4522 						     + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4523 	eop_base_addr = ring->eop_gpu_addr >> 8;
4524 	mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4525 	mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4526 
4527 	/* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4528 	tmp = RREG32(mmCP_HQD_EOP_CONTROL);
4529 	tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4530 			(order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1));
4531 
4532 	mqd->cp_hqd_eop_control = tmp;
4533 
4534 	/* enable doorbell? */
4535 	tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL),
4536 			    CP_HQD_PQ_DOORBELL_CONTROL,
4537 			    DOORBELL_EN,
4538 			    ring->use_doorbell ? 1 : 0);
4539 
4540 	mqd->cp_hqd_pq_doorbell_control = tmp;
4541 
4542 	/* set the pointer to the MQD */
4543 	mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
4544 	mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
4545 
4546 	/* set MQD vmid to 0 */
4547 	tmp = RREG32(mmCP_MQD_CONTROL);
4548 	tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4549 	mqd->cp_mqd_control = tmp;
4550 
4551 	/* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4552 	hqd_gpu_addr = ring->gpu_addr >> 8;
4553 	mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4554 	mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4555 
4556 	/* set up the HQD, this is similar to CP_RB0_CNTL */
4557 	tmp = RREG32(mmCP_HQD_PQ_CONTROL);
4558 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4559 			    (order_base_2(ring->ring_size / 4) - 1));
4560 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4561 			((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
4562 #ifdef __BIG_ENDIAN
4563 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
4564 #endif
4565 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
4566 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
4567 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4568 	tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4569 	mqd->cp_hqd_pq_control = tmp;
4570 
4571 	/* set the wb address whether it's enabled or not */
4572 	wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
4573 	mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4574 	mqd->cp_hqd_pq_rptr_report_addr_hi =
4575 		upper_32_bits(wb_gpu_addr) & 0xffff;
4576 
4577 	/* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4578 	wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
4579 	mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4580 	mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4581 
4582 	tmp = 0;
4583 	/* enable the doorbell if requested */
4584 	if (ring->use_doorbell) {
4585 		tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
4586 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4587 				DOORBELL_OFFSET, ring->doorbell_index);
4588 
4589 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4590 					 DOORBELL_EN, 1);
4591 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4592 					 DOORBELL_SOURCE, 0);
4593 		tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4594 					 DOORBELL_HIT, 0);
4595 	}
4596 
4597 	mqd->cp_hqd_pq_doorbell_control = tmp;
4598 
4599 	/* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4600 	ring->wptr = 0;
4601 	mqd->cp_hqd_pq_wptr = ring->wptr;
4602 	mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
4603 
4604 	/* set the vmid for the queue */
4605 	mqd->cp_hqd_vmid = 0;
4606 
4607 	tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
4608 	tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
4609 	mqd->cp_hqd_persistent_state = tmp;
4610 
4611 	/* set MTYPE */
4612 	tmp = RREG32(mmCP_HQD_IB_CONTROL);
4613 	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4614 	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3);
4615 	mqd->cp_hqd_ib_control = tmp;
4616 
4617 	tmp = RREG32(mmCP_HQD_IQ_TIMER);
4618 	tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3);
4619 	mqd->cp_hqd_iq_timer = tmp;
4620 
4621 	tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL);
4622 	tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3);
4623 	mqd->cp_hqd_ctx_save_control = tmp;
4624 
4625 	/* defaults */
4626 	mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR);
4627 	mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR);
4628 	mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY);
4629 	mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY);
4630 	mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
4631 	mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO);
4632 	mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI);
4633 	mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET);
4634 	mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE);
4635 	mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET);
4636 	mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE);
4637 	mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS);
4638 	mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR);
4639 	mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM);
4640 	mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES);
4641 
4642 	/* activate the queue */
4643 	mqd->cp_hqd_active = 1;
4644 
4645 	return 0;
4646 }
4647 
4648 int gfx_v8_0_mqd_commit(struct amdgpu_device *adev,
4649 			struct vi_mqd *mqd)
4650 {
4651 	uint32_t mqd_reg;
4652 	uint32_t *mqd_data;
4653 
4654 	/* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */
4655 	mqd_data = &mqd->cp_mqd_base_addr_lo;
4656 
4657 	/* disable wptr polling */
4658 	WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0);
4659 
4660 	/* program all HQD registers */
4661 	for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++)
4662 		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4663 
4664 	/* Tonga errata: EOP RPTR/WPTR should be left unmodified.
4665 	 * This is safe since EOP RPTR==WPTR for any inactive HQD
4666 	 * on ASICs that do not support context-save.
4667 	 * EOP writes/reads can start anywhere in the ring.
4668 	 */
4669 	if (adev->asic_type != CHIP_TONGA) {
4670 		WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr);
4671 		WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr);
4672 		WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem);
4673 	}
4674 
4675 	for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++)
4676 		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4677 
4678 	/* activate the HQD */
4679 	for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)
4680 		WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4681 
4682 	return 0;
4683 }
4684 
4685 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
4686 {
4687 	struct amdgpu_device *adev = ring->adev;
4688 	struct vi_mqd *mqd = ring->mqd_ptr;
4689 	int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
4690 
4691 	gfx_v8_0_kiq_setting(ring);
4692 
4693 	if (adev->in_gpu_reset) { /* for GPU_RESET case */
4694 		/* reset MQD to a clean status */
4695 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4696 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4697 
4698 		/* reset ring buffer */
4699 		ring->wptr = 0;
4700 		amdgpu_ring_clear_ring(ring);
4701 		mutex_lock(&adev->srbm_mutex);
4702 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4703 		gfx_v8_0_mqd_commit(adev, mqd);
4704 		vi_srbm_select(adev, 0, 0, 0, 0);
4705 		mutex_unlock(&adev->srbm_mutex);
4706 	} else {
4707 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4708 		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4709 		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4710 		mutex_lock(&adev->srbm_mutex);
4711 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4712 		gfx_v8_0_mqd_init(ring);
4713 		gfx_v8_0_mqd_commit(adev, mqd);
4714 		vi_srbm_select(adev, 0, 0, 0, 0);
4715 		mutex_unlock(&adev->srbm_mutex);
4716 
4717 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4718 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4719 	}
4720 
4721 	return 0;
4722 }
4723 
4724 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
4725 {
4726 	struct amdgpu_device *adev = ring->adev;
4727 	struct vi_mqd *mqd = ring->mqd_ptr;
4728 	int mqd_idx = ring - &adev->gfx.compute_ring[0];
4729 
4730 	if (!adev->in_gpu_reset && !adev->in_suspend) {
4731 		memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4732 		((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4733 		((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4734 		mutex_lock(&adev->srbm_mutex);
4735 		vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4736 		gfx_v8_0_mqd_init(ring);
4737 		vi_srbm_select(adev, 0, 0, 0, 0);
4738 		mutex_unlock(&adev->srbm_mutex);
4739 
4740 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4741 			memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4742 	} else if (adev->in_gpu_reset) { /* for GPU_RESET case */
4743 		/* reset MQD to a clean status */
4744 		if (adev->gfx.mec.mqd_backup[mqd_idx])
4745 			memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4746 		/* reset ring buffer */
4747 		ring->wptr = 0;
4748 		amdgpu_ring_clear_ring(ring);
4749 	} else {
4750 		amdgpu_ring_clear_ring(ring);
4751 	}
4752 	return 0;
4753 }
4754 
4755 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev)
4756 {
4757 	if (adev->asic_type > CHIP_TONGA) {
4758 		WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, AMDGPU_DOORBELL_KIQ << 2);
4759 		WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, AMDGPU_DOORBELL_MEC_RING7 << 2);
4760 	}
4761 	/* enable doorbells */
4762 	WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4763 }
4764 
4765 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
4766 {
4767 	struct amdgpu_ring *ring;
4768 	int r;
4769 
4770 	ring = &adev->gfx.kiq.ring;
4771 
4772 	r = amdgpu_bo_reserve(ring->mqd_obj, false);
4773 	if (unlikely(r != 0))
4774 		return r;
4775 
4776 	r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4777 	if (unlikely(r != 0))
4778 		return r;
4779 
4780 	gfx_v8_0_kiq_init_queue(ring);
4781 	amdgpu_bo_kunmap(ring->mqd_obj);
4782 	ring->mqd_ptr = NULL;
4783 	amdgpu_bo_unreserve(ring->mqd_obj);
4784 	ring->ready = true;
4785 	return 0;
4786 }
4787 
4788 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev)
4789 {
4790 	struct amdgpu_ring *ring = NULL;
4791 	int r = 0, i;
4792 
4793 	gfx_v8_0_cp_compute_enable(adev, true);
4794 
4795 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4796 		ring = &adev->gfx.compute_ring[i];
4797 
4798 		r = amdgpu_bo_reserve(ring->mqd_obj, false);
4799 		if (unlikely(r != 0))
4800 			goto done;
4801 		r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr);
4802 		if (!r) {
4803 			r = gfx_v8_0_kcq_init_queue(ring);
4804 			amdgpu_bo_kunmap(ring->mqd_obj);
4805 			ring->mqd_ptr = NULL;
4806 		}
4807 		amdgpu_bo_unreserve(ring->mqd_obj);
4808 		if (r)
4809 			goto done;
4810 	}
4811 
4812 	gfx_v8_0_set_mec_doorbell_range(adev);
4813 
4814 	r = gfx_v8_0_kiq_kcq_enable(adev);
4815 	if (r)
4816 		goto done;
4817 
4818 	/* Test KCQs - reversing the order of rings seems to fix ring test failure
4819 	 * after GPU reset
4820 	 */
4821 	for (i = adev->gfx.num_compute_rings - 1; i >= 0; i--) {
4822 		ring = &adev->gfx.compute_ring[i];
4823 		ring->ready = true;
4824 		r = amdgpu_ring_test_ring(ring);
4825 		if (r)
4826 			ring->ready = false;
4827 	}
4828 
4829 done:
4830 	return r;
4831 }
4832 
4833 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
4834 {
4835 	int r;
4836 
4837 	if (!(adev->flags & AMD_IS_APU))
4838 		gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4839 
4840 	r = gfx_v8_0_kiq_resume(adev);
4841 	if (r)
4842 		return r;
4843 
4844 	r = gfx_v8_0_cp_gfx_resume(adev);
4845 	if (r)
4846 		return r;
4847 
4848 	r = gfx_v8_0_kcq_resume(adev);
4849 	if (r)
4850 		return r;
4851 	gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4852 
4853 	return 0;
4854 }
4855 
4856 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)
4857 {
4858 	gfx_v8_0_cp_gfx_enable(adev, enable);
4859 	gfx_v8_0_cp_compute_enable(adev, enable);
4860 }
4861 
4862 static int gfx_v8_0_hw_init(void *handle)
4863 {
4864 	int r;
4865 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4866 
4867 	gfx_v8_0_init_golden_registers(adev);
4868 	gfx_v8_0_constants_init(adev);
4869 
4870 	r = gfx_v8_0_rlc_resume(adev);
4871 	if (r)
4872 		return r;
4873 
4874 	r = gfx_v8_0_cp_resume(adev);
4875 
4876 	return r;
4877 }
4878 
4879 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev)
4880 {
4881 	int r, i;
4882 	struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring;
4883 
4884 	r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings);
4885 	if (r)
4886 		DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4887 
4888 	for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4889 		struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4890 
4891 		amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
4892 		amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
4893 						PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
4894 						PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
4895 						PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
4896 						PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
4897 		amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
4898 		amdgpu_ring_write(kiq_ring, 0);
4899 		amdgpu_ring_write(kiq_ring, 0);
4900 		amdgpu_ring_write(kiq_ring, 0);
4901 	}
4902 	r = amdgpu_ring_test_ring(kiq_ring);
4903 	if (r)
4904 		DRM_ERROR("KCQ disable failed\n");
4905 
4906 	return r;
4907 }
4908 
4909 static bool gfx_v8_0_is_idle(void *handle)
4910 {
4911 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4912 
4913 	if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE)
4914 		|| RREG32(mmGRBM_STATUS2) != 0x8)
4915 		return false;
4916 	else
4917 		return true;
4918 }
4919 
4920 static bool gfx_v8_0_rlc_is_idle(void *handle)
4921 {
4922 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4923 
4924 	if (RREG32(mmGRBM_STATUS2) != 0x8)
4925 		return false;
4926 	else
4927 		return true;
4928 }
4929 
4930 static int gfx_v8_0_wait_for_rlc_idle(void *handle)
4931 {
4932 	unsigned int i;
4933 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4934 
4935 	for (i = 0; i < adev->usec_timeout; i++) {
4936 		if (gfx_v8_0_rlc_is_idle(handle))
4937 			return 0;
4938 
4939 		udelay(1);
4940 	}
4941 	return -ETIMEDOUT;
4942 }
4943 
4944 static int gfx_v8_0_wait_for_idle(void *handle)
4945 {
4946 	unsigned int i;
4947 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4948 
4949 	for (i = 0; i < adev->usec_timeout; i++) {
4950 		if (gfx_v8_0_is_idle(handle))
4951 			return 0;
4952 
4953 		udelay(1);
4954 	}
4955 	return -ETIMEDOUT;
4956 }
4957 
4958 static int gfx_v8_0_hw_fini(void *handle)
4959 {
4960 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4961 
4962 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4963 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4964 
4965 	amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4966 
4967 	amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0);
4968 
4969 	/* disable KCQ to avoid CPC touch memory not valid anymore */
4970 	gfx_v8_0_kcq_disable(adev);
4971 
4972 	if (amdgpu_sriov_vf(adev)) {
4973 		pr_debug("For SRIOV client, shouldn't do anything.\n");
4974 		return 0;
4975 	}
4976 	adev->gfx.rlc.funcs->enter_safe_mode(adev);
4977 	if (!gfx_v8_0_wait_for_idle(adev))
4978 		gfx_v8_0_cp_enable(adev, false);
4979 	else
4980 		pr_err("cp is busy, skip halt cp\n");
4981 	if (!gfx_v8_0_wait_for_rlc_idle(adev))
4982 		gfx_v8_0_rlc_stop(adev);
4983 	else
4984 		pr_err("rlc is busy, skip halt rlc\n");
4985 	adev->gfx.rlc.funcs->exit_safe_mode(adev);
4986 	return 0;
4987 }
4988 
4989 static int gfx_v8_0_suspend(void *handle)
4990 {
4991 	return gfx_v8_0_hw_fini(handle);
4992 }
4993 
4994 static int gfx_v8_0_resume(void *handle)
4995 {
4996 	return gfx_v8_0_hw_init(handle);
4997 }
4998 
4999 static bool gfx_v8_0_check_soft_reset(void *handle)
5000 {
5001 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5002 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5003 	u32 tmp;
5004 
5005 	/* GRBM_STATUS */
5006 	tmp = RREG32(mmGRBM_STATUS);
5007 	if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
5008 		   GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
5009 		   GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
5010 		   GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
5011 		   GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
5012 		   GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
5013 		   GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
5014 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
5015 						GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
5016 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
5017 						GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
5018 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5019 						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
5020 	}
5021 
5022 	/* GRBM_STATUS2 */
5023 	tmp = RREG32(mmGRBM_STATUS2);
5024 	if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
5025 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
5026 						GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
5027 
5028 	if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
5029 	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
5030 	    REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
5031 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5032 						SOFT_RESET_CPF, 1);
5033 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5034 						SOFT_RESET_CPC, 1);
5035 		grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
5036 						SOFT_RESET_CPG, 1);
5037 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
5038 						SOFT_RESET_GRBM, 1);
5039 	}
5040 
5041 	/* SRBM_STATUS */
5042 	tmp = RREG32(mmSRBM_STATUS);
5043 	if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
5044 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5045 						SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
5046 	if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
5047 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
5048 						SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
5049 
5050 	if (grbm_soft_reset || srbm_soft_reset) {
5051 		adev->gfx.grbm_soft_reset = grbm_soft_reset;
5052 		adev->gfx.srbm_soft_reset = srbm_soft_reset;
5053 		return true;
5054 	} else {
5055 		adev->gfx.grbm_soft_reset = 0;
5056 		adev->gfx.srbm_soft_reset = 0;
5057 		return false;
5058 	}
5059 }
5060 
5061 static int gfx_v8_0_pre_soft_reset(void *handle)
5062 {
5063 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5064 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5065 
5066 	if ((!adev->gfx.grbm_soft_reset) &&
5067 	    (!adev->gfx.srbm_soft_reset))
5068 		return 0;
5069 
5070 	grbm_soft_reset = adev->gfx.grbm_soft_reset;
5071 	srbm_soft_reset = adev->gfx.srbm_soft_reset;
5072 
5073 	/* stop the rlc */
5074 	gfx_v8_0_rlc_stop(adev);
5075 
5076 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5077 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5078 		/* Disable GFX parsing/prefetching */
5079 		gfx_v8_0_cp_gfx_enable(adev, false);
5080 
5081 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5082 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5083 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5084 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5085 		int i;
5086 
5087 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5088 			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5089 
5090 			mutex_lock(&adev->srbm_mutex);
5091 			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5092 			gfx_v8_0_deactivate_hqd(adev, 2);
5093 			vi_srbm_select(adev, 0, 0, 0, 0);
5094 			mutex_unlock(&adev->srbm_mutex);
5095 		}
5096 		/* Disable MEC parsing/prefetching */
5097 		gfx_v8_0_cp_compute_enable(adev, false);
5098 	}
5099 
5100        return 0;
5101 }
5102 
5103 static int gfx_v8_0_soft_reset(void *handle)
5104 {
5105 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5106 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5107 	u32 tmp;
5108 
5109 	if ((!adev->gfx.grbm_soft_reset) &&
5110 	    (!adev->gfx.srbm_soft_reset))
5111 		return 0;
5112 
5113 	grbm_soft_reset = adev->gfx.grbm_soft_reset;
5114 	srbm_soft_reset = adev->gfx.srbm_soft_reset;
5115 
5116 	if (grbm_soft_reset || srbm_soft_reset) {
5117 		tmp = RREG32(mmGMCON_DEBUG);
5118 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
5119 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
5120 		WREG32(mmGMCON_DEBUG, tmp);
5121 		udelay(50);
5122 	}
5123 
5124 	if (grbm_soft_reset) {
5125 		tmp = RREG32(mmGRBM_SOFT_RESET);
5126 		tmp |= grbm_soft_reset;
5127 		dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5128 		WREG32(mmGRBM_SOFT_RESET, tmp);
5129 		tmp = RREG32(mmGRBM_SOFT_RESET);
5130 
5131 		udelay(50);
5132 
5133 		tmp &= ~grbm_soft_reset;
5134 		WREG32(mmGRBM_SOFT_RESET, tmp);
5135 		tmp = RREG32(mmGRBM_SOFT_RESET);
5136 	}
5137 
5138 	if (srbm_soft_reset) {
5139 		tmp = RREG32(mmSRBM_SOFT_RESET);
5140 		tmp |= srbm_soft_reset;
5141 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5142 		WREG32(mmSRBM_SOFT_RESET, tmp);
5143 		tmp = RREG32(mmSRBM_SOFT_RESET);
5144 
5145 		udelay(50);
5146 
5147 		tmp &= ~srbm_soft_reset;
5148 		WREG32(mmSRBM_SOFT_RESET, tmp);
5149 		tmp = RREG32(mmSRBM_SOFT_RESET);
5150 	}
5151 
5152 	if (grbm_soft_reset || srbm_soft_reset) {
5153 		tmp = RREG32(mmGMCON_DEBUG);
5154 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);
5155 		tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);
5156 		WREG32(mmGMCON_DEBUG, tmp);
5157 	}
5158 
5159 	/* Wait a little for things to settle down */
5160 	udelay(50);
5161 
5162 	return 0;
5163 }
5164 
5165 static int gfx_v8_0_post_soft_reset(void *handle)
5166 {
5167 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5168 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5169 
5170 	if ((!adev->gfx.grbm_soft_reset) &&
5171 	    (!adev->gfx.srbm_soft_reset))
5172 		return 0;
5173 
5174 	grbm_soft_reset = adev->gfx.grbm_soft_reset;
5175 	srbm_soft_reset = adev->gfx.srbm_soft_reset;
5176 
5177 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5178 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5179 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5180 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5181 		int i;
5182 
5183 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5184 			struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5185 
5186 			mutex_lock(&adev->srbm_mutex);
5187 			vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5188 			gfx_v8_0_deactivate_hqd(adev, 2);
5189 			vi_srbm_select(adev, 0, 0, 0, 0);
5190 			mutex_unlock(&adev->srbm_mutex);
5191 		}
5192 		gfx_v8_0_kiq_resume(adev);
5193 		gfx_v8_0_kcq_resume(adev);
5194 	}
5195 
5196 	if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5197 	    REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5198 		gfx_v8_0_cp_gfx_resume(adev);
5199 
5200 	gfx_v8_0_rlc_start(adev);
5201 
5202 	return 0;
5203 }
5204 
5205 /**
5206  * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
5207  *
5208  * @adev: amdgpu_device pointer
5209  *
5210  * Fetches a GPU clock counter snapshot.
5211  * Returns the 64 bit clock counter snapshot.
5212  */
5213 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
5214 {
5215 	uint64_t clock;
5216 
5217 	mutex_lock(&adev->gfx.gpu_clock_mutex);
5218 	WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5219 	clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
5220 		((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5221 	mutex_unlock(&adev->gfx.gpu_clock_mutex);
5222 	return clock;
5223 }
5224 
5225 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5226 					  uint32_t vmid,
5227 					  uint32_t gds_base, uint32_t gds_size,
5228 					  uint32_t gws_base, uint32_t gws_size,
5229 					  uint32_t oa_base, uint32_t oa_size)
5230 {
5231 	/* GDS Base */
5232 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5233 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5234 				WRITE_DATA_DST_SEL(0)));
5235 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
5236 	amdgpu_ring_write(ring, 0);
5237 	amdgpu_ring_write(ring, gds_base);
5238 
5239 	/* GDS Size */
5240 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5241 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5242 				WRITE_DATA_DST_SEL(0)));
5243 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
5244 	amdgpu_ring_write(ring, 0);
5245 	amdgpu_ring_write(ring, gds_size);
5246 
5247 	/* GWS */
5248 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5249 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5250 				WRITE_DATA_DST_SEL(0)));
5251 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
5252 	amdgpu_ring_write(ring, 0);
5253 	amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5254 
5255 	/* OA */
5256 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5257 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5258 				WRITE_DATA_DST_SEL(0)));
5259 	amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
5260 	amdgpu_ring_write(ring, 0);
5261 	amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
5262 }
5263 
5264 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
5265 {
5266 	WREG32(mmSQ_IND_INDEX,
5267 		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5268 		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5269 		(address << SQ_IND_INDEX__INDEX__SHIFT) |
5270 		(SQ_IND_INDEX__FORCE_READ_MASK));
5271 	return RREG32(mmSQ_IND_DATA);
5272 }
5273 
5274 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
5275 			   uint32_t wave, uint32_t thread,
5276 			   uint32_t regno, uint32_t num, uint32_t *out)
5277 {
5278 	WREG32(mmSQ_IND_INDEX,
5279 		(wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5280 		(simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5281 		(regno << SQ_IND_INDEX__INDEX__SHIFT) |
5282 		(thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
5283 		(SQ_IND_INDEX__FORCE_READ_MASK) |
5284 		(SQ_IND_INDEX__AUTO_INCR_MASK));
5285 	while (num--)
5286 		*(out++) = RREG32(mmSQ_IND_DATA);
5287 }
5288 
5289 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
5290 {
5291 	/* type 0 wave data */
5292 	dst[(*no_fields)++] = 0;
5293 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
5294 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
5295 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
5296 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
5297 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
5298 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
5299 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
5300 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
5301 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
5302 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
5303 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
5304 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
5305 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
5306 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
5307 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
5308 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
5309 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
5310 	dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
5311 }
5312 
5313 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
5314 				     uint32_t wave, uint32_t start,
5315 				     uint32_t size, uint32_t *dst)
5316 {
5317 	wave_read_regs(
5318 		adev, simd, wave, 0,
5319 		start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
5320 }
5321 
5322 
5323 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
5324 	.get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
5325 	.select_se_sh = &gfx_v8_0_select_se_sh,
5326 	.read_wave_data = &gfx_v8_0_read_wave_data,
5327 	.read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
5328 	.select_me_pipe_q = &gfx_v8_0_select_me_pipe_q
5329 };
5330 
5331 static int gfx_v8_0_early_init(void *handle)
5332 {
5333 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5334 
5335 	adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;
5336 	adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
5337 	adev->gfx.funcs = &gfx_v8_0_gfx_funcs;
5338 	gfx_v8_0_set_ring_funcs(adev);
5339 	gfx_v8_0_set_irq_funcs(adev);
5340 	gfx_v8_0_set_gds_init(adev);
5341 	gfx_v8_0_set_rlc_funcs(adev);
5342 
5343 	return 0;
5344 }
5345 
5346 static int gfx_v8_0_late_init(void *handle)
5347 {
5348 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5349 	int r;
5350 
5351 	r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5352 	if (r)
5353 		return r;
5354 
5355 	r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5356 	if (r)
5357 		return r;
5358 
5359 	/* requires IBs so do in late init after IB pool is initialized */
5360 	r = gfx_v8_0_do_edc_gpr_workarounds(adev);
5361 	if (r)
5362 		return r;
5363 
5364 	r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
5365 	if (r) {
5366 		DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r);
5367 		return r;
5368 	}
5369 
5370 	r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0);
5371 	if (r) {
5372 		DRM_ERROR(
5373 			"amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n",
5374 			r);
5375 		return r;
5376 	}
5377 
5378 	return 0;
5379 }
5380 
5381 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
5382 						       bool enable)
5383 {
5384 	if (((adev->asic_type == CHIP_POLARIS11) ||
5385 	    (adev->asic_type == CHIP_POLARIS12) ||
5386 	    (adev->asic_type == CHIP_VEGAM)) &&
5387 	    adev->powerplay.pp_funcs->set_powergating_by_smu)
5388 		/* Send msg to SMU via Powerplay */
5389 		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable);
5390 
5391 	WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
5392 }
5393 
5394 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
5395 							bool enable)
5396 {
5397 	WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);
5398 }
5399 
5400 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
5401 		bool enable)
5402 {
5403 	WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);
5404 }
5405 
5406 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
5407 					  bool enable)
5408 {
5409 	WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);
5410 }
5411 
5412 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
5413 						bool enable)
5414 {
5415 	WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);
5416 
5417 	/* Read any GFX register to wake up GFX. */
5418 	if (!enable)
5419 		RREG32(mmDB_RENDER_CONTROL);
5420 }
5421 
5422 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
5423 					  bool enable)
5424 {
5425 	if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
5426 		cz_enable_gfx_cg_power_gating(adev, true);
5427 		if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
5428 			cz_enable_gfx_pipeline_power_gating(adev, true);
5429 	} else {
5430 		cz_enable_gfx_cg_power_gating(adev, false);
5431 		cz_enable_gfx_pipeline_power_gating(adev, false);
5432 	}
5433 }
5434 
5435 static int gfx_v8_0_set_powergating_state(void *handle,
5436 					  enum amd_powergating_state state)
5437 {
5438 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5439 	bool enable = (state == AMD_PG_STATE_GATE);
5440 
5441 	if (amdgpu_sriov_vf(adev))
5442 		return 0;
5443 
5444 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5445 				AMD_PG_SUPPORT_RLC_SMU_HS |
5446 				AMD_PG_SUPPORT_CP |
5447 				AMD_PG_SUPPORT_GFX_DMG))
5448 		adev->gfx.rlc.funcs->enter_safe_mode(adev);
5449 	switch (adev->asic_type) {
5450 	case CHIP_CARRIZO:
5451 	case CHIP_STONEY:
5452 
5453 		if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5454 			cz_enable_sck_slow_down_on_power_up(adev, true);
5455 			cz_enable_sck_slow_down_on_power_down(adev, true);
5456 		} else {
5457 			cz_enable_sck_slow_down_on_power_up(adev, false);
5458 			cz_enable_sck_slow_down_on_power_down(adev, false);
5459 		}
5460 		if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5461 			cz_enable_cp_power_gating(adev, true);
5462 		else
5463 			cz_enable_cp_power_gating(adev, false);
5464 
5465 		cz_update_gfx_cg_power_gating(adev, enable);
5466 
5467 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5468 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5469 		else
5470 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5471 
5472 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5473 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5474 		else
5475 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5476 		break;
5477 	case CHIP_POLARIS11:
5478 	case CHIP_POLARIS12:
5479 	case CHIP_VEGAM:
5480 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5481 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5482 		else
5483 			gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5484 
5485 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5486 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5487 		else
5488 			gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5489 
5490 		if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)
5491 			polaris11_enable_gfx_quick_mg_power_gating(adev, true);
5492 		else
5493 			polaris11_enable_gfx_quick_mg_power_gating(adev, false);
5494 		break;
5495 	default:
5496 		break;
5497 	}
5498 	if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5499 				AMD_PG_SUPPORT_RLC_SMU_HS |
5500 				AMD_PG_SUPPORT_CP |
5501 				AMD_PG_SUPPORT_GFX_DMG))
5502 		adev->gfx.rlc.funcs->exit_safe_mode(adev);
5503 	return 0;
5504 }
5505 
5506 static void gfx_v8_0_get_clockgating_state(void *handle, u32 *flags)
5507 {
5508 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5509 	int data;
5510 
5511 	if (amdgpu_sriov_vf(adev))
5512 		*flags = 0;
5513 
5514 	/* AMD_CG_SUPPORT_GFX_MGCG */
5515 	data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5516 	if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))
5517 		*flags |= AMD_CG_SUPPORT_GFX_MGCG;
5518 
5519 	/* AMD_CG_SUPPORT_GFX_CGLG */
5520 	data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5521 	if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5522 		*flags |= AMD_CG_SUPPORT_GFX_CGCG;
5523 
5524 	/* AMD_CG_SUPPORT_GFX_CGLS */
5525 	if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5526 		*flags |= AMD_CG_SUPPORT_GFX_CGLS;
5527 
5528 	/* AMD_CG_SUPPORT_GFX_CGTS */
5529 	data = RREG32(mmCGTS_SM_CTRL_REG);
5530 	if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))
5531 		*flags |= AMD_CG_SUPPORT_GFX_CGTS;
5532 
5533 	/* AMD_CG_SUPPORT_GFX_CGTS_LS */
5534 	if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))
5535 		*flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;
5536 
5537 	/* AMD_CG_SUPPORT_GFX_RLC_LS */
5538 	data = RREG32(mmRLC_MEM_SLP_CNTL);
5539 	if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5540 		*flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5541 
5542 	/* AMD_CG_SUPPORT_GFX_CP_LS */
5543 	data = RREG32(mmCP_MEM_SLP_CNTL);
5544 	if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5545 		*flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5546 }
5547 
5548 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
5549 				     uint32_t reg_addr, uint32_t cmd)
5550 {
5551 	uint32_t data;
5552 
5553 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
5554 
5555 	WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5556 	WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5557 
5558 	data = RREG32(mmRLC_SERDES_WR_CTRL);
5559 	if (adev->asic_type == CHIP_STONEY)
5560 		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5561 			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5562 			  RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5563 			  RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5564 			  RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5565 			  RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5566 			  RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5567 			  RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5568 			  RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5569 	else
5570 		data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5571 			  RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5572 			  RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5573 			  RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5574 			  RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5575 			  RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5576 			  RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5577 			  RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5578 			  RLC_SERDES_WR_CTRL__BPM_DATA_MASK |
5579 			  RLC_SERDES_WR_CTRL__REG_ADDR_MASK |
5580 			  RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5581 	data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |
5582 		 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |
5583 		 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |
5584 		 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));
5585 
5586 	WREG32(mmRLC_SERDES_WR_CTRL, data);
5587 }
5588 
5589 #define MSG_ENTER_RLC_SAFE_MODE     1
5590 #define MSG_EXIT_RLC_SAFE_MODE      0
5591 #define RLC_GPR_REG2__REQ_MASK 0x00000001
5592 #define RLC_GPR_REG2__REQ__SHIFT 0
5593 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
5594 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
5595 
5596 static void iceland_enter_rlc_safe_mode(struct amdgpu_device *adev)
5597 {
5598 	u32 data;
5599 	unsigned i;
5600 
5601 	data = RREG32(mmRLC_CNTL);
5602 	if (!(data & RLC_CNTL__RLC_ENABLE_F32_MASK))
5603 		return;
5604 
5605 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) {
5606 		data |= RLC_SAFE_MODE__CMD_MASK;
5607 		data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5608 		data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5609 		WREG32(mmRLC_SAFE_MODE, data);
5610 
5611 		for (i = 0; i < adev->usec_timeout; i++) {
5612 			if ((RREG32(mmRLC_GPM_STAT) &
5613 			     (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5614 			      RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==
5615 			    (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5616 			     RLC_GPM_STAT__GFX_POWER_STATUS_MASK))
5617 				break;
5618 			udelay(1);
5619 		}
5620 
5621 		for (i = 0; i < adev->usec_timeout; i++) {
5622 			if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5623 				break;
5624 			udelay(1);
5625 		}
5626 		adev->gfx.rlc.in_safe_mode = true;
5627 	}
5628 }
5629 
5630 static void iceland_exit_rlc_safe_mode(struct amdgpu_device *adev)
5631 {
5632 	u32 data = 0;
5633 	unsigned i;
5634 
5635 	data = RREG32(mmRLC_CNTL);
5636 	if (!(data & RLC_CNTL__RLC_ENABLE_F32_MASK))
5637 		return;
5638 
5639 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_MGCG)) {
5640 		if (adev->gfx.rlc.in_safe_mode) {
5641 			data |= RLC_SAFE_MODE__CMD_MASK;
5642 			data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5643 			WREG32(mmRLC_SAFE_MODE, data);
5644 			adev->gfx.rlc.in_safe_mode = false;
5645 		}
5646 	}
5647 
5648 	for (i = 0; i < adev->usec_timeout; i++) {
5649 		if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5650 			break;
5651 		udelay(1);
5652 	}
5653 }
5654 
5655 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {
5656 	.enter_safe_mode = iceland_enter_rlc_safe_mode,
5657 	.exit_safe_mode = iceland_exit_rlc_safe_mode
5658 };
5659 
5660 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5661 						      bool enable)
5662 {
5663 	uint32_t temp, data;
5664 
5665 	adev->gfx.rlc.funcs->enter_safe_mode(adev);
5666 
5667 	/* It is disabled by HW by default */
5668 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
5669 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5670 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)
5671 				/* 1 - RLC memory Light sleep */
5672 				WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);
5673 
5674 			if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)
5675 				WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);
5676 		}
5677 
5678 		/* 3 - RLC_CGTT_MGCG_OVERRIDE */
5679 		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5680 		if (adev->flags & AMD_IS_APU)
5681 			data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5682 				  RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5683 				  RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK);
5684 		else
5685 			data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5686 				  RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5687 				  RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5688 				  RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5689 
5690 		if (temp != data)
5691 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5692 
5693 		/* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5694 		gfx_v8_0_wait_for_rlc_serdes(adev);
5695 
5696 		/* 5 - clear mgcg override */
5697 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5698 
5699 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
5700 			/* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */
5701 			temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5702 			data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);
5703 			data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
5704 			data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
5705 			data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
5706 			if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
5707 			    (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
5708 				data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
5709 			data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
5710 			data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
5711 			if (temp != data)
5712 				WREG32(mmCGTS_SM_CTRL_REG, data);
5713 		}
5714 		udelay(50);
5715 
5716 		/* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5717 		gfx_v8_0_wait_for_rlc_serdes(adev);
5718 	} else {
5719 		/* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */
5720 		temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5721 		data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5722 				RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5723 				RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5724 				RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5725 		if (temp != data)
5726 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5727 
5728 		/* 2 - disable MGLS in RLC */
5729 		data = RREG32(mmRLC_MEM_SLP_CNTL);
5730 		if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5731 			data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5732 			WREG32(mmRLC_MEM_SLP_CNTL, data);
5733 		}
5734 
5735 		/* 3 - disable MGLS in CP */
5736 		data = RREG32(mmCP_MEM_SLP_CNTL);
5737 		if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5738 			data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5739 			WREG32(mmCP_MEM_SLP_CNTL, data);
5740 		}
5741 
5742 		/* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */
5743 		temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5744 		data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |
5745 				CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);
5746 		if (temp != data)
5747 			WREG32(mmCGTS_SM_CTRL_REG, data);
5748 
5749 		/* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5750 		gfx_v8_0_wait_for_rlc_serdes(adev);
5751 
5752 		/* 6 - set mgcg override */
5753 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5754 
5755 		udelay(50);
5756 
5757 		/* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5758 		gfx_v8_0_wait_for_rlc_serdes(adev);
5759 	}
5760 
5761 	adev->gfx.rlc.funcs->exit_safe_mode(adev);
5762 }
5763 
5764 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5765 						      bool enable)
5766 {
5767 	uint32_t temp, temp1, data, data1;
5768 
5769 	temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5770 
5771 	adev->gfx.rlc.funcs->enter_safe_mode(adev);
5772 
5773 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5774 		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5775 		data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
5776 		if (temp1 != data1)
5777 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5778 
5779 		/* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5780 		gfx_v8_0_wait_for_rlc_serdes(adev);
5781 
5782 		/* 2 - clear cgcg override */
5783 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5784 
5785 		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5786 		gfx_v8_0_wait_for_rlc_serdes(adev);
5787 
5788 		/* 3 - write cmd to set CGLS */
5789 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
5790 
5791 		/* 4 - enable cgcg */
5792 		data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5793 
5794 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5795 			/* enable cgls*/
5796 			data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5797 
5798 			temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5799 			data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;
5800 
5801 			if (temp1 != data1)
5802 				WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5803 		} else {
5804 			data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5805 		}
5806 
5807 		if (temp != data)
5808 			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5809 
5810 		/* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/
5811 		 * Cmp_busy/GFX_Idle interrupts
5812 		 */
5813 		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5814 	} else {
5815 		/* disable cntx_empty_int_enable & GFX Idle interrupt */
5816 		gfx_v8_0_enable_gui_idle_interrupt(adev, false);
5817 
5818 		/* TEST CGCG */
5819 		temp1 = data1 =	RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5820 		data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |
5821 				RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);
5822 		if (temp1 != data1)
5823 			WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5824 
5825 		/* read gfx register to wake up cgcg */
5826 		RREG32(mmCB_CGTT_SCLK_CTRL);
5827 		RREG32(mmCB_CGTT_SCLK_CTRL);
5828 		RREG32(mmCB_CGTT_SCLK_CTRL);
5829 		RREG32(mmCB_CGTT_SCLK_CTRL);
5830 
5831 		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5832 		gfx_v8_0_wait_for_rlc_serdes(adev);
5833 
5834 		/* write cmd to Set CGCG Overrride */
5835 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5836 
5837 		/* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5838 		gfx_v8_0_wait_for_rlc_serdes(adev);
5839 
5840 		/* write cmd to Clear CGLS */
5841 		gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);
5842 
5843 		/* disable cgcg, cgls should be disabled too. */
5844 		data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
5845 			  RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5846 		if (temp != data)
5847 			WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5848 		/* enable interrupts again for PG */
5849 		gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5850 	}
5851 
5852 	gfx_v8_0_wait_for_rlc_serdes(adev);
5853 
5854 	adev->gfx.rlc.funcs->exit_safe_mode(adev);
5855 }
5856 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5857 					    bool enable)
5858 {
5859 	if (enable) {
5860 		/* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)
5861 		 * ===  MGCG + MGLS + TS(CG/LS) ===
5862 		 */
5863 		gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5864 		gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5865 	} else {
5866 		/* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)
5867 		 * ===  CGCG + CGLS ===
5868 		 */
5869 		gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5870 		gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5871 	}
5872 	return 0;
5873 }
5874 
5875 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
5876 					  enum amd_clockgating_state state)
5877 {
5878 	uint32_t msg_id, pp_state = 0;
5879 	uint32_t pp_support_state = 0;
5880 
5881 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5882 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5883 			pp_support_state = PP_STATE_SUPPORT_LS;
5884 			pp_state = PP_STATE_LS;
5885 		}
5886 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5887 			pp_support_state |= PP_STATE_SUPPORT_CG;
5888 			pp_state |= PP_STATE_CG;
5889 		}
5890 		if (state == AMD_CG_STATE_UNGATE)
5891 			pp_state = 0;
5892 
5893 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5894 				PP_BLOCK_GFX_CG,
5895 				pp_support_state,
5896 				pp_state);
5897 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5898 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5899 	}
5900 
5901 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5902 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5903 			pp_support_state = PP_STATE_SUPPORT_LS;
5904 			pp_state = PP_STATE_LS;
5905 		}
5906 
5907 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5908 			pp_support_state |= PP_STATE_SUPPORT_CG;
5909 			pp_state |= PP_STATE_CG;
5910 		}
5911 
5912 		if (state == AMD_CG_STATE_UNGATE)
5913 			pp_state = 0;
5914 
5915 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5916 				PP_BLOCK_GFX_MG,
5917 				pp_support_state,
5918 				pp_state);
5919 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5920 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5921 	}
5922 
5923 	return 0;
5924 }
5925 
5926 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
5927 					  enum amd_clockgating_state state)
5928 {
5929 
5930 	uint32_t msg_id, pp_state = 0;
5931 	uint32_t pp_support_state = 0;
5932 
5933 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5934 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5935 			pp_support_state = PP_STATE_SUPPORT_LS;
5936 			pp_state = PP_STATE_LS;
5937 		}
5938 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5939 			pp_support_state |= PP_STATE_SUPPORT_CG;
5940 			pp_state |= PP_STATE_CG;
5941 		}
5942 		if (state == AMD_CG_STATE_UNGATE)
5943 			pp_state = 0;
5944 
5945 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5946 				PP_BLOCK_GFX_CG,
5947 				pp_support_state,
5948 				pp_state);
5949 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5950 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5951 	}
5952 
5953 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
5954 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5955 			pp_support_state = PP_STATE_SUPPORT_LS;
5956 			pp_state = PP_STATE_LS;
5957 		}
5958 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5959 			pp_support_state |= PP_STATE_SUPPORT_CG;
5960 			pp_state |= PP_STATE_CG;
5961 		}
5962 		if (state == AMD_CG_STATE_UNGATE)
5963 			pp_state = 0;
5964 
5965 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5966 				PP_BLOCK_GFX_3D,
5967 				pp_support_state,
5968 				pp_state);
5969 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5970 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5971 	}
5972 
5973 	if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5974 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5975 			pp_support_state = PP_STATE_SUPPORT_LS;
5976 			pp_state = PP_STATE_LS;
5977 		}
5978 
5979 		if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5980 			pp_support_state |= PP_STATE_SUPPORT_CG;
5981 			pp_state |= PP_STATE_CG;
5982 		}
5983 
5984 		if (state == AMD_CG_STATE_UNGATE)
5985 			pp_state = 0;
5986 
5987 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5988 				PP_BLOCK_GFX_MG,
5989 				pp_support_state,
5990 				pp_state);
5991 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
5992 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5993 	}
5994 
5995 	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
5996 		pp_support_state = PP_STATE_SUPPORT_LS;
5997 
5998 		if (state == AMD_CG_STATE_UNGATE)
5999 			pp_state = 0;
6000 		else
6001 			pp_state = PP_STATE_LS;
6002 
6003 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6004 				PP_BLOCK_GFX_RLC,
6005 				pp_support_state,
6006 				pp_state);
6007 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6008 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6009 	}
6010 
6011 	if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
6012 		pp_support_state = PP_STATE_SUPPORT_LS;
6013 
6014 		if (state == AMD_CG_STATE_UNGATE)
6015 			pp_state = 0;
6016 		else
6017 			pp_state = PP_STATE_LS;
6018 		msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
6019 			PP_BLOCK_GFX_CP,
6020 			pp_support_state,
6021 			pp_state);
6022 		if (adev->powerplay.pp_funcs->set_clockgating_by_smu)
6023 			amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
6024 	}
6025 
6026 	return 0;
6027 }
6028 
6029 static int gfx_v8_0_set_clockgating_state(void *handle,
6030 					  enum amd_clockgating_state state)
6031 {
6032 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
6033 
6034 	if (amdgpu_sriov_vf(adev))
6035 		return 0;
6036 
6037 	switch (adev->asic_type) {
6038 	case CHIP_FIJI:
6039 	case CHIP_CARRIZO:
6040 	case CHIP_STONEY:
6041 		gfx_v8_0_update_gfx_clock_gating(adev,
6042 						 state == AMD_CG_STATE_GATE);
6043 		break;
6044 	case CHIP_TONGA:
6045 		gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);
6046 		break;
6047 	case CHIP_POLARIS10:
6048 	case CHIP_POLARIS11:
6049 	case CHIP_POLARIS12:
6050 	case CHIP_VEGAM:
6051 		gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
6052 		break;
6053 	default:
6054 		break;
6055 	}
6056 	return 0;
6057 }
6058 
6059 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
6060 {
6061 	return ring->adev->wb.wb[ring->rptr_offs];
6062 }
6063 
6064 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
6065 {
6066 	struct amdgpu_device *adev = ring->adev;
6067 
6068 	if (ring->use_doorbell)
6069 		/* XXX check if swapping is necessary on BE */
6070 		return ring->adev->wb.wb[ring->wptr_offs];
6071 	else
6072 		return RREG32(mmCP_RB0_WPTR);
6073 }
6074 
6075 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
6076 {
6077 	struct amdgpu_device *adev = ring->adev;
6078 
6079 	if (ring->use_doorbell) {
6080 		/* XXX check if swapping is necessary on BE */
6081 		adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
6082 		WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6083 	} else {
6084 		WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
6085 		(void)RREG32(mmCP_RB0_WPTR);
6086 	}
6087 }
6088 
6089 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
6090 {
6091 	u32 ref_and_mask, reg_mem_engine;
6092 
6093 	if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
6094 	    (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
6095 		switch (ring->me) {
6096 		case 1:
6097 			ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
6098 			break;
6099 		case 2:
6100 			ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
6101 			break;
6102 		default:
6103 			return;
6104 		}
6105 		reg_mem_engine = 0;
6106 	} else {
6107 		ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
6108 		reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
6109 	}
6110 
6111 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6112 	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
6113 				 WAIT_REG_MEM_FUNCTION(3) |  /* == */
6114 				 reg_mem_engine));
6115 	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
6116 	amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
6117 	amdgpu_ring_write(ring, ref_and_mask);
6118 	amdgpu_ring_write(ring, ref_and_mask);
6119 	amdgpu_ring_write(ring, 0x20); /* poll interval */
6120 }
6121 
6122 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
6123 {
6124 	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6125 	amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
6126 		EVENT_INDEX(4));
6127 
6128 	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6129 	amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
6130 		EVENT_INDEX(0));
6131 }
6132 
6133 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
6134 				      struct amdgpu_ib *ib,
6135 				      unsigned vmid, bool ctx_switch)
6136 {
6137 	u32 header, control = 0;
6138 
6139 	if (ib->flags & AMDGPU_IB_FLAG_CE)
6140 		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
6141 	else
6142 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
6143 
6144 	control |= ib->length_dw | (vmid << 24);
6145 
6146 	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
6147 		control |= INDIRECT_BUFFER_PRE_ENB(1);
6148 
6149 		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
6150 			gfx_v8_0_ring_emit_de_meta(ring);
6151 	}
6152 
6153 	amdgpu_ring_write(ring, header);
6154 	amdgpu_ring_write(ring,
6155 #ifdef __BIG_ENDIAN
6156 			  (2 << 0) |
6157 #endif
6158 			  (ib->gpu_addr & 0xFFFFFFFC));
6159 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6160 	amdgpu_ring_write(ring, control);
6161 }
6162 
6163 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
6164 					  struct amdgpu_ib *ib,
6165 					  unsigned vmid, bool ctx_switch)
6166 {
6167 	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
6168 
6169 	amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
6170 	amdgpu_ring_write(ring,
6171 #ifdef __BIG_ENDIAN
6172 				(2 << 0) |
6173 #endif
6174 				(ib->gpu_addr & 0xFFFFFFFC));
6175 	amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6176 	amdgpu_ring_write(ring, control);
6177 }
6178 
6179 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
6180 					 u64 seq, unsigned flags)
6181 {
6182 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6183 	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6184 
6185 	/* EVENT_WRITE_EOP - flush caches, send int */
6186 	amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6187 	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6188 				 EOP_TC_ACTION_EN |
6189 				 EOP_TC_WB_ACTION_EN |
6190 				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6191 				 EVENT_INDEX(5)));
6192 	amdgpu_ring_write(ring, addr & 0xfffffffc);
6193 	amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6194 			  DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6195 	amdgpu_ring_write(ring, lower_32_bits(seq));
6196 	amdgpu_ring_write(ring, upper_32_bits(seq));
6197 
6198 }
6199 
6200 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
6201 {
6202 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6203 	uint32_t seq = ring->fence_drv.sync_seq;
6204 	uint64_t addr = ring->fence_drv.gpu_addr;
6205 
6206 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6207 	amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
6208 				 WAIT_REG_MEM_FUNCTION(3) | /* equal */
6209 				 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */
6210 	amdgpu_ring_write(ring, addr & 0xfffffffc);
6211 	amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
6212 	amdgpu_ring_write(ring, seq);
6213 	amdgpu_ring_write(ring, 0xffffffff);
6214 	amdgpu_ring_write(ring, 4); /* poll interval */
6215 }
6216 
6217 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
6218 					unsigned vmid, uint64_t pd_addr)
6219 {
6220 	int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6221 
6222 	amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
6223 
6224 	/* wait for the invalidate to complete */
6225 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6226 	amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6227 				 WAIT_REG_MEM_FUNCTION(0) |  /* always */
6228 				 WAIT_REG_MEM_ENGINE(0))); /* me */
6229 	amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
6230 	amdgpu_ring_write(ring, 0);
6231 	amdgpu_ring_write(ring, 0); /* ref */
6232 	amdgpu_ring_write(ring, 0); /* mask */
6233 	amdgpu_ring_write(ring, 0x20); /* poll interval */
6234 
6235 	/* compute doesn't have PFP */
6236 	if (usepfp) {
6237 		/* sync PFP to ME, otherwise we might get invalid PFP reads */
6238 		amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6239 		amdgpu_ring_write(ring, 0x0);
6240 	}
6241 }
6242 
6243 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
6244 {
6245 	return ring->adev->wb.wb[ring->wptr_offs];
6246 }
6247 
6248 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
6249 {
6250 	struct amdgpu_device *adev = ring->adev;
6251 
6252 	/* XXX check if swapping is necessary on BE */
6253 	adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr);
6254 	WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6255 }
6256 
6257 static void gfx_v8_0_ring_set_pipe_percent(struct amdgpu_ring *ring,
6258 					   bool acquire)
6259 {
6260 	struct amdgpu_device *adev = ring->adev;
6261 	int pipe_num, tmp, reg;
6262 	int pipe_percent = acquire ? SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK : 0x1;
6263 
6264 	pipe_num = ring->me * adev->gfx.mec.num_pipe_per_mec + ring->pipe;
6265 
6266 	/* first me only has 2 entries, GFX and HP3D */
6267 	if (ring->me > 0)
6268 		pipe_num -= 2;
6269 
6270 	reg = mmSPI_WCL_PIPE_PERCENT_GFX + pipe_num;
6271 	tmp = RREG32(reg);
6272 	tmp = REG_SET_FIELD(tmp, SPI_WCL_PIPE_PERCENT_GFX, VALUE, pipe_percent);
6273 	WREG32(reg, tmp);
6274 }
6275 
6276 static void gfx_v8_0_pipe_reserve_resources(struct amdgpu_device *adev,
6277 					    struct amdgpu_ring *ring,
6278 					    bool acquire)
6279 {
6280 	int i, pipe;
6281 	bool reserve;
6282 	struct amdgpu_ring *iring;
6283 
6284 	mutex_lock(&adev->gfx.pipe_reserve_mutex);
6285 	pipe = amdgpu_gfx_queue_to_bit(adev, ring->me, ring->pipe, 0);
6286 	if (acquire)
6287 		set_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6288 	else
6289 		clear_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6290 
6291 	if (!bitmap_weight(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES)) {
6292 		/* Clear all reservations - everyone reacquires all resources */
6293 		for (i = 0; i < adev->gfx.num_gfx_rings; ++i)
6294 			gfx_v8_0_ring_set_pipe_percent(&adev->gfx.gfx_ring[i],
6295 						       true);
6296 
6297 		for (i = 0; i < adev->gfx.num_compute_rings; ++i)
6298 			gfx_v8_0_ring_set_pipe_percent(&adev->gfx.compute_ring[i],
6299 						       true);
6300 	} else {
6301 		/* Lower all pipes without a current reservation */
6302 		for (i = 0; i < adev->gfx.num_gfx_rings; ++i) {
6303 			iring = &adev->gfx.gfx_ring[i];
6304 			pipe = amdgpu_gfx_queue_to_bit(adev,
6305 						       iring->me,
6306 						       iring->pipe,
6307 						       0);
6308 			reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6309 			gfx_v8_0_ring_set_pipe_percent(iring, reserve);
6310 		}
6311 
6312 		for (i = 0; i < adev->gfx.num_compute_rings; ++i) {
6313 			iring = &adev->gfx.compute_ring[i];
6314 			pipe = amdgpu_gfx_queue_to_bit(adev,
6315 						       iring->me,
6316 						       iring->pipe,
6317 						       0);
6318 			reserve = test_bit(pipe, adev->gfx.pipe_reserve_bitmap);
6319 			gfx_v8_0_ring_set_pipe_percent(iring, reserve);
6320 		}
6321 	}
6322 
6323 	mutex_unlock(&adev->gfx.pipe_reserve_mutex);
6324 }
6325 
6326 static void gfx_v8_0_hqd_set_priority(struct amdgpu_device *adev,
6327 				      struct amdgpu_ring *ring,
6328 				      bool acquire)
6329 {
6330 	uint32_t pipe_priority = acquire ? 0x2 : 0x0;
6331 	uint32_t queue_priority = acquire ? 0xf : 0x0;
6332 
6333 	mutex_lock(&adev->srbm_mutex);
6334 	vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
6335 
6336 	WREG32(mmCP_HQD_PIPE_PRIORITY, pipe_priority);
6337 	WREG32(mmCP_HQD_QUEUE_PRIORITY, queue_priority);
6338 
6339 	vi_srbm_select(adev, 0, 0, 0, 0);
6340 	mutex_unlock(&adev->srbm_mutex);
6341 }
6342 static void gfx_v8_0_ring_set_priority_compute(struct amdgpu_ring *ring,
6343 					       enum drm_sched_priority priority)
6344 {
6345 	struct amdgpu_device *adev = ring->adev;
6346 	bool acquire = priority == DRM_SCHED_PRIORITY_HIGH_HW;
6347 
6348 	if (ring->funcs->type != AMDGPU_RING_TYPE_COMPUTE)
6349 		return;
6350 
6351 	gfx_v8_0_hqd_set_priority(adev, ring, acquire);
6352 	gfx_v8_0_pipe_reserve_resources(adev, ring, acquire);
6353 }
6354 
6355 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
6356 					     u64 addr, u64 seq,
6357 					     unsigned flags)
6358 {
6359 	bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6360 	bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6361 
6362 	/* RELEASE_MEM - flush caches, send int */
6363 	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
6364 	amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6365 				 EOP_TC_ACTION_EN |
6366 				 EOP_TC_WB_ACTION_EN |
6367 				 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6368 				 EVENT_INDEX(5)));
6369 	amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6370 	amdgpu_ring_write(ring, addr & 0xfffffffc);
6371 	amdgpu_ring_write(ring, upper_32_bits(addr));
6372 	amdgpu_ring_write(ring, lower_32_bits(seq));
6373 	amdgpu_ring_write(ring, upper_32_bits(seq));
6374 }
6375 
6376 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6377 					 u64 seq, unsigned int flags)
6378 {
6379 	/* we only allocate 32bit for each seq wb address */
6380 	BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
6381 
6382 	/* write fence seq to the "addr" */
6383 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6384 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6385 				 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6386 	amdgpu_ring_write(ring, lower_32_bits(addr));
6387 	amdgpu_ring_write(ring, upper_32_bits(addr));
6388 	amdgpu_ring_write(ring, lower_32_bits(seq));
6389 
6390 	if (flags & AMDGPU_FENCE_FLAG_INT) {
6391 		/* set register to trigger INT */
6392 		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6393 		amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6394 					 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6395 		amdgpu_ring_write(ring, mmCPC_INT_STATUS);
6396 		amdgpu_ring_write(ring, 0);
6397 		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6398 	}
6399 }
6400 
6401 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
6402 {
6403 	amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
6404 	amdgpu_ring_write(ring, 0);
6405 }
6406 
6407 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
6408 {
6409 	uint32_t dw2 = 0;
6410 
6411 	if (amdgpu_sriov_vf(ring->adev))
6412 		gfx_v8_0_ring_emit_ce_meta(ring);
6413 
6414 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6415 	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
6416 		gfx_v8_0_ring_emit_vgt_flush(ring);
6417 		/* set load_global_config & load_global_uconfig */
6418 		dw2 |= 0x8001;
6419 		/* set load_cs_sh_regs */
6420 		dw2 |= 0x01000000;
6421 		/* set load_per_context_state & load_gfx_sh_regs for GFX */
6422 		dw2 |= 0x10002;
6423 
6424 		/* set load_ce_ram if preamble presented */
6425 		if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
6426 			dw2 |= 0x10000000;
6427 	} else {
6428 		/* still load_ce_ram if this is the first time preamble presented
6429 		 * although there is no context switch happens.
6430 		 */
6431 		if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
6432 			dw2 |= 0x10000000;
6433 	}
6434 
6435 	amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6436 	amdgpu_ring_write(ring, dw2);
6437 	amdgpu_ring_write(ring, 0);
6438 }
6439 
6440 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
6441 {
6442 	unsigned ret;
6443 
6444 	amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
6445 	amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
6446 	amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
6447 	amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
6448 	ret = ring->wptr & ring->buf_mask;
6449 	amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
6450 	return ret;
6451 }
6452 
6453 static void gfx_v8_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
6454 {
6455 	unsigned cur;
6456 
6457 	BUG_ON(offset > ring->buf_mask);
6458 	BUG_ON(ring->ring[offset] != 0x55aa55aa);
6459 
6460 	cur = (ring->wptr & ring->buf_mask) - 1;
6461 	if (likely(cur > offset))
6462 		ring->ring[offset] = cur - offset;
6463 	else
6464 		ring->ring[offset] = (ring->ring_size >> 2) - offset + cur;
6465 }
6466 
6467 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
6468 {
6469 	struct amdgpu_device *adev = ring->adev;
6470 
6471 	amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6472 	amdgpu_ring_write(ring, 0 |	/* src: register*/
6473 				(5 << 8) |	/* dst: memory */
6474 				(1 << 20));	/* write confirm */
6475 	amdgpu_ring_write(ring, reg);
6476 	amdgpu_ring_write(ring, 0);
6477 	amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6478 				adev->virt.reg_val_offs * 4));
6479 	amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6480 				adev->virt.reg_val_offs * 4));
6481 }
6482 
6483 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6484 				  uint32_t val)
6485 {
6486 	uint32_t cmd;
6487 
6488 	switch (ring->funcs->type) {
6489 	case AMDGPU_RING_TYPE_GFX:
6490 		cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6491 		break;
6492 	case AMDGPU_RING_TYPE_KIQ:
6493 		cmd = 1 << 16; /* no inc addr */
6494 		break;
6495 	default:
6496 		cmd = WR_CONFIRM;
6497 		break;
6498 	}
6499 
6500 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6501 	amdgpu_ring_write(ring, cmd);
6502 	amdgpu_ring_write(ring, reg);
6503 	amdgpu_ring_write(ring, 0);
6504 	amdgpu_ring_write(ring, val);
6505 }
6506 
6507 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
6508 {
6509 	struct amdgpu_device *adev = ring->adev;
6510 	uint32_t value = 0;
6511 
6512 	value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6513 	value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6514 	value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6515 	value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6516 	WREG32(mmSQ_CMD, value);
6517 }
6518 
6519 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6520 						 enum amdgpu_interrupt_state state)
6521 {
6522 	WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,
6523 		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6524 }
6525 
6526 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6527 						     int me, int pipe,
6528 						     enum amdgpu_interrupt_state state)
6529 {
6530 	u32 mec_int_cntl, mec_int_cntl_reg;
6531 
6532 	/*
6533 	 * amdgpu controls only the first MEC. That's why this function only
6534 	 * handles the setting of interrupts for this specific MEC. All other
6535 	 * pipes' interrupts are set by amdkfd.
6536 	 */
6537 
6538 	if (me == 1) {
6539 		switch (pipe) {
6540 		case 0:
6541 			mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
6542 			break;
6543 		case 1:
6544 			mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;
6545 			break;
6546 		case 2:
6547 			mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;
6548 			break;
6549 		case 3:
6550 			mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;
6551 			break;
6552 		default:
6553 			DRM_DEBUG("invalid pipe %d\n", pipe);
6554 			return;
6555 		}
6556 	} else {
6557 		DRM_DEBUG("invalid me %d\n", me);
6558 		return;
6559 	}
6560 
6561 	switch (state) {
6562 	case AMDGPU_IRQ_STATE_DISABLE:
6563 		mec_int_cntl = RREG32(mec_int_cntl_reg);
6564 		mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6565 		WREG32(mec_int_cntl_reg, mec_int_cntl);
6566 		break;
6567 	case AMDGPU_IRQ_STATE_ENABLE:
6568 		mec_int_cntl = RREG32(mec_int_cntl_reg);
6569 		mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6570 		WREG32(mec_int_cntl_reg, mec_int_cntl);
6571 		break;
6572 	default:
6573 		break;
6574 	}
6575 }
6576 
6577 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6578 					     struct amdgpu_irq_src *source,
6579 					     unsigned type,
6580 					     enum amdgpu_interrupt_state state)
6581 {
6582 	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,
6583 		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6584 
6585 	return 0;
6586 }
6587 
6588 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6589 					      struct amdgpu_irq_src *source,
6590 					      unsigned type,
6591 					      enum amdgpu_interrupt_state state)
6592 {
6593 	WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,
6594 		     state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6595 
6596 	return 0;
6597 }
6598 
6599 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6600 					    struct amdgpu_irq_src *src,
6601 					    unsigned type,
6602 					    enum amdgpu_interrupt_state state)
6603 {
6604 	switch (type) {
6605 	case AMDGPU_CP_IRQ_GFX_EOP:
6606 		gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);
6607 		break;
6608 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6609 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6610 		break;
6611 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6612 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6613 		break;
6614 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6615 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6616 		break;
6617 	case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6618 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6619 		break;
6620 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6621 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6622 		break;
6623 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6624 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6625 		break;
6626 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6627 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6628 		break;
6629 	case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6630 		gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6631 		break;
6632 	default:
6633 		break;
6634 	}
6635 	return 0;
6636 }
6637 
6638 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev,
6639 					 struct amdgpu_irq_src *source,
6640 					 unsigned int type,
6641 					 enum amdgpu_interrupt_state state)
6642 {
6643 	int enable_flag;
6644 
6645 	switch (state) {
6646 	case AMDGPU_IRQ_STATE_DISABLE:
6647 		enable_flag = 0;
6648 		break;
6649 
6650 	case AMDGPU_IRQ_STATE_ENABLE:
6651 		enable_flag = 1;
6652 		break;
6653 
6654 	default:
6655 		return -EINVAL;
6656 	}
6657 
6658 	WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6659 	WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6660 	WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6661 	WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6662 	WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6663 	WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6664 		     enable_flag);
6665 	WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6666 		     enable_flag);
6667 	WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6668 		     enable_flag);
6669 	WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6670 		     enable_flag);
6671 	WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6672 		     enable_flag);
6673 	WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6674 		     enable_flag);
6675 	WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6676 		     enable_flag);
6677 	WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6678 		     enable_flag);
6679 
6680 	return 0;
6681 }
6682 
6683 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev,
6684 				     struct amdgpu_irq_src *source,
6685 				     unsigned int type,
6686 				     enum amdgpu_interrupt_state state)
6687 {
6688 	int enable_flag;
6689 
6690 	switch (state) {
6691 	case AMDGPU_IRQ_STATE_DISABLE:
6692 		enable_flag = 1;
6693 		break;
6694 
6695 	case AMDGPU_IRQ_STATE_ENABLE:
6696 		enable_flag = 0;
6697 		break;
6698 
6699 	default:
6700 		return -EINVAL;
6701 	}
6702 
6703 	WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL,
6704 		     enable_flag);
6705 
6706 	return 0;
6707 }
6708 
6709 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,
6710 			    struct amdgpu_irq_src *source,
6711 			    struct amdgpu_iv_entry *entry)
6712 {
6713 	int i;
6714 	u8 me_id, pipe_id, queue_id;
6715 	struct amdgpu_ring *ring;
6716 
6717 	DRM_DEBUG("IH: CP EOP\n");
6718 	me_id = (entry->ring_id & 0x0c) >> 2;
6719 	pipe_id = (entry->ring_id & 0x03) >> 0;
6720 	queue_id = (entry->ring_id & 0x70) >> 4;
6721 
6722 	switch (me_id) {
6723 	case 0:
6724 		amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6725 		break;
6726 	case 1:
6727 	case 2:
6728 		for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6729 			ring = &adev->gfx.compute_ring[i];
6730 			/* Per-queue interrupt is supported for MEC starting from VI.
6731 			  * The interrupt can only be enabled/disabled per pipe instead of per queue.
6732 			  */
6733 			if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6734 				amdgpu_fence_process(ring);
6735 		}
6736 		break;
6737 	}
6738 	return 0;
6739 }
6740 
6741 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,
6742 				 struct amdgpu_irq_src *source,
6743 				 struct amdgpu_iv_entry *entry)
6744 {
6745 	DRM_ERROR("Illegal register access in command stream\n");
6746 	schedule_work(&adev->reset_work);
6747 	return 0;
6748 }
6749 
6750 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
6751 				  struct amdgpu_irq_src *source,
6752 				  struct amdgpu_iv_entry *entry)
6753 {
6754 	DRM_ERROR("Illegal instruction in command stream\n");
6755 	schedule_work(&adev->reset_work);
6756 	return 0;
6757 }
6758 
6759 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,
6760 				     struct amdgpu_irq_src *source,
6761 				     struct amdgpu_iv_entry *entry)
6762 {
6763 	DRM_ERROR("CP EDC/ECC error detected.");
6764 	return 0;
6765 }
6766 
6767 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data)
6768 {
6769 	u32 enc, se_id, sh_id, cu_id;
6770 	char type[20];
6771 	int sq_edc_source = -1;
6772 
6773 	enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING);
6774 	se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID);
6775 
6776 	switch (enc) {
6777 		case 0:
6778 			DRM_INFO("SQ general purpose intr detected:"
6779 					"se_id %d, immed_overflow %d, host_reg_overflow %d,"
6780 					"host_cmd_overflow %d, cmd_timestamp %d,"
6781 					"reg_timestamp %d, thread_trace_buff_full %d,"
6782 					"wlt %d, thread_trace %d.\n",
6783 					se_id,
6784 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW),
6785 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW),
6786 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW),
6787 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP),
6788 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP),
6789 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL),
6790 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT),
6791 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE)
6792 					);
6793 			break;
6794 		case 1:
6795 		case 2:
6796 
6797 			cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID);
6798 			sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID);
6799 
6800 			/*
6801 			 * This function can be called either directly from ISR
6802 			 * or from BH in which case we can access SQ_EDC_INFO
6803 			 * instance
6804 			 */
6805 			if (in_task()) {
6806 				mutex_lock(&adev->grbm_idx_mutex);
6807 				gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id);
6808 
6809 				sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE);
6810 
6811 				gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6812 				mutex_unlock(&adev->grbm_idx_mutex);
6813 			}
6814 
6815 			if (enc == 1)
6816 				sprintf(type, "instruction intr");
6817 			else
6818 				sprintf(type, "EDC/ECC error");
6819 
6820 			DRM_INFO(
6821 				"SQ %s detected: "
6822 					"se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d "
6823 					"trap %s, sq_ed_info.source %s.\n",
6824 					type, se_id, sh_id, cu_id,
6825 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID),
6826 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID),
6827 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID),
6828 					REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false",
6829 					(sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable"
6830 				);
6831 			break;
6832 		default:
6833 			DRM_ERROR("SQ invalid encoding type\n.");
6834 	}
6835 }
6836 
6837 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)
6838 {
6839 
6840 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work);
6841 	struct sq_work *sq_work = container_of(work, struct sq_work, work);
6842 
6843 	gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data);
6844 }
6845 
6846 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
6847 			   struct amdgpu_irq_src *source,
6848 			   struct amdgpu_iv_entry *entry)
6849 {
6850 	unsigned ih_data = entry->src_data[0];
6851 
6852 	/*
6853 	 * Try to submit work so SQ_EDC_INFO can be accessed from
6854 	 * BH. If previous work submission hasn't finished yet
6855 	 * just print whatever info is possible directly from the ISR.
6856 	 */
6857 	if (work_pending(&adev->gfx.sq_work.work)) {
6858 		gfx_v8_0_parse_sq_irq(adev, ih_data);
6859 	} else {
6860 		adev->gfx.sq_work.ih_data = ih_data;
6861 		schedule_work(&adev->gfx.sq_work.work);
6862 	}
6863 
6864 	return 0;
6865 }
6866 
6867 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
6868 	.name = "gfx_v8_0",
6869 	.early_init = gfx_v8_0_early_init,
6870 	.late_init = gfx_v8_0_late_init,
6871 	.sw_init = gfx_v8_0_sw_init,
6872 	.sw_fini = gfx_v8_0_sw_fini,
6873 	.hw_init = gfx_v8_0_hw_init,
6874 	.hw_fini = gfx_v8_0_hw_fini,
6875 	.suspend = gfx_v8_0_suspend,
6876 	.resume = gfx_v8_0_resume,
6877 	.is_idle = gfx_v8_0_is_idle,
6878 	.wait_for_idle = gfx_v8_0_wait_for_idle,
6879 	.check_soft_reset = gfx_v8_0_check_soft_reset,
6880 	.pre_soft_reset = gfx_v8_0_pre_soft_reset,
6881 	.soft_reset = gfx_v8_0_soft_reset,
6882 	.post_soft_reset = gfx_v8_0_post_soft_reset,
6883 	.set_clockgating_state = gfx_v8_0_set_clockgating_state,
6884 	.set_powergating_state = gfx_v8_0_set_powergating_state,
6885 	.get_clockgating_state = gfx_v8_0_get_clockgating_state,
6886 };
6887 
6888 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
6889 	.type = AMDGPU_RING_TYPE_GFX,
6890 	.align_mask = 0xff,
6891 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6892 	.support_64bit_ptrs = false,
6893 	.get_rptr = gfx_v8_0_ring_get_rptr,
6894 	.get_wptr = gfx_v8_0_ring_get_wptr_gfx,
6895 	.set_wptr = gfx_v8_0_ring_set_wptr_gfx,
6896 	.emit_frame_size = /* maximum 215dw if count 16 IBs in */
6897 		5 +  /* COND_EXEC */
6898 		7 +  /* PIPELINE_SYNC */
6899 		VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */
6900 		8 +  /* FENCE for VM_FLUSH */
6901 		20 + /* GDS switch */
6902 		4 + /* double SWITCH_BUFFER,
6903 		       the first COND_EXEC jump to the place just
6904 			   prior to this double SWITCH_BUFFER  */
6905 		5 + /* COND_EXEC */
6906 		7 +	 /*	HDP_flush */
6907 		4 +	 /*	VGT_flush */
6908 		14 + /*	CE_META */
6909 		31 + /*	DE_META */
6910 		3 + /* CNTX_CTRL */
6911 		5 + /* HDP_INVL */
6912 		8 + 8 + /* FENCE x2 */
6913 		2, /* SWITCH_BUFFER */
6914 	.emit_ib_size =	4, /* gfx_v8_0_ring_emit_ib_gfx */
6915 	.emit_ib = gfx_v8_0_ring_emit_ib_gfx,
6916 	.emit_fence = gfx_v8_0_ring_emit_fence_gfx,
6917 	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6918 	.emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6919 	.emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6920 	.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6921 	.test_ring = gfx_v8_0_ring_test_ring,
6922 	.test_ib = gfx_v8_0_ring_test_ib,
6923 	.insert_nop = amdgpu_ring_insert_nop,
6924 	.pad_ib = amdgpu_ring_generic_pad_ib,
6925 	.emit_switch_buffer = gfx_v8_ring_emit_sb,
6926 	.emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
6927 	.init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec,
6928 	.patch_cond_exec = gfx_v8_0_ring_emit_patch_cond_exec,
6929 	.emit_wreg = gfx_v8_0_ring_emit_wreg,
6930 	.soft_recovery = gfx_v8_0_ring_soft_recovery,
6931 };
6932 
6933 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
6934 	.type = AMDGPU_RING_TYPE_COMPUTE,
6935 	.align_mask = 0xff,
6936 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6937 	.support_64bit_ptrs = false,
6938 	.get_rptr = gfx_v8_0_ring_get_rptr,
6939 	.get_wptr = gfx_v8_0_ring_get_wptr_compute,
6940 	.set_wptr = gfx_v8_0_ring_set_wptr_compute,
6941 	.emit_frame_size =
6942 		20 + /* gfx_v8_0_ring_emit_gds_switch */
6943 		7 + /* gfx_v8_0_ring_emit_hdp_flush */
6944 		5 + /* hdp_invalidate */
6945 		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6946 		VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */
6947 		7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
6948 	.emit_ib_size =	4, /* gfx_v8_0_ring_emit_ib_compute */
6949 	.emit_ib = gfx_v8_0_ring_emit_ib_compute,
6950 	.emit_fence = gfx_v8_0_ring_emit_fence_compute,
6951 	.emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6952 	.emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6953 	.emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6954 	.emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6955 	.test_ring = gfx_v8_0_ring_test_ring,
6956 	.test_ib = gfx_v8_0_ring_test_ib,
6957 	.insert_nop = amdgpu_ring_insert_nop,
6958 	.pad_ib = amdgpu_ring_generic_pad_ib,
6959 	.set_priority = gfx_v8_0_ring_set_priority_compute,
6960 	.emit_wreg = gfx_v8_0_ring_emit_wreg,
6961 };
6962 
6963 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
6964 	.type = AMDGPU_RING_TYPE_KIQ,
6965 	.align_mask = 0xff,
6966 	.nop = PACKET3(PACKET3_NOP, 0x3FFF),
6967 	.support_64bit_ptrs = false,
6968 	.get_rptr = gfx_v8_0_ring_get_rptr,
6969 	.get_wptr = gfx_v8_0_ring_get_wptr_compute,
6970 	.set_wptr = gfx_v8_0_ring_set_wptr_compute,
6971 	.emit_frame_size =
6972 		20 + /* gfx_v8_0_ring_emit_gds_switch */
6973 		7 + /* gfx_v8_0_ring_emit_hdp_flush */
6974 		5 + /* hdp_invalidate */
6975 		7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6976 		17 + /* gfx_v8_0_ring_emit_vm_flush */
6977 		7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6978 	.emit_ib_size =	4, /* gfx_v8_0_ring_emit_ib_compute */
6979 	.emit_ib = gfx_v8_0_ring_emit_ib_compute,
6980 	.emit_fence = gfx_v8_0_ring_emit_fence_kiq,
6981 	.test_ring = gfx_v8_0_ring_test_ring,
6982 	.test_ib = gfx_v8_0_ring_test_ib,
6983 	.insert_nop = amdgpu_ring_insert_nop,
6984 	.pad_ib = amdgpu_ring_generic_pad_ib,
6985 	.emit_rreg = gfx_v8_0_ring_emit_rreg,
6986 	.emit_wreg = gfx_v8_0_ring_emit_wreg,
6987 };
6988 
6989 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
6990 {
6991 	int i;
6992 
6993 	adev->gfx.kiq.ring.funcs = &gfx_v8_0_ring_funcs_kiq;
6994 
6995 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6996 		adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;
6997 
6998 	for (i = 0; i < adev->gfx.num_compute_rings; i++)
6999 		adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute;
7000 }
7001 
7002 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = {
7003 	.set = gfx_v8_0_set_eop_interrupt_state,
7004 	.process = gfx_v8_0_eop_irq,
7005 };
7006 
7007 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = {
7008 	.set = gfx_v8_0_set_priv_reg_fault_state,
7009 	.process = gfx_v8_0_priv_reg_irq,
7010 };
7011 
7012 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {
7013 	.set = gfx_v8_0_set_priv_inst_fault_state,
7014 	.process = gfx_v8_0_priv_inst_irq,
7015 };
7016 
7017 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = {
7018 	.set = gfx_v8_0_set_cp_ecc_int_state,
7019 	.process = gfx_v8_0_cp_ecc_error_irq,
7020 };
7021 
7022 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = {
7023 	.set = gfx_v8_0_set_sq_int_state,
7024 	.process = gfx_v8_0_sq_irq,
7025 };
7026 
7027 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
7028 {
7029 	adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7030 	adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs;
7031 
7032 	adev->gfx.priv_reg_irq.num_types = 1;
7033 	adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs;
7034 
7035 	adev->gfx.priv_inst_irq.num_types = 1;
7036 	adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;
7037 
7038 	adev->gfx.cp_ecc_error_irq.num_types = 1;
7039 	adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs;
7040 
7041 	adev->gfx.sq_irq.num_types = 1;
7042 	adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs;
7043 }
7044 
7045 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
7046 {
7047 	adev->gfx.rlc.funcs = &iceland_rlc_funcs;
7048 }
7049 
7050 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
7051 {
7052 	/* init asci gds info */
7053 	adev->gds.mem.total_size = RREG32(mmGDS_VMID0_SIZE);
7054 	adev->gds.gws.total_size = 64;
7055 	adev->gds.oa.total_size = 16;
7056 
7057 	if (adev->gds.mem.total_size == 64 * 1024) {
7058 		adev->gds.mem.gfx_partition_size = 4096;
7059 		adev->gds.mem.cs_partition_size = 4096;
7060 
7061 		adev->gds.gws.gfx_partition_size = 4;
7062 		adev->gds.gws.cs_partition_size = 4;
7063 
7064 		adev->gds.oa.gfx_partition_size = 4;
7065 		adev->gds.oa.cs_partition_size = 1;
7066 	} else {
7067 		adev->gds.mem.gfx_partition_size = 1024;
7068 		adev->gds.mem.cs_partition_size = 1024;
7069 
7070 		adev->gds.gws.gfx_partition_size = 16;
7071 		adev->gds.gws.cs_partition_size = 16;
7072 
7073 		adev->gds.oa.gfx_partition_size = 4;
7074 		adev->gds.oa.cs_partition_size = 4;
7075 	}
7076 }
7077 
7078 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7079 						 u32 bitmap)
7080 {
7081 	u32 data;
7082 
7083 	if (!bitmap)
7084 		return;
7085 
7086 	data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7087 	data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7088 
7089 	WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
7090 }
7091 
7092 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7093 {
7094 	u32 data, mask;
7095 
7096 	data =  RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
7097 		RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
7098 
7099 	mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7100 
7101 	return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
7102 }
7103 
7104 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
7105 {
7106 	int i, j, k, counter, active_cu_number = 0;
7107 	u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7108 	struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
7109 	unsigned disable_masks[4 * 2];
7110 	u32 ao_cu_num;
7111 
7112 	memset(cu_info, 0, sizeof(*cu_info));
7113 
7114 	if (adev->flags & AMD_IS_APU)
7115 		ao_cu_num = 2;
7116 	else
7117 		ao_cu_num = adev->gfx.config.max_cu_per_sh;
7118 
7119 	amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
7120 
7121 	mutex_lock(&adev->grbm_idx_mutex);
7122 	for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7123 		for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7124 			mask = 1;
7125 			ao_bitmap = 0;
7126 			counter = 0;
7127 			gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
7128 			if (i < 4 && j < 2)
7129 				gfx_v8_0_set_user_cu_inactive_bitmap(
7130 					adev, disable_masks[i * 2 + j]);
7131 			bitmap = gfx_v8_0_get_cu_active_bitmap(adev);
7132 			cu_info->bitmap[i][j] = bitmap;
7133 
7134 			for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7135 				if (bitmap & mask) {
7136 					if (counter < ao_cu_num)
7137 						ao_bitmap |= mask;
7138 					counter ++;
7139 				}
7140 				mask <<= 1;
7141 			}
7142 			active_cu_number += counter;
7143 			if (i < 2 && j < 2)
7144 				ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7145 			cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
7146 		}
7147 	}
7148 	gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
7149 	mutex_unlock(&adev->grbm_idx_mutex);
7150 
7151 	cu_info->number = active_cu_number;
7152 	cu_info->ao_cu_mask = ao_cu_mask;
7153 	cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7154 	cu_info->max_waves_per_simd = 10;
7155 	cu_info->max_scratch_slots_per_cu = 32;
7156 	cu_info->wave_front_size = 64;
7157 	cu_info->lds_size = 64;
7158 }
7159 
7160 const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
7161 {
7162 	.type = AMD_IP_BLOCK_TYPE_GFX,
7163 	.major = 8,
7164 	.minor = 0,
7165 	.rev = 0,
7166 	.funcs = &gfx_v8_0_ip_funcs,
7167 };
7168 
7169 const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
7170 {
7171 	.type = AMD_IP_BLOCK_TYPE_GFX,
7172 	.major = 8,
7173 	.minor = 1,
7174 	.rev = 0,
7175 	.funcs = &gfx_v8_0_ip_funcs,
7176 };
7177 
7178 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
7179 {
7180 	uint64_t ce_payload_addr;
7181 	int cnt_ce;
7182 	union {
7183 		struct vi_ce_ib_state regular;
7184 		struct vi_ce_ib_state_chained_ib chained;
7185 	} ce_payload = {};
7186 
7187 	if (ring->adev->virt.chained_ib_support) {
7188 		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7189 			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
7190 		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
7191 	} else {
7192 		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7193 			offsetof(struct vi_gfx_meta_data, ce_payload);
7194 		cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
7195 	}
7196 
7197 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));
7198 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
7199 				WRITE_DATA_DST_SEL(8) |
7200 				WR_CONFIRM) |
7201 				WRITE_DATA_CACHE_POLICY(0));
7202 	amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));
7203 	amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));
7204 	amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
7205 }
7206 
7207 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
7208 {
7209 	uint64_t de_payload_addr, gds_addr, csa_addr;
7210 	int cnt_de;
7211 	union {
7212 		struct vi_de_ib_state regular;
7213 		struct vi_de_ib_state_chained_ib chained;
7214 	} de_payload = {};
7215 
7216 	csa_addr = amdgpu_csa_vaddr(ring->adev);
7217 	gds_addr = csa_addr + 4096;
7218 	if (ring->adev->virt.chained_ib_support) {
7219 		de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
7220 		de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);
7221 		de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload);
7222 		cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;
7223 	} else {
7224 		de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);
7225 		de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);
7226 		de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload);
7227 		cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;
7228 	}
7229 
7230 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));
7231 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
7232 				WRITE_DATA_DST_SEL(8) |
7233 				WR_CONFIRM) |
7234 				WRITE_DATA_CACHE_POLICY(0));
7235 	amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));
7236 	amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));
7237 	amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);
7238 }
7239