1 /*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 *
22 */
23
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
29
30 #include "amdgpu.h"
31 #include "amdgpu_gfx.h"
32 #include "amdgpu_ring.h"
33 #include "vi.h"
34 #include "vi_structs.h"
35 #include "vid.h"
36 #include "amdgpu_ucode.h"
37 #include "amdgpu_atombios.h"
38 #include "atombios_i2c.h"
39 #include "clearstate_vi.h"
40
41 #include "gmc/gmc_8_2_d.h"
42 #include "gmc/gmc_8_2_sh_mask.h"
43
44 #include "oss/oss_3_0_d.h"
45 #include "oss/oss_3_0_sh_mask.h"
46
47 #include "bif/bif_5_0_d.h"
48 #include "bif/bif_5_0_sh_mask.h"
49 #include "gca/gfx_8_0_d.h"
50 #include "gca/gfx_8_0_enum.h"
51 #include "gca/gfx_8_0_sh_mask.h"
52
53 #include "dce/dce_10_0_d.h"
54 #include "dce/dce_10_0_sh_mask.h"
55
56 #include "smu/smu_7_1_3_d.h"
57
58 #include "ivsrcid/ivsrcid_vislands30.h"
59
60 #define GFX8_NUM_GFX_RINGS 1
61 #define GFX8_MEC_HPD_SIZE 4096
62
63 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001
64 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001
65 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002
66 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003
67
68 #define ARRAY_MODE(x) ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT)
69 #define PIPE_CONFIG(x) ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT)
70 #define TILE_SPLIT(x) ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT)
71 #define MICRO_TILE_MODE_NEW(x) ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT)
72 #define SAMPLE_SPLIT(x) ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT)
73 #define BANK_WIDTH(x) ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT)
74 #define BANK_HEIGHT(x) ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT)
75 #define MACRO_TILE_ASPECT(x) ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT)
76 #define NUM_BANKS(x) ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT)
77
78 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK 0x00000001L
79 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK 0x00000002L
80 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK 0x00000004L
81 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK 0x00000008L
82 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK 0x00000010L
83 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK 0x00000020L
84
85 /* BPM SERDES CMD */
86 #define SET_BPM_SERDES_CMD 1
87 #define CLE_BPM_SERDES_CMD 0
88
89 /* BPM Register Address*/
90 enum {
91 BPM_REG_CGLS_EN = 0, /* Enable/Disable CGLS */
92 BPM_REG_CGLS_ON, /* ON/OFF CGLS: shall be controlled by RLC FW */
93 BPM_REG_CGCG_OVERRIDE, /* Set/Clear CGCG Override */
94 BPM_REG_MGCG_OVERRIDE, /* Set/Clear MGCG Override */
95 BPM_REG_FGCG_OVERRIDE, /* Set/Clear FGCG Override */
96 BPM_REG_FGCG_MAX
97 };
98
99 #define RLC_FormatDirectRegListLength 14
100
101 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin");
102 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin");
103 MODULE_FIRMWARE("amdgpu/carrizo_me.bin");
104 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin");
105 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin");
106 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin");
107
108 MODULE_FIRMWARE("amdgpu/stoney_ce.bin");
109 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin");
110 MODULE_FIRMWARE("amdgpu/stoney_me.bin");
111 MODULE_FIRMWARE("amdgpu/stoney_mec.bin");
112 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin");
113
114 MODULE_FIRMWARE("amdgpu/tonga_ce.bin");
115 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin");
116 MODULE_FIRMWARE("amdgpu/tonga_me.bin");
117 MODULE_FIRMWARE("amdgpu/tonga_mec.bin");
118 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin");
119 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin");
120
121 MODULE_FIRMWARE("amdgpu/topaz_ce.bin");
122 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin");
123 MODULE_FIRMWARE("amdgpu/topaz_me.bin");
124 MODULE_FIRMWARE("amdgpu/topaz_mec.bin");
125 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin");
126
127 MODULE_FIRMWARE("amdgpu/fiji_ce.bin");
128 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin");
129 MODULE_FIRMWARE("amdgpu/fiji_me.bin");
130 MODULE_FIRMWARE("amdgpu/fiji_mec.bin");
131 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin");
132 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin");
133
134 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin");
135 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin");
136 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin");
137 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin");
138 MODULE_FIRMWARE("amdgpu/polaris10_me.bin");
139 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin");
140 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin");
141 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin");
142 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin");
143 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin");
144 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin");
145
146 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin");
147 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin");
148 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin");
149 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin");
150 MODULE_FIRMWARE("amdgpu/polaris11_me.bin");
151 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin");
152 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin");
153 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin");
154 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin");
155 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin");
156 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin");
157
158 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin");
159 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin");
160 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin");
161 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin");
162 MODULE_FIRMWARE("amdgpu/polaris12_me.bin");
163 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin");
164 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin");
165 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin");
166 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin");
167 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin");
168 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin");
169
170 MODULE_FIRMWARE("amdgpu/vegam_ce.bin");
171 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin");
172 MODULE_FIRMWARE("amdgpu/vegam_me.bin");
173 MODULE_FIRMWARE("amdgpu/vegam_mec.bin");
174 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin");
175 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin");
176
177 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] =
178 {
179 {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0},
180 {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1},
181 {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2},
182 {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3},
183 {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4},
184 {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5},
185 {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6},
186 {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7},
187 {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8},
188 {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9},
189 {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10},
190 {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11},
191 {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12},
192 {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13},
193 {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14},
194 {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15}
195 };
196
197 static const u32 golden_settings_tonga_a11[] =
198 {
199 mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208,
200 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
201 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
202 mmGB_GPU_ID, 0x0000000f, 0x00000000,
203 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
204 mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc,
205 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
206 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
207 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
208 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
209 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
210 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
211 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb,
212 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b,
213 mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876,
214 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
215 };
216
217 static const u32 tonga_golden_common_all[] =
218 {
219 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
220 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
221 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
222 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
223 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
224 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
225 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
226 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
227 };
228
229 static const u32 tonga_mgcg_cgcg_init[] =
230 {
231 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
232 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
233 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
234 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
235 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
236 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
237 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
238 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
239 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
240 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
241 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
242 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
243 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
244 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
245 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
246 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
247 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
248 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
249 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
250 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
251 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
252 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
253 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
254 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
255 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
256 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
257 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
258 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
259 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
260 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
261 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
262 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
263 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
264 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
265 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
266 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
267 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
268 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
269 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
270 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
271 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
272 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
273 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
274 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
275 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
276 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
277 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
278 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
279 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
280 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
281 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
282 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
283 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
284 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
285 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
286 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
287 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
288 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
289 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
290 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
291 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
292 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
293 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
294 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
295 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
296 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
297 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
298 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
299 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
300 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
301 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
302 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
303 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
304 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
305 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
306 };
307
308 static const u32 golden_settings_vegam_a11[] =
309 {
310 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
311 mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000,
312 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
313 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
314 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
315 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
316 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a,
317 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e,
318 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
319 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
320 mmSQ_CONFIG, 0x07f80000, 0x01180000,
321 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
322 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
323 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
324 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
325 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054,
326 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
327 };
328
329 static const u32 vegam_golden_common_all[] =
330 {
331 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
332 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
333 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
334 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
335 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
336 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
337 };
338
339 static const u32 golden_settings_polaris11_a11[] =
340 {
341 mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208,
342 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
343 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
344 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
345 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
346 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
347 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
348 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
349 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
350 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
351 mmSQ_CONFIG, 0x07f80000, 0x01180000,
352 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
353 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
354 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3,
355 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
356 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210,
357 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
358 };
359
360 static const u32 polaris11_golden_common_all[] =
361 {
362 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
363 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002,
364 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
365 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
366 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
367 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
368 };
369
370 static const u32 golden_settings_polaris10_a11[] =
371 {
372 mmATC_MISC_CG, 0x000c0fc0, 0x000c0200,
373 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208,
374 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000,
375 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
376 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
377 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
378 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
379 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012,
380 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a,
381 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
382 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c,
383 mmSQ_CONFIG, 0x07f80000, 0x07180000,
384 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
385 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
386 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7,
387 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
388 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
389 };
390
391 static const u32 polaris10_golden_common_all[] =
392 {
393 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
394 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012,
395 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A,
396 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
397 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
398 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
399 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
400 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
401 };
402
403 static const u32 fiji_golden_common_all[] =
404 {
405 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
406 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a,
407 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e,
408 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003,
409 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
410 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
411 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
412 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
413 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
414 mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009,
415 };
416
417 static const u32 golden_settings_fiji_a10[] =
418 {
419 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040,
420 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
421 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
422 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
423 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
424 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
425 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
426 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
427 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
428 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff,
429 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004,
430 };
431
432 static const u32 fiji_mgcg_cgcg_init[] =
433 {
434 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
435 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
436 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
437 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
438 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
439 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
440 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100,
441 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
442 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
443 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
444 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
445 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
446 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
447 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
448 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
449 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
450 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
451 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
452 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
453 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
454 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
455 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
456 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
457 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
458 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
459 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
460 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
461 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
462 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
463 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
464 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
465 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
466 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
467 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
468 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
469 };
470
471 static const u32 golden_settings_iceland_a11[] =
472 {
473 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
474 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
475 mmDB_DEBUG3, 0xc0000000, 0xc0000000,
476 mmGB_GPU_ID, 0x0000000f, 0x00000000,
477 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
478 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
479 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002,
480 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000,
481 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
482 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
483 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
484 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
485 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
486 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1,
487 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000,
488 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010,
489 };
490
491 static const u32 iceland_golden_common_all[] =
492 {
493 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
494 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
495 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
496 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
497 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
498 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
499 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
500 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
501 };
502
503 static const u32 iceland_mgcg_cgcg_init[] =
504 {
505 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
506 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
507 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
508 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
509 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100,
510 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100,
511 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100,
512 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
513 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
514 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
515 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
516 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
517 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
518 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
519 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
520 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
521 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
522 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
523 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
524 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
525 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
526 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
527 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100,
528 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
529 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
530 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
531 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
532 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
533 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
534 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
535 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
536 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
537 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
538 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
539 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
540 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
541 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
542 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
543 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
544 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
545 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
546 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
547 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
548 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
549 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
550 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
551 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
552 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
553 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
554 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
555 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
556 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
557 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
558 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87,
559 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
560 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
561 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
562 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
563 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
564 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
565 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
566 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
567 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
568 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c,
569 };
570
571 static const u32 cz_golden_settings_a11[] =
572 {
573 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040,
574 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
575 mmGB_GPU_ID, 0x0000000f, 0x00000000,
576 mmPA_SC_ENHANCE, 0xffffffff, 0x00000001,
577 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
578 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c,
579 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd,
580 mmTA_CNTL_AUX, 0x000f000f, 0x00010000,
581 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
582 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
583 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3,
584 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302
585 };
586
587 static const u32 cz_golden_common_all[] =
588 {
589 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
590 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002,
591 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
592 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001,
593 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
594 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
595 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
596 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF
597 };
598
599 static const u32 cz_mgcg_cgcg_init[] =
600 {
601 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff,
602 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
603 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
604 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100,
605 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100,
606 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100,
607 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100,
608 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100,
609 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100,
610 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100,
611 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100,
612 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100,
613 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100,
614 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100,
615 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100,
616 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100,
617 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100,
618 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100,
619 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100,
620 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100,
621 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100,
622 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100,
623 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100,
624 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100,
625 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100,
626 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100,
627 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100,
628 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
629 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100,
630 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100,
631 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
632 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000,
633 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
634 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
635 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005,
636 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
637 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000,
638 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
639 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007,
640 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005,
641 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
642 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000,
643 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
644 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007,
645 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005,
646 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
647 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000,
648 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
649 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007,
650 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005,
651 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
652 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000,
653 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
654 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007,
655 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005,
656 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
657 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000,
658 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
659 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007,
660 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005,
661 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
662 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000,
663 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
664 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007,
665 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005,
666 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
667 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000,
668 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002,
669 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007,
670 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005,
671 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008,
672 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200,
673 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100,
674 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
675 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001,
676 };
677
678 static const u32 stoney_golden_settings_a11[] =
679 {
680 mmDB_DEBUG2, 0xf00fffff, 0x00000400,
681 mmGB_GPU_ID, 0x0000000f, 0x00000000,
682 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001,
683 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000,
684 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c,
685 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000,
686 mmTCC_CTRL, 0x00100000, 0xf31fff7f,
687 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002,
688 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1,
689 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010,
690 };
691
692 static const u32 stoney_golden_common_all[] =
693 {
694 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
695 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000,
696 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000,
697 mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001,
698 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800,
699 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800,
700 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF,
701 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF,
702 };
703
704 static const u32 stoney_mgcg_cgcg_init[] =
705 {
706 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000,
707 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f,
708 mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
709 mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201,
710 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200,
711 };
712
713
714 static const char * const sq_edc_source_names[] = {
715 "SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred",
716 "SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch",
717 "SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return",
718 "SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR",
719 "SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS",
720 "SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS",
721 "SQ_EDC_INFO_SOURCE_TA: EDC source is TA",
722 };
723
724 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev);
725 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev);
726 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev);
727 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
728 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
729 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
730 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
731 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);
732
733 #define CG_ACLK_CNTL__ACLK_DIVIDER_MASK 0x0000007fL
734 #define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT 0x00000000L
735
gfx_v8_0_init_golden_registers(struct amdgpu_device * adev)736 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
737 {
738 uint32_t data;
739
740 switch (adev->asic_type) {
741 case CHIP_TOPAZ:
742 amdgpu_device_program_register_sequence(adev,
743 iceland_mgcg_cgcg_init,
744 ARRAY_SIZE(iceland_mgcg_cgcg_init));
745 amdgpu_device_program_register_sequence(adev,
746 golden_settings_iceland_a11,
747 ARRAY_SIZE(golden_settings_iceland_a11));
748 amdgpu_device_program_register_sequence(adev,
749 iceland_golden_common_all,
750 ARRAY_SIZE(iceland_golden_common_all));
751 break;
752 case CHIP_FIJI:
753 amdgpu_device_program_register_sequence(adev,
754 fiji_mgcg_cgcg_init,
755 ARRAY_SIZE(fiji_mgcg_cgcg_init));
756 amdgpu_device_program_register_sequence(adev,
757 golden_settings_fiji_a10,
758 ARRAY_SIZE(golden_settings_fiji_a10));
759 amdgpu_device_program_register_sequence(adev,
760 fiji_golden_common_all,
761 ARRAY_SIZE(fiji_golden_common_all));
762 break;
763
764 case CHIP_TONGA:
765 amdgpu_device_program_register_sequence(adev,
766 tonga_mgcg_cgcg_init,
767 ARRAY_SIZE(tonga_mgcg_cgcg_init));
768 amdgpu_device_program_register_sequence(adev,
769 golden_settings_tonga_a11,
770 ARRAY_SIZE(golden_settings_tonga_a11));
771 amdgpu_device_program_register_sequence(adev,
772 tonga_golden_common_all,
773 ARRAY_SIZE(tonga_golden_common_all));
774 break;
775 case CHIP_VEGAM:
776 amdgpu_device_program_register_sequence(adev,
777 golden_settings_vegam_a11,
778 ARRAY_SIZE(golden_settings_vegam_a11));
779 amdgpu_device_program_register_sequence(adev,
780 vegam_golden_common_all,
781 ARRAY_SIZE(vegam_golden_common_all));
782 break;
783 case CHIP_POLARIS11:
784 case CHIP_POLARIS12:
785 amdgpu_device_program_register_sequence(adev,
786 golden_settings_polaris11_a11,
787 ARRAY_SIZE(golden_settings_polaris11_a11));
788 amdgpu_device_program_register_sequence(adev,
789 polaris11_golden_common_all,
790 ARRAY_SIZE(polaris11_golden_common_all));
791 break;
792 case CHIP_POLARIS10:
793 amdgpu_device_program_register_sequence(adev,
794 golden_settings_polaris10_a11,
795 ARRAY_SIZE(golden_settings_polaris10_a11));
796 amdgpu_device_program_register_sequence(adev,
797 polaris10_golden_common_all,
798 ARRAY_SIZE(polaris10_golden_common_all));
799 data = RREG32_SMC(ixCG_ACLK_CNTL);
800 data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK;
801 data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT;
802 WREG32_SMC(ixCG_ACLK_CNTL, data);
803 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) &&
804 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) ||
805 (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) ||
806 (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) {
807 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD);
808 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0);
809 }
810 break;
811 case CHIP_CARRIZO:
812 amdgpu_device_program_register_sequence(adev,
813 cz_mgcg_cgcg_init,
814 ARRAY_SIZE(cz_mgcg_cgcg_init));
815 amdgpu_device_program_register_sequence(adev,
816 cz_golden_settings_a11,
817 ARRAY_SIZE(cz_golden_settings_a11));
818 amdgpu_device_program_register_sequence(adev,
819 cz_golden_common_all,
820 ARRAY_SIZE(cz_golden_common_all));
821 break;
822 case CHIP_STONEY:
823 amdgpu_device_program_register_sequence(adev,
824 stoney_mgcg_cgcg_init,
825 ARRAY_SIZE(stoney_mgcg_cgcg_init));
826 amdgpu_device_program_register_sequence(adev,
827 stoney_golden_settings_a11,
828 ARRAY_SIZE(stoney_golden_settings_a11));
829 amdgpu_device_program_register_sequence(adev,
830 stoney_golden_common_all,
831 ARRAY_SIZE(stoney_golden_common_all));
832 break;
833 default:
834 break;
835 }
836 }
837
gfx_v8_0_ring_test_ring(struct amdgpu_ring * ring)838 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
839 {
840 struct amdgpu_device *adev = ring->adev;
841 uint32_t tmp = 0;
842 unsigned i;
843 int r;
844
845 WREG32(mmSCRATCH_REG0, 0xCAFEDEAD);
846 r = amdgpu_ring_alloc(ring, 3);
847 if (r)
848 return r;
849
850 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
851 amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START);
852 amdgpu_ring_write(ring, 0xDEADBEEF);
853 amdgpu_ring_commit(ring);
854
855 for (i = 0; i < adev->usec_timeout; i++) {
856 tmp = RREG32(mmSCRATCH_REG0);
857 if (tmp == 0xDEADBEEF)
858 break;
859 udelay(1);
860 }
861
862 if (i >= adev->usec_timeout)
863 r = -ETIMEDOUT;
864
865 return r;
866 }
867
gfx_v8_0_ring_test_ib(struct amdgpu_ring * ring,long timeout)868 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
869 {
870 struct amdgpu_device *adev = ring->adev;
871 struct amdgpu_ib ib;
872 struct dma_fence *f = NULL;
873
874 unsigned int index;
875 uint64_t gpu_addr;
876 uint32_t tmp;
877 long r;
878
879 r = amdgpu_device_wb_get(adev, &index);
880 if (r)
881 return r;
882
883 gpu_addr = adev->wb.gpu_addr + (index * 4);
884 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
885 memset(&ib, 0, sizeof(ib));
886
887 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib);
888 if (r)
889 goto err1;
890
891 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
892 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
893 ib.ptr[2] = lower_32_bits(gpu_addr);
894 ib.ptr[3] = upper_32_bits(gpu_addr);
895 ib.ptr[4] = 0xDEADBEEF;
896 ib.length_dw = 5;
897
898 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
899 if (r)
900 goto err2;
901
902 r = dma_fence_wait_timeout(f, false, timeout);
903 if (r == 0) {
904 r = -ETIMEDOUT;
905 goto err2;
906 } else if (r < 0) {
907 goto err2;
908 }
909
910 tmp = adev->wb.wb[index];
911 if (tmp == 0xDEADBEEF)
912 r = 0;
913 else
914 r = -EINVAL;
915
916 err2:
917 amdgpu_ib_free(&ib, NULL);
918 dma_fence_put(f);
919 err1:
920 amdgpu_device_wb_free(adev, index);
921 return r;
922 }
923
924
gfx_v8_0_free_microcode(struct amdgpu_device * adev)925 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev)
926 {
927 amdgpu_ucode_release(&adev->gfx.pfp_fw);
928 amdgpu_ucode_release(&adev->gfx.me_fw);
929 amdgpu_ucode_release(&adev->gfx.ce_fw);
930 amdgpu_ucode_release(&adev->gfx.rlc_fw);
931 amdgpu_ucode_release(&adev->gfx.mec_fw);
932 if ((adev->asic_type != CHIP_STONEY) &&
933 (adev->asic_type != CHIP_TOPAZ))
934 amdgpu_ucode_release(&adev->gfx.mec2_fw);
935
936 kfree(adev->gfx.rlc.register_list_format);
937 }
938
gfx_v8_0_init_microcode(struct amdgpu_device * adev)939 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev)
940 {
941 const char *chip_name;
942 int err;
943 struct amdgpu_firmware_info *info = NULL;
944 const struct common_firmware_header *header = NULL;
945 const struct gfx_firmware_header_v1_0 *cp_hdr;
946 const struct rlc_firmware_header_v2_0 *rlc_hdr;
947 unsigned int *tmp = NULL, i;
948
949 DRM_DEBUG("\n");
950
951 switch (adev->asic_type) {
952 case CHIP_TOPAZ:
953 chip_name = "topaz";
954 break;
955 case CHIP_TONGA:
956 chip_name = "tonga";
957 break;
958 case CHIP_CARRIZO:
959 chip_name = "carrizo";
960 break;
961 case CHIP_FIJI:
962 chip_name = "fiji";
963 break;
964 case CHIP_STONEY:
965 chip_name = "stoney";
966 break;
967 case CHIP_POLARIS10:
968 chip_name = "polaris10";
969 break;
970 case CHIP_POLARIS11:
971 chip_name = "polaris11";
972 break;
973 case CHIP_POLARIS12:
974 chip_name = "polaris12";
975 break;
976 case CHIP_VEGAM:
977 chip_name = "vegam";
978 break;
979 default:
980 BUG();
981 }
982
983 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
984 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
985 AMDGPU_UCODE_OPTIONAL,
986 "amdgpu/%s_pfp_2.bin", chip_name);
987 if (err == -ENODEV) {
988 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
989 AMDGPU_UCODE_REQUIRED,
990 "amdgpu/%s_pfp.bin", chip_name);
991 }
992 } else {
993 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw,
994 AMDGPU_UCODE_REQUIRED,
995 "amdgpu/%s_pfp.bin", chip_name);
996 }
997 if (err)
998 goto out;
999 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1000 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1001 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1002
1003 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1004 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1005 AMDGPU_UCODE_OPTIONAL,
1006 "amdgpu/%s_me_2.bin", chip_name);
1007 if (err == -ENODEV) {
1008 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1009 AMDGPU_UCODE_REQUIRED,
1010 "amdgpu/%s_me.bin", chip_name);
1011 }
1012 } else {
1013 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw,
1014 AMDGPU_UCODE_REQUIRED,
1015 "amdgpu/%s_me.bin", chip_name);
1016 }
1017 if (err)
1018 goto out;
1019 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1020 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1021
1022 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1023
1024 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1025 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1026 AMDGPU_UCODE_OPTIONAL,
1027 "amdgpu/%s_ce_2.bin", chip_name);
1028 if (err == -ENODEV) {
1029 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1030 AMDGPU_UCODE_REQUIRED,
1031 "amdgpu/%s_ce.bin", chip_name);
1032 }
1033 } else {
1034 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw,
1035 AMDGPU_UCODE_REQUIRED,
1036 "amdgpu/%s_ce.bin", chip_name);
1037 }
1038 if (err)
1039 goto out;
1040 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1041 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1042 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1043
1044 /*
1045 * Support for MCBP/Virtualization in combination with chained IBs is
1046 * formal released on feature version #46
1047 */
1048 if (adev->gfx.ce_feature_version >= 46 &&
1049 adev->gfx.pfp_feature_version >= 46) {
1050 adev->virt.chained_ib_support = true;
1051 drm_info(adev_to_drm(adev), "Chained IB support enabled!\n");
1052 } else
1053 adev->virt.chained_ib_support = false;
1054
1055 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
1056 AMDGPU_UCODE_REQUIRED,
1057 "amdgpu/%s_rlc.bin", chip_name);
1058 if (err)
1059 goto out;
1060 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1061 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1062 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1063
1064 adev->gfx.rlc.save_and_restore_offset =
1065 le32_to_cpu(rlc_hdr->save_and_restore_offset);
1066 adev->gfx.rlc.clear_state_descriptor_offset =
1067 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1068 adev->gfx.rlc.avail_scratch_ram_locations =
1069 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1070 adev->gfx.rlc.reg_restore_list_size =
1071 le32_to_cpu(rlc_hdr->reg_restore_list_size);
1072 adev->gfx.rlc.reg_list_format_start =
1073 le32_to_cpu(rlc_hdr->reg_list_format_start);
1074 adev->gfx.rlc.reg_list_format_separate_start =
1075 le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1076 adev->gfx.rlc.starting_offsets_start =
1077 le32_to_cpu(rlc_hdr->starting_offsets_start);
1078 adev->gfx.rlc.reg_list_format_size_bytes =
1079 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1080 adev->gfx.rlc.reg_list_size_bytes =
1081 le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1082
1083 adev->gfx.rlc.register_list_format =
1084 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1085 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1086
1087 if (!adev->gfx.rlc.register_list_format) {
1088 err = -ENOMEM;
1089 goto out;
1090 }
1091
1092 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1093 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1094 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1095 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1096
1097 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1098
1099 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1100 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1101 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1102 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1103
1104 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1105 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1106 AMDGPU_UCODE_OPTIONAL,
1107 "amdgpu/%s_mec_2.bin", chip_name);
1108 if (err == -ENODEV) {
1109 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1110 AMDGPU_UCODE_REQUIRED,
1111 "amdgpu/%s_mec.bin", chip_name);
1112 }
1113 } else {
1114 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw,
1115 AMDGPU_UCODE_REQUIRED,
1116 "amdgpu/%s_mec.bin", chip_name);
1117 }
1118 if (err)
1119 goto out;
1120 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1121 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1122 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1123
1124 if ((adev->asic_type != CHIP_STONEY) &&
1125 (adev->asic_type != CHIP_TOPAZ)) {
1126 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) {
1127 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1128 AMDGPU_UCODE_OPTIONAL,
1129 "amdgpu/%s_mec2_2.bin", chip_name);
1130 if (err == -ENODEV) {
1131 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1132 AMDGPU_UCODE_REQUIRED,
1133 "amdgpu/%s_mec2.bin", chip_name);
1134 }
1135 } else {
1136 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw,
1137 AMDGPU_UCODE_REQUIRED,
1138 "amdgpu/%s_mec2.bin", chip_name);
1139 }
1140 if (!err) {
1141 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1142 adev->gfx.mec2_fw->data;
1143 adev->gfx.mec2_fw_version =
1144 le32_to_cpu(cp_hdr->header.ucode_version);
1145 adev->gfx.mec2_feature_version =
1146 le32_to_cpu(cp_hdr->ucode_feature_version);
1147 } else {
1148 err = 0;
1149 adev->gfx.mec2_fw = NULL;
1150 }
1151 }
1152
1153 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1154 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1155 info->fw = adev->gfx.pfp_fw;
1156 header = (const struct common_firmware_header *)info->fw->data;
1157 adev->firmware.fw_size +=
1158 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1159
1160 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1161 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1162 info->fw = adev->gfx.me_fw;
1163 header = (const struct common_firmware_header *)info->fw->data;
1164 adev->firmware.fw_size +=
1165 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1166
1167 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1168 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1169 info->fw = adev->gfx.ce_fw;
1170 header = (const struct common_firmware_header *)info->fw->data;
1171 adev->firmware.fw_size +=
1172 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1173
1174 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1175 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1176 info->fw = adev->gfx.rlc_fw;
1177 header = (const struct common_firmware_header *)info->fw->data;
1178 adev->firmware.fw_size +=
1179 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1180
1181 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1182 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1183 info->fw = adev->gfx.mec_fw;
1184 header = (const struct common_firmware_header *)info->fw->data;
1185 adev->firmware.fw_size +=
1186 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1187
1188 /* we need account JT in */
1189 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1190 adev->firmware.fw_size +=
1191 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE);
1192
1193 if (amdgpu_sriov_vf(adev)) {
1194 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE];
1195 info->ucode_id = AMDGPU_UCODE_ID_STORAGE;
1196 info->fw = adev->gfx.mec_fw;
1197 adev->firmware.fw_size +=
1198 ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE);
1199 }
1200
1201 if (adev->gfx.mec2_fw) {
1202 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1203 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1204 info->fw = adev->gfx.mec2_fw;
1205 header = (const struct common_firmware_header *)info->fw->data;
1206 adev->firmware.fw_size +=
1207 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1208 }
1209
1210 out:
1211 if (err) {
1212 dev_err(adev->dev, "gfx8: Failed to load firmware %s gfx firmware\n", chip_name);
1213 amdgpu_ucode_release(&adev->gfx.pfp_fw);
1214 amdgpu_ucode_release(&adev->gfx.me_fw);
1215 amdgpu_ucode_release(&adev->gfx.ce_fw);
1216 amdgpu_ucode_release(&adev->gfx.rlc_fw);
1217 amdgpu_ucode_release(&adev->gfx.mec_fw);
1218 amdgpu_ucode_release(&adev->gfx.mec2_fw);
1219 }
1220 return err;
1221 }
1222
gfx_v8_0_get_csb_buffer(struct amdgpu_device * adev,u32 * buffer)1223 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer)
1224 {
1225 u32 count = 0;
1226
1227 if (adev->gfx.rlc.cs_data == NULL)
1228 return;
1229 if (buffer == NULL)
1230 return;
1231
1232 count = amdgpu_gfx_csb_preamble_start(buffer);
1233 count = amdgpu_gfx_csb_data_parser(adev, buffer, count);
1234
1235 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
1236 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
1237 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config);
1238 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1);
1239
1240 amdgpu_gfx_csb_preamble_end(buffer, count);
1241 }
1242
gfx_v8_0_cp_jump_table_num(struct amdgpu_device * adev)1243 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev)
1244 {
1245 if (adev->asic_type == CHIP_CARRIZO)
1246 return 5;
1247 else
1248 return 4;
1249 }
1250
gfx_v8_0_rlc_init(struct amdgpu_device * adev)1251 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
1252 {
1253 const struct cs_section_def *cs_data;
1254 int r;
1255
1256 adev->gfx.rlc.cs_data = vi_cs_data;
1257
1258 cs_data = adev->gfx.rlc.cs_data;
1259
1260 if (cs_data) {
1261 /* init clear state block */
1262 r = amdgpu_gfx_rlc_init_csb(adev);
1263 if (r)
1264 return r;
1265 }
1266
1267 if ((adev->asic_type == CHIP_CARRIZO) ||
1268 (adev->asic_type == CHIP_STONEY)) {
1269 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1270 r = amdgpu_gfx_rlc_init_cpt(adev);
1271 if (r)
1272 return r;
1273 }
1274
1275 /* init spm vmid with 0xf */
1276 if (adev->gfx.rlc.funcs->update_spm_vmid)
1277 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf);
1278
1279 return 0;
1280 }
1281
gfx_v8_0_mec_fini(struct amdgpu_device * adev)1282 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev)
1283 {
1284 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1285 }
1286
gfx_v8_0_mec_init(struct amdgpu_device * adev)1287 static int gfx_v8_0_mec_init(struct amdgpu_device *adev)
1288 {
1289 int r;
1290 u32 *hpd;
1291 size_t mec_hpd_size;
1292
1293 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1294
1295 /* take ownership of the relevant compute queues */
1296 amdgpu_gfx_compute_queue_acquire(adev);
1297
1298 mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE;
1299 if (mec_hpd_size) {
1300 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1301 AMDGPU_GEM_DOMAIN_VRAM |
1302 AMDGPU_GEM_DOMAIN_GTT,
1303 &adev->gfx.mec.hpd_eop_obj,
1304 &adev->gfx.mec.hpd_eop_gpu_addr,
1305 (void **)&hpd);
1306 if (r) {
1307 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1308 return r;
1309 }
1310
1311 memset(hpd, 0, mec_hpd_size);
1312
1313 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1314 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1315 }
1316
1317 return 0;
1318 }
1319
1320 static const u32 vgpr_init_compute_shader[] =
1321 {
1322 0x7e000209, 0x7e020208,
1323 0x7e040207, 0x7e060206,
1324 0x7e080205, 0x7e0a0204,
1325 0x7e0c0203, 0x7e0e0202,
1326 0x7e100201, 0x7e120200,
1327 0x7e140209, 0x7e160208,
1328 0x7e180207, 0x7e1a0206,
1329 0x7e1c0205, 0x7e1e0204,
1330 0x7e200203, 0x7e220202,
1331 0x7e240201, 0x7e260200,
1332 0x7e280209, 0x7e2a0208,
1333 0x7e2c0207, 0x7e2e0206,
1334 0x7e300205, 0x7e320204,
1335 0x7e340203, 0x7e360202,
1336 0x7e380201, 0x7e3a0200,
1337 0x7e3c0209, 0x7e3e0208,
1338 0x7e400207, 0x7e420206,
1339 0x7e440205, 0x7e460204,
1340 0x7e480203, 0x7e4a0202,
1341 0x7e4c0201, 0x7e4e0200,
1342 0x7e500209, 0x7e520208,
1343 0x7e540207, 0x7e560206,
1344 0x7e580205, 0x7e5a0204,
1345 0x7e5c0203, 0x7e5e0202,
1346 0x7e600201, 0x7e620200,
1347 0x7e640209, 0x7e660208,
1348 0x7e680207, 0x7e6a0206,
1349 0x7e6c0205, 0x7e6e0204,
1350 0x7e700203, 0x7e720202,
1351 0x7e740201, 0x7e760200,
1352 0x7e780209, 0x7e7a0208,
1353 0x7e7c0207, 0x7e7e0206,
1354 0xbf8a0000, 0xbf810000,
1355 };
1356
1357 static const u32 sgpr_init_compute_shader[] =
1358 {
1359 0xbe8a0100, 0xbe8c0102,
1360 0xbe8e0104, 0xbe900106,
1361 0xbe920108, 0xbe940100,
1362 0xbe960102, 0xbe980104,
1363 0xbe9a0106, 0xbe9c0108,
1364 0xbe9e0100, 0xbea00102,
1365 0xbea20104, 0xbea40106,
1366 0xbea60108, 0xbea80100,
1367 0xbeaa0102, 0xbeac0104,
1368 0xbeae0106, 0xbeb00108,
1369 0xbeb20100, 0xbeb40102,
1370 0xbeb60104, 0xbeb80106,
1371 0xbeba0108, 0xbebc0100,
1372 0xbebe0102, 0xbec00104,
1373 0xbec20106, 0xbec40108,
1374 0xbec60100, 0xbec80102,
1375 0xbee60004, 0xbee70005,
1376 0xbeea0006, 0xbeeb0007,
1377 0xbee80008, 0xbee90009,
1378 0xbefc0000, 0xbf8a0000,
1379 0xbf810000, 0x00000000,
1380 };
1381
1382 static const u32 vgpr_init_regs[] =
1383 {
1384 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff,
1385 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1386 mmCOMPUTE_NUM_THREAD_X, 256*4,
1387 mmCOMPUTE_NUM_THREAD_Y, 1,
1388 mmCOMPUTE_NUM_THREAD_Z, 1,
1389 mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */
1390 mmCOMPUTE_PGM_RSRC2, 20,
1391 mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1392 mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1393 mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1394 mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1395 mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1396 mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1397 mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1398 mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1399 mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1400 mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1401 };
1402
1403 static const u32 sgpr1_init_regs[] =
1404 {
1405 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f,
1406 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */
1407 mmCOMPUTE_NUM_THREAD_X, 256*5,
1408 mmCOMPUTE_NUM_THREAD_Y, 1,
1409 mmCOMPUTE_NUM_THREAD_Z, 1,
1410 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1411 mmCOMPUTE_PGM_RSRC2, 20,
1412 mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1413 mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1414 mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1415 mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1416 mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1417 mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1418 mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1419 mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1420 mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1421 mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1422 };
1423
1424 static const u32 sgpr2_init_regs[] =
1425 {
1426 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0,
1427 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000,
1428 mmCOMPUTE_NUM_THREAD_X, 256*5,
1429 mmCOMPUTE_NUM_THREAD_Y, 1,
1430 mmCOMPUTE_NUM_THREAD_Z, 1,
1431 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */
1432 mmCOMPUTE_PGM_RSRC2, 20,
1433 mmCOMPUTE_USER_DATA_0, 0xedcedc00,
1434 mmCOMPUTE_USER_DATA_1, 0xedcedc01,
1435 mmCOMPUTE_USER_DATA_2, 0xedcedc02,
1436 mmCOMPUTE_USER_DATA_3, 0xedcedc03,
1437 mmCOMPUTE_USER_DATA_4, 0xedcedc04,
1438 mmCOMPUTE_USER_DATA_5, 0xedcedc05,
1439 mmCOMPUTE_USER_DATA_6, 0xedcedc06,
1440 mmCOMPUTE_USER_DATA_7, 0xedcedc07,
1441 mmCOMPUTE_USER_DATA_8, 0xedcedc08,
1442 mmCOMPUTE_USER_DATA_9, 0xedcedc09,
1443 };
1444
1445 static const u32 sec_ded_counter_registers[] =
1446 {
1447 mmCPC_EDC_ATC_CNT,
1448 mmCPC_EDC_SCRATCH_CNT,
1449 mmCPC_EDC_UCODE_CNT,
1450 mmCPF_EDC_ATC_CNT,
1451 mmCPF_EDC_ROQ_CNT,
1452 mmCPF_EDC_TAG_CNT,
1453 mmCPG_EDC_ATC_CNT,
1454 mmCPG_EDC_DMA_CNT,
1455 mmCPG_EDC_TAG_CNT,
1456 mmDC_EDC_CSINVOC_CNT,
1457 mmDC_EDC_RESTORE_CNT,
1458 mmDC_EDC_STATE_CNT,
1459 mmGDS_EDC_CNT,
1460 mmGDS_EDC_GRBM_CNT,
1461 mmGDS_EDC_OA_DED,
1462 mmSPI_EDC_CNT,
1463 mmSQC_ATC_EDC_GATCL1_CNT,
1464 mmSQC_EDC_CNT,
1465 mmSQ_EDC_DED_CNT,
1466 mmSQ_EDC_INFO,
1467 mmSQ_EDC_SEC_CNT,
1468 mmTCC_EDC_CNT,
1469 mmTCP_ATC_EDC_GATCL1_CNT,
1470 mmTCP_EDC_CNT,
1471 mmTD_EDC_CNT
1472 };
1473
gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device * adev)1474 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
1475 {
1476 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
1477 struct amdgpu_ib ib;
1478 struct dma_fence *f = NULL;
1479 int r, i;
1480 u32 tmp;
1481 unsigned total_size, vgpr_offset, sgpr_offset;
1482 u64 gpu_addr;
1483
1484 /* only supported on CZ */
1485 if (adev->asic_type != CHIP_CARRIZO)
1486 return 0;
1487
1488 /* bail if the compute ring is not ready */
1489 if (!ring->sched.ready)
1490 return 0;
1491
1492 tmp = RREG32(mmGB_EDC_MODE);
1493 WREG32(mmGB_EDC_MODE, 0);
1494
1495 total_size =
1496 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1497 total_size +=
1498 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1499 total_size +=
1500 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4;
1501 total_size = ALIGN(total_size, 256);
1502 vgpr_offset = total_size;
1503 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256);
1504 sgpr_offset = total_size;
1505 total_size += sizeof(sgpr_init_compute_shader);
1506
1507 /* allocate an indirect buffer to put the commands in */
1508 memset(&ib, 0, sizeof(ib));
1509 r = amdgpu_ib_get(adev, NULL, total_size,
1510 AMDGPU_IB_POOL_DIRECT, &ib);
1511 if (r) {
1512 drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r);
1513 return r;
1514 }
1515
1516 /* load the compute shaders */
1517 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++)
1518 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i];
1519
1520 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
1521 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
1522
1523 /* init the ib length to 0 */
1524 ib.length_dw = 0;
1525
1526 /* VGPR */
1527 /* write the register state for the compute dispatch */
1528 for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) {
1529 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1530 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START;
1531 ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1];
1532 }
1533 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1534 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
1535 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1536 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1537 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1538 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1539
1540 /* write dispatch packet */
1541 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1542 ib.ptr[ib.length_dw++] = 8; /* x */
1543 ib.ptr[ib.length_dw++] = 1; /* y */
1544 ib.ptr[ib.length_dw++] = 1; /* z */
1545 ib.ptr[ib.length_dw++] =
1546 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1547
1548 /* write CS partial flush packet */
1549 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1550 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1551
1552 /* SGPR1 */
1553 /* write the register state for the compute dispatch */
1554 for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) {
1555 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1556 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START;
1557 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1];
1558 }
1559 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1560 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1561 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1562 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1563 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1564 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1565
1566 /* write dispatch packet */
1567 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1568 ib.ptr[ib.length_dw++] = 8; /* x */
1569 ib.ptr[ib.length_dw++] = 1; /* y */
1570 ib.ptr[ib.length_dw++] = 1; /* z */
1571 ib.ptr[ib.length_dw++] =
1572 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1573
1574 /* write CS partial flush packet */
1575 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1576 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1577
1578 /* SGPR2 */
1579 /* write the register state for the compute dispatch */
1580 for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) {
1581 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
1582 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START;
1583 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1];
1584 }
1585 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
1586 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
1587 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
1588 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START;
1589 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
1590 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
1591
1592 /* write dispatch packet */
1593 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
1594 ib.ptr[ib.length_dw++] = 8; /* x */
1595 ib.ptr[ib.length_dw++] = 1; /* y */
1596 ib.ptr[ib.length_dw++] = 1; /* z */
1597 ib.ptr[ib.length_dw++] =
1598 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
1599
1600 /* write CS partial flush packet */
1601 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
1602 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
1603
1604 /* shedule the ib on the ring */
1605 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1606 if (r) {
1607 drm_err(adev_to_drm(adev), "ib submit failed (%d).\n", r);
1608 goto fail;
1609 }
1610
1611 /* wait for the GPU to finish processing the IB */
1612 r = dma_fence_wait(f, false);
1613 if (r) {
1614 drm_err(adev_to_drm(adev), "fence wait failed (%d).\n", r);
1615 goto fail;
1616 }
1617
1618 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2);
1619 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1);
1620 WREG32(mmGB_EDC_MODE, tmp);
1621
1622 tmp = RREG32(mmCC_GC_EDC_CONFIG);
1623 tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1;
1624 WREG32(mmCC_GC_EDC_CONFIG, tmp);
1625
1626
1627 /* read back registers to clear the counters */
1628 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++)
1629 RREG32(sec_ded_counter_registers[i]);
1630
1631 fail:
1632 amdgpu_ib_free(&ib, NULL);
1633 dma_fence_put(f);
1634
1635 return r;
1636 }
1637
gfx_v8_0_gpu_early_init(struct amdgpu_device * adev)1638 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev)
1639 {
1640 u32 gb_addr_config;
1641 u32 mc_arb_ramcfg;
1642 u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map;
1643 u32 tmp;
1644 int ret;
1645
1646 switch (adev->asic_type) {
1647 case CHIP_TOPAZ:
1648 adev->gfx.config.max_shader_engines = 1;
1649 adev->gfx.config.max_tile_pipes = 2;
1650 adev->gfx.config.max_cu_per_sh = 6;
1651 adev->gfx.config.max_sh_per_se = 1;
1652 adev->gfx.config.max_backends_per_se = 2;
1653 adev->gfx.config.max_texture_channel_caches = 2;
1654 adev->gfx.config.max_gprs = 256;
1655 adev->gfx.config.max_gs_threads = 32;
1656 adev->gfx.config.max_hw_contexts = 8;
1657
1658 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1659 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1660 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1661 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1662 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN;
1663 break;
1664 case CHIP_FIJI:
1665 adev->gfx.config.max_shader_engines = 4;
1666 adev->gfx.config.max_tile_pipes = 16;
1667 adev->gfx.config.max_cu_per_sh = 16;
1668 adev->gfx.config.max_sh_per_se = 1;
1669 adev->gfx.config.max_backends_per_se = 4;
1670 adev->gfx.config.max_texture_channel_caches = 16;
1671 adev->gfx.config.max_gprs = 256;
1672 adev->gfx.config.max_gs_threads = 32;
1673 adev->gfx.config.max_hw_contexts = 8;
1674
1675 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1676 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1677 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1678 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1679 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1680 break;
1681 case CHIP_POLARIS11:
1682 case CHIP_POLARIS12:
1683 ret = amdgpu_atombios_get_gfx_info(adev);
1684 if (ret)
1685 return ret;
1686 adev->gfx.config.max_gprs = 256;
1687 adev->gfx.config.max_gs_threads = 32;
1688 adev->gfx.config.max_hw_contexts = 8;
1689
1690 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1691 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1692 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1693 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1694 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN;
1695 break;
1696 case CHIP_POLARIS10:
1697 case CHIP_VEGAM:
1698 ret = amdgpu_atombios_get_gfx_info(adev);
1699 if (ret)
1700 return ret;
1701 adev->gfx.config.max_gprs = 256;
1702 adev->gfx.config.max_gs_threads = 32;
1703 adev->gfx.config.max_hw_contexts = 8;
1704
1705 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1706 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1707 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1708 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1709 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1710 break;
1711 case CHIP_TONGA:
1712 adev->gfx.config.max_shader_engines = 4;
1713 adev->gfx.config.max_tile_pipes = 8;
1714 adev->gfx.config.max_cu_per_sh = 8;
1715 adev->gfx.config.max_sh_per_se = 1;
1716 adev->gfx.config.max_backends_per_se = 2;
1717 adev->gfx.config.max_texture_channel_caches = 8;
1718 adev->gfx.config.max_gprs = 256;
1719 adev->gfx.config.max_gs_threads = 32;
1720 adev->gfx.config.max_hw_contexts = 8;
1721
1722 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1723 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1724 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1725 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1726 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1727 break;
1728 case CHIP_CARRIZO:
1729 adev->gfx.config.max_shader_engines = 1;
1730 adev->gfx.config.max_tile_pipes = 2;
1731 adev->gfx.config.max_sh_per_se = 1;
1732 adev->gfx.config.max_backends_per_se = 2;
1733 adev->gfx.config.max_cu_per_sh = 8;
1734 adev->gfx.config.max_texture_channel_caches = 2;
1735 adev->gfx.config.max_gprs = 256;
1736 adev->gfx.config.max_gs_threads = 32;
1737 adev->gfx.config.max_hw_contexts = 8;
1738
1739 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1740 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1741 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1742 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1743 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1744 break;
1745 case CHIP_STONEY:
1746 adev->gfx.config.max_shader_engines = 1;
1747 adev->gfx.config.max_tile_pipes = 2;
1748 adev->gfx.config.max_sh_per_se = 1;
1749 adev->gfx.config.max_backends_per_se = 1;
1750 adev->gfx.config.max_cu_per_sh = 3;
1751 adev->gfx.config.max_texture_channel_caches = 2;
1752 adev->gfx.config.max_gprs = 256;
1753 adev->gfx.config.max_gs_threads = 16;
1754 adev->gfx.config.max_hw_contexts = 8;
1755
1756 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1757 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1758 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1759 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1760 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN;
1761 break;
1762 default:
1763 adev->gfx.config.max_shader_engines = 2;
1764 adev->gfx.config.max_tile_pipes = 4;
1765 adev->gfx.config.max_cu_per_sh = 2;
1766 adev->gfx.config.max_sh_per_se = 1;
1767 adev->gfx.config.max_backends_per_se = 2;
1768 adev->gfx.config.max_texture_channel_caches = 4;
1769 adev->gfx.config.max_gprs = 256;
1770 adev->gfx.config.max_gs_threads = 32;
1771 adev->gfx.config.max_hw_contexts = 8;
1772
1773 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
1774 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
1775 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
1776 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130;
1777 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN;
1778 break;
1779 }
1780
1781 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG);
1782 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg;
1783
1784 adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg,
1785 MC_ARB_RAMCFG, NOOFBANK);
1786 adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg,
1787 MC_ARB_RAMCFG, NOOFRANKS);
1788
1789 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes;
1790 adev->gfx.config.mem_max_burst_length_bytes = 256;
1791 if (adev->flags & AMD_IS_APU) {
1792 /* Get memory bank mapping mode. */
1793 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING);
1794 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1795 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1796
1797 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING);
1798 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP);
1799 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP);
1800
1801 /* Validate settings in case only one DIMM installed. */
1802 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12))
1803 dimm00_addr_map = 0;
1804 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12))
1805 dimm01_addr_map = 0;
1806 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12))
1807 dimm10_addr_map = 0;
1808 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12))
1809 dimm11_addr_map = 0;
1810
1811 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */
1812 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */
1813 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11))
1814 adev->gfx.config.mem_row_size_in_kb = 2;
1815 else
1816 adev->gfx.config.mem_row_size_in_kb = 1;
1817 } else {
1818 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS);
1819 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
1820 if (adev->gfx.config.mem_row_size_in_kb > 4)
1821 adev->gfx.config.mem_row_size_in_kb = 4;
1822 }
1823
1824 adev->gfx.config.shader_engine_tile_size = 32;
1825 adev->gfx.config.num_gpus = 1;
1826 adev->gfx.config.multi_gpu_tile_size = 64;
1827
1828 /* fix up row size */
1829 switch (adev->gfx.config.mem_row_size_in_kb) {
1830 case 1:
1831 default:
1832 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0);
1833 break;
1834 case 2:
1835 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1);
1836 break;
1837 case 4:
1838 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2);
1839 break;
1840 }
1841 adev->gfx.config.gb_addr_config = gb_addr_config;
1842
1843 return 0;
1844 }
1845
gfx_v8_0_compute_ring_init(struct amdgpu_device * adev,int ring_id,int mec,int pipe,int queue)1846 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
1847 int mec, int pipe, int queue)
1848 {
1849 int r;
1850 unsigned irq_type;
1851 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
1852 unsigned int hw_prio;
1853
1854 ring = &adev->gfx.compute_ring[ring_id];
1855
1856 /* mec0 is me1 */
1857 ring->me = mec + 1;
1858 ring->pipe = pipe;
1859 ring->queue = queue;
1860
1861 ring->ring_obj = NULL;
1862 ring->use_doorbell = true;
1863 ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id;
1864 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
1865 + (ring_id * GFX8_MEC_HPD_SIZE);
1866 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
1867
1868 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
1869 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
1870 + ring->pipe;
1871
1872 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ?
1873 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT;
1874 /* type-2 packets are deprecated on MEC, use type-3 instead */
1875 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type,
1876 hw_prio, NULL);
1877 if (r)
1878 return r;
1879
1880
1881 return 0;
1882 }
1883
1884 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work);
1885
gfx_v8_0_sw_init(struct amdgpu_ip_block * ip_block)1886 static int gfx_v8_0_sw_init(struct amdgpu_ip_block *ip_block)
1887 {
1888 int i, j, k, r, ring_id;
1889 int xcc_id = 0;
1890 struct amdgpu_ring *ring;
1891 struct amdgpu_device *adev = ip_block->adev;
1892
1893 switch (adev->asic_type) {
1894 case CHIP_TONGA:
1895 case CHIP_CARRIZO:
1896 case CHIP_FIJI:
1897 case CHIP_POLARIS10:
1898 case CHIP_POLARIS11:
1899 case CHIP_POLARIS12:
1900 case CHIP_VEGAM:
1901 adev->gfx.mec.num_mec = 2;
1902 break;
1903 case CHIP_TOPAZ:
1904 case CHIP_STONEY:
1905 default:
1906 adev->gfx.mec.num_mec = 1;
1907 break;
1908 }
1909
1910 adev->gfx.mec.num_pipe_per_mec = 4;
1911 adev->gfx.mec.num_queue_per_pipe = 8;
1912
1913 /* EOP Event */
1914 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq);
1915 if (r)
1916 return r;
1917
1918 /* Privileged reg */
1919 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT,
1920 &adev->gfx.priv_reg_irq);
1921 if (r)
1922 return r;
1923
1924 /* Privileged inst */
1925 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT,
1926 &adev->gfx.priv_inst_irq);
1927 if (r)
1928 return r;
1929
1930 /* Add CP EDC/ECC irq */
1931 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR,
1932 &adev->gfx.cp_ecc_error_irq);
1933 if (r)
1934 return r;
1935
1936 /* SQ interrupts. */
1937 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG,
1938 &adev->gfx.sq_irq);
1939 if (r) {
1940 DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r);
1941 return r;
1942 }
1943
1944 INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func);
1945
1946 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
1947
1948 r = gfx_v8_0_init_microcode(adev);
1949 if (r) {
1950 DRM_ERROR("Failed to load gfx firmware!\n");
1951 return r;
1952 }
1953
1954 r = adev->gfx.rlc.funcs->init(adev);
1955 if (r) {
1956 DRM_ERROR("Failed to init rlc BOs!\n");
1957 return r;
1958 }
1959
1960 r = gfx_v8_0_mec_init(adev);
1961 if (r) {
1962 DRM_ERROR("Failed to init MEC BOs!\n");
1963 return r;
1964 }
1965
1966 /* set up the gfx ring */
1967 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
1968 ring = &adev->gfx.gfx_ring[i];
1969 ring->ring_obj = NULL;
1970 sprintf(ring->name, "gfx");
1971 /* no gfx doorbells on iceland */
1972 if (adev->asic_type != CHIP_TOPAZ) {
1973 ring->use_doorbell = true;
1974 ring->doorbell_index = adev->doorbell_index.gfx_ring0;
1975 }
1976
1977 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq,
1978 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
1979 AMDGPU_RING_PRIO_DEFAULT, NULL);
1980 if (r)
1981 return r;
1982 }
1983
1984
1985 /* set up the compute queues - allocate horizontally across pipes */
1986 ring_id = 0;
1987 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
1988 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
1989 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
1990 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i,
1991 k, j))
1992 continue;
1993
1994 r = gfx_v8_0_compute_ring_init(adev,
1995 ring_id,
1996 i, k, j);
1997 if (r)
1998 return r;
1999
2000 ring_id++;
2001 }
2002 }
2003 }
2004
2005 r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE, 0);
2006 if (r) {
2007 DRM_ERROR("Failed to init KIQ BOs!\n");
2008 return r;
2009 }
2010
2011 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id);
2012 if (r)
2013 return r;
2014
2015 /* create MQD for all compute queues as well as KIQ for SRIOV case */
2016 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation), 0);
2017 if (r)
2018 return r;
2019
2020 adev->gfx.ce_ram_size = 0x8000;
2021
2022 r = gfx_v8_0_gpu_early_init(adev);
2023 if (r)
2024 return r;
2025
2026 adev->gfx.gfx_supported_reset =
2027 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]);
2028 adev->gfx.compute_supported_reset =
2029 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]);
2030
2031 return 0;
2032 }
2033
gfx_v8_0_sw_fini(struct amdgpu_ip_block * ip_block)2034 static int gfx_v8_0_sw_fini(struct amdgpu_ip_block *ip_block)
2035 {
2036 struct amdgpu_device *adev = ip_block->adev;
2037 int i;
2038
2039 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2040 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2041 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2042 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2043
2044 amdgpu_gfx_mqd_sw_fini(adev, 0);
2045 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring);
2046 amdgpu_gfx_kiq_fini(adev, 0);
2047
2048 gfx_v8_0_mec_fini(adev);
2049 amdgpu_gfx_rlc_fini(adev);
2050 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj,
2051 &adev->gfx.rlc.clear_state_gpu_addr,
2052 (void **)&adev->gfx.rlc.cs_ptr);
2053 if ((adev->asic_type == CHIP_CARRIZO) ||
2054 (adev->asic_type == CHIP_STONEY)) {
2055 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2056 &adev->gfx.rlc.cp_table_gpu_addr,
2057 (void **)&adev->gfx.rlc.cp_table_ptr);
2058 }
2059 gfx_v8_0_free_microcode(adev);
2060
2061 return 0;
2062 }
2063
gfx_v8_0_tiling_mode_table_init(struct amdgpu_device * adev)2064 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
2065 {
2066 uint32_t *modearray, *mod2array;
2067 const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array);
2068 const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array);
2069 u32 reg_offset;
2070
2071 modearray = adev->gfx.config.tile_mode_array;
2072 mod2array = adev->gfx.config.macrotile_mode_array;
2073
2074 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2075 modearray[reg_offset] = 0;
2076
2077 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2078 mod2array[reg_offset] = 0;
2079
2080 switch (adev->asic_type) {
2081 case CHIP_TOPAZ:
2082 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2083 PIPE_CONFIG(ADDR_SURF_P2) |
2084 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2085 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2086 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2087 PIPE_CONFIG(ADDR_SURF_P2) |
2088 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2089 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2090 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2091 PIPE_CONFIG(ADDR_SURF_P2) |
2092 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2093 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2094 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2095 PIPE_CONFIG(ADDR_SURF_P2) |
2096 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2097 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2098 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2099 PIPE_CONFIG(ADDR_SURF_P2) |
2100 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2101 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2102 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2103 PIPE_CONFIG(ADDR_SURF_P2) |
2104 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2106 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2107 PIPE_CONFIG(ADDR_SURF_P2) |
2108 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2109 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2110 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2111 PIPE_CONFIG(ADDR_SURF_P2));
2112 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2113 PIPE_CONFIG(ADDR_SURF_P2) |
2114 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2115 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2116 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2117 PIPE_CONFIG(ADDR_SURF_P2) |
2118 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2119 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2120 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2121 PIPE_CONFIG(ADDR_SURF_P2) |
2122 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2123 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2124 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2125 PIPE_CONFIG(ADDR_SURF_P2) |
2126 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2127 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2128 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2129 PIPE_CONFIG(ADDR_SURF_P2) |
2130 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2131 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2132 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2133 PIPE_CONFIG(ADDR_SURF_P2) |
2134 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2135 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2136 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2137 PIPE_CONFIG(ADDR_SURF_P2) |
2138 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2139 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2140 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2141 PIPE_CONFIG(ADDR_SURF_P2) |
2142 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2143 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2144 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2145 PIPE_CONFIG(ADDR_SURF_P2) |
2146 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2147 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2148 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2149 PIPE_CONFIG(ADDR_SURF_P2) |
2150 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2151 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2152 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2153 PIPE_CONFIG(ADDR_SURF_P2) |
2154 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2155 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2156 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2157 PIPE_CONFIG(ADDR_SURF_P2) |
2158 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2159 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2160 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2161 PIPE_CONFIG(ADDR_SURF_P2) |
2162 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2163 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2164 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2165 PIPE_CONFIG(ADDR_SURF_P2) |
2166 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2167 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2168 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2169 PIPE_CONFIG(ADDR_SURF_P2) |
2170 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2171 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2172 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2173 PIPE_CONFIG(ADDR_SURF_P2) |
2174 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2175 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2176 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2177 PIPE_CONFIG(ADDR_SURF_P2) |
2178 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2179 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2180 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2181 PIPE_CONFIG(ADDR_SURF_P2) |
2182 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2184
2185 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2188 NUM_BANKS(ADDR_SURF_8_BANK));
2189 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2190 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2191 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2192 NUM_BANKS(ADDR_SURF_8_BANK));
2193 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2194 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2195 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2196 NUM_BANKS(ADDR_SURF_8_BANK));
2197 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2200 NUM_BANKS(ADDR_SURF_8_BANK));
2201 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2202 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2203 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2204 NUM_BANKS(ADDR_SURF_8_BANK));
2205 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2206 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2207 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2208 NUM_BANKS(ADDR_SURF_8_BANK));
2209 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2210 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2211 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2212 NUM_BANKS(ADDR_SURF_8_BANK));
2213 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2214 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2215 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2216 NUM_BANKS(ADDR_SURF_16_BANK));
2217 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2218 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2219 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2220 NUM_BANKS(ADDR_SURF_16_BANK));
2221 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2222 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2223 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2224 NUM_BANKS(ADDR_SURF_16_BANK));
2225 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2226 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2227 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2228 NUM_BANKS(ADDR_SURF_16_BANK));
2229 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2230 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2231 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2232 NUM_BANKS(ADDR_SURF_16_BANK));
2233 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2234 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2235 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2236 NUM_BANKS(ADDR_SURF_16_BANK));
2237 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2238 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2239 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2240 NUM_BANKS(ADDR_SURF_8_BANK));
2241
2242 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2243 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
2244 reg_offset != 23)
2245 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2246
2247 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2248 if (reg_offset != 7)
2249 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2250
2251 break;
2252 case CHIP_FIJI:
2253 case CHIP_VEGAM:
2254 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2255 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2256 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2257 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2258 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2259 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2260 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2261 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2262 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2263 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2264 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2265 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2266 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2267 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2268 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2270 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2271 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2272 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2274 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2275 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2276 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2277 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2278 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2279 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2280 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2281 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2282 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2283 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2286 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2287 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2288 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2289 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2290 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2291 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2292 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2293 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2294 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2295 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2296 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2297 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2298 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2299 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2300 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2301 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2302 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2304 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2305 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2306 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2307 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2308 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2309 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2310 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2311 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2312 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2313 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2314 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2315 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2316 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2317 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2318 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2320 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2321 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2322 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2323 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2324 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2325 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2326 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2327 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2328 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2329 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2330 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2332 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2333 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2335 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2336 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2337 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2338 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2339 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2340 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2341 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2342 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2343 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2344 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2345 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2346 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2347 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2348 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2349 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2350 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2351 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2352 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2354 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2355 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2356 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2357 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2358 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2359 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2360 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2361 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2362 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2363 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2364 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2365 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2366 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2367 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2368 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2369 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2370 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2371 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2372 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2373 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2374 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2376
2377 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2378 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2379 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2380 NUM_BANKS(ADDR_SURF_8_BANK));
2381 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2382 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2383 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2384 NUM_BANKS(ADDR_SURF_8_BANK));
2385 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2386 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2387 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2388 NUM_BANKS(ADDR_SURF_8_BANK));
2389 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2390 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2391 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2392 NUM_BANKS(ADDR_SURF_8_BANK));
2393 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2394 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2395 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2396 NUM_BANKS(ADDR_SURF_8_BANK));
2397 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2400 NUM_BANKS(ADDR_SURF_8_BANK));
2401 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2402 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2403 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2404 NUM_BANKS(ADDR_SURF_8_BANK));
2405 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2406 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2407 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2408 NUM_BANKS(ADDR_SURF_8_BANK));
2409 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2412 NUM_BANKS(ADDR_SURF_8_BANK));
2413 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2416 NUM_BANKS(ADDR_SURF_8_BANK));
2417 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2418 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2419 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2420 NUM_BANKS(ADDR_SURF_8_BANK));
2421 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2424 NUM_BANKS(ADDR_SURF_8_BANK));
2425 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2426 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2427 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2428 NUM_BANKS(ADDR_SURF_8_BANK));
2429 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2430 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2431 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2432 NUM_BANKS(ADDR_SURF_4_BANK));
2433
2434 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2435 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2436
2437 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2438 if (reg_offset != 7)
2439 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2440
2441 break;
2442 case CHIP_TONGA:
2443 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2444 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2445 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2446 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2447 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2448 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2449 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2450 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2451 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2452 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2453 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2454 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2455 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2457 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2458 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2459 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2460 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2461 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2462 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2463 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2464 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2465 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2466 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2467 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2468 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2469 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2470 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2471 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2472 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2473 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2474 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2475 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2476 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2477 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2478 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2480 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2481 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2482 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2483 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2484 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2485 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2486 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2487 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2488 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2489 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2490 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2491 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2492 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2493 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2494 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2495 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2496 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2497 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2498 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2499 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2500 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2501 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2502 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2503 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2504 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2505 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2506 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2507 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2508 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2509 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2510 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2512 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2513 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2514 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2516 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2517 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2518 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2519 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2520 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2521 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2522 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2523 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2524 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2525 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2526 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2529 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2530 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2531 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2532 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2533 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2534 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2535 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2536 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2537 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2538 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2539 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2540 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2541 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2542 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2543 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2544 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2545 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2546 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2547 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2548 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2549 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2550 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2551 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2553 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2555 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2556 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2557 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2559 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2560 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2561 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2562 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2563 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2564 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2565
2566 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2567 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2568 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2569 NUM_BANKS(ADDR_SURF_16_BANK));
2570 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2571 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2572 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2573 NUM_BANKS(ADDR_SURF_16_BANK));
2574 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2575 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2576 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2577 NUM_BANKS(ADDR_SURF_16_BANK));
2578 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2579 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2580 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2581 NUM_BANKS(ADDR_SURF_16_BANK));
2582 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2583 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2584 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2585 NUM_BANKS(ADDR_SURF_16_BANK));
2586 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2587 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2588 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2589 NUM_BANKS(ADDR_SURF_16_BANK));
2590 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2591 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2592 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2593 NUM_BANKS(ADDR_SURF_16_BANK));
2594 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2595 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2596 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2597 NUM_BANKS(ADDR_SURF_16_BANK));
2598 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2599 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2600 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2601 NUM_BANKS(ADDR_SURF_16_BANK));
2602 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2603 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2604 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2605 NUM_BANKS(ADDR_SURF_16_BANK));
2606 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2607 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2608 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2609 NUM_BANKS(ADDR_SURF_16_BANK));
2610 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2611 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2612 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2613 NUM_BANKS(ADDR_SURF_8_BANK));
2614 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2615 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2616 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2617 NUM_BANKS(ADDR_SURF_4_BANK));
2618 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2621 NUM_BANKS(ADDR_SURF_4_BANK));
2622
2623 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2624 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2625
2626 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2627 if (reg_offset != 7)
2628 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2629
2630 break;
2631 case CHIP_POLARIS11:
2632 case CHIP_POLARIS12:
2633 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2634 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2635 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2636 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2637 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2638 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2639 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2640 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2641 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2642 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2644 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2645 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2646 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2647 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2649 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2651 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2652 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2653 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2654 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2655 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2656 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2657 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2658 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2659 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2660 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2661 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2662 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2664 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2665 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2666 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2667 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2668 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2669 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2670 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2671 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2672 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2673 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2674 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2675 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2676 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2677 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2678 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2679 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2680 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2681 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2682 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2683 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2684 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2685 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2686 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2687 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2688 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2689 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2690 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2691 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2692 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2693 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2694 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2695 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2696 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2697 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2698 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2699 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2700 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2701 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2702 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2703 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2704 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2705 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2706 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2707 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2708 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2709 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2710 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2711 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2712 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2713 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2715 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2716 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2717 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2718 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2719 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2720 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2721 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2722 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2723 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2724 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2725 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2726 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2727 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2728 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2729 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2730 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2731 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2732 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2733 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2734 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2735 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2736 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2737 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2738 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2739 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2740 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2741 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2742 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2743 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2744 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2745 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2746 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2747 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2748 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2749 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2750 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2751 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2752 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2753 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2754 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2755
2756 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2757 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2758 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2759 NUM_BANKS(ADDR_SURF_16_BANK));
2760
2761 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2764 NUM_BANKS(ADDR_SURF_16_BANK));
2765
2766 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2767 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2768 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2769 NUM_BANKS(ADDR_SURF_16_BANK));
2770
2771 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2772 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2773 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2774 NUM_BANKS(ADDR_SURF_16_BANK));
2775
2776 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2777 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2778 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2779 NUM_BANKS(ADDR_SURF_16_BANK));
2780
2781 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2782 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2783 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2784 NUM_BANKS(ADDR_SURF_16_BANK));
2785
2786 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2787 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2788 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2789 NUM_BANKS(ADDR_SURF_16_BANK));
2790
2791 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2792 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2793 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2794 NUM_BANKS(ADDR_SURF_16_BANK));
2795
2796 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2797 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2798 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2799 NUM_BANKS(ADDR_SURF_16_BANK));
2800
2801 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2802 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2803 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2804 NUM_BANKS(ADDR_SURF_16_BANK));
2805
2806 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2809 NUM_BANKS(ADDR_SURF_16_BANK));
2810
2811 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2812 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2813 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2814 NUM_BANKS(ADDR_SURF_16_BANK));
2815
2816 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2817 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2818 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2819 NUM_BANKS(ADDR_SURF_8_BANK));
2820
2821 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2822 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2823 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2824 NUM_BANKS(ADDR_SURF_4_BANK));
2825
2826 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2827 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
2828
2829 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
2830 if (reg_offset != 7)
2831 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
2832
2833 break;
2834 case CHIP_POLARIS10:
2835 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2836 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2837 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2838 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2839 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2840 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2841 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2842 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2843 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2844 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2845 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2846 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2847 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2848 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2851 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2852 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2853 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2854 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2855 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2856 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2857 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2858 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2859 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2860 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2861 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2862 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2863 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2864 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2865 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
2866 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2867 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2868 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2869 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2870 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2871 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2872 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2873 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2874 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2875 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2876 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2877 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2878 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2879 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2880 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2881 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2882 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2883 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2885 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2886 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2887 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2888 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2889 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2890 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2891 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2892 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2893 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
2894 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2895 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2897 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2898 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2899 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2900 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2901 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2902 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2903 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2904 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2905 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2906 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2907 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2908 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2909 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
2910 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2911 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2912 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2913 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2914 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2915 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2917 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
2918 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2919 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2920 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2921 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2922 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2923 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2924 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2925 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
2926 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2927 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2928 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2929 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
2930 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2931 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2932 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2933 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
2934 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2935 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2936 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2937 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
2938 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2939 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
2940 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
2941 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2942 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2943 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2945 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2946 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2947 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2948 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2949 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2950 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2951 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2953 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2954 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2955 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2956 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
2957
2958 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2959 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2960 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2961 NUM_BANKS(ADDR_SURF_16_BANK));
2962
2963 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2964 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2965 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2966 NUM_BANKS(ADDR_SURF_16_BANK));
2967
2968 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2969 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2970 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2971 NUM_BANKS(ADDR_SURF_16_BANK));
2972
2973 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2974 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2975 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2976 NUM_BANKS(ADDR_SURF_16_BANK));
2977
2978 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2981 NUM_BANKS(ADDR_SURF_16_BANK));
2982
2983 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2984 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2985 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2986 NUM_BANKS(ADDR_SURF_16_BANK));
2987
2988 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2991 NUM_BANKS(ADDR_SURF_16_BANK));
2992
2993 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2994 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2995 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2996 NUM_BANKS(ADDR_SURF_16_BANK));
2997
2998 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2999 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3000 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3001 NUM_BANKS(ADDR_SURF_16_BANK));
3002
3003 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3004 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3005 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3006 NUM_BANKS(ADDR_SURF_16_BANK));
3007
3008 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3011 NUM_BANKS(ADDR_SURF_16_BANK));
3012
3013 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3014 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3015 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3016 NUM_BANKS(ADDR_SURF_8_BANK));
3017
3018 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3019 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3020 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3021 NUM_BANKS(ADDR_SURF_4_BANK));
3022
3023 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
3026 NUM_BANKS(ADDR_SURF_4_BANK));
3027
3028 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3029 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3030
3031 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3032 if (reg_offset != 7)
3033 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3034
3035 break;
3036 case CHIP_STONEY:
3037 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3038 PIPE_CONFIG(ADDR_SURF_P2) |
3039 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3040 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3041 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3042 PIPE_CONFIG(ADDR_SURF_P2) |
3043 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3044 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3045 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3046 PIPE_CONFIG(ADDR_SURF_P2) |
3047 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3048 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3049 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3050 PIPE_CONFIG(ADDR_SURF_P2) |
3051 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3052 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3053 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3054 PIPE_CONFIG(ADDR_SURF_P2) |
3055 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3056 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3057 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3058 PIPE_CONFIG(ADDR_SURF_P2) |
3059 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3060 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3061 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3062 PIPE_CONFIG(ADDR_SURF_P2) |
3063 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3064 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3065 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3066 PIPE_CONFIG(ADDR_SURF_P2));
3067 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3068 PIPE_CONFIG(ADDR_SURF_P2) |
3069 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3070 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3071 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3072 PIPE_CONFIG(ADDR_SURF_P2) |
3073 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3074 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3075 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3076 PIPE_CONFIG(ADDR_SURF_P2) |
3077 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3078 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3079 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3080 PIPE_CONFIG(ADDR_SURF_P2) |
3081 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3082 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3083 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3084 PIPE_CONFIG(ADDR_SURF_P2) |
3085 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3086 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3087 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3088 PIPE_CONFIG(ADDR_SURF_P2) |
3089 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3090 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3091 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3092 PIPE_CONFIG(ADDR_SURF_P2) |
3093 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3094 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3095 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3096 PIPE_CONFIG(ADDR_SURF_P2) |
3097 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3098 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3099 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3100 PIPE_CONFIG(ADDR_SURF_P2) |
3101 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3102 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3103 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3104 PIPE_CONFIG(ADDR_SURF_P2) |
3105 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3106 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3107 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3108 PIPE_CONFIG(ADDR_SURF_P2) |
3109 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3110 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3111 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3112 PIPE_CONFIG(ADDR_SURF_P2) |
3113 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3114 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3115 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3116 PIPE_CONFIG(ADDR_SURF_P2) |
3117 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3118 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3119 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3120 PIPE_CONFIG(ADDR_SURF_P2) |
3121 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3122 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3123 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3124 PIPE_CONFIG(ADDR_SURF_P2) |
3125 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3126 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3127 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3128 PIPE_CONFIG(ADDR_SURF_P2) |
3129 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3130 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3131 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3132 PIPE_CONFIG(ADDR_SURF_P2) |
3133 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3134 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3135 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3136 PIPE_CONFIG(ADDR_SURF_P2) |
3137 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3138 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3139
3140 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3141 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3142 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3143 NUM_BANKS(ADDR_SURF_8_BANK));
3144 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3145 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3146 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3147 NUM_BANKS(ADDR_SURF_8_BANK));
3148 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3149 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3150 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3151 NUM_BANKS(ADDR_SURF_8_BANK));
3152 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3153 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3154 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3155 NUM_BANKS(ADDR_SURF_8_BANK));
3156 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3157 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3158 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3159 NUM_BANKS(ADDR_SURF_8_BANK));
3160 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3161 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3162 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3163 NUM_BANKS(ADDR_SURF_8_BANK));
3164 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3165 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3166 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3167 NUM_BANKS(ADDR_SURF_8_BANK));
3168 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3169 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3170 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3171 NUM_BANKS(ADDR_SURF_16_BANK));
3172 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3173 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3174 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3175 NUM_BANKS(ADDR_SURF_16_BANK));
3176 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3177 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3178 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3179 NUM_BANKS(ADDR_SURF_16_BANK));
3180 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3181 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3182 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3183 NUM_BANKS(ADDR_SURF_16_BANK));
3184 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3185 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3186 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3187 NUM_BANKS(ADDR_SURF_16_BANK));
3188 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3189 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3190 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3191 NUM_BANKS(ADDR_SURF_16_BANK));
3192 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3193 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3194 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3195 NUM_BANKS(ADDR_SURF_8_BANK));
3196
3197 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3198 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3199 reg_offset != 23)
3200 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3201
3202 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3203 if (reg_offset != 7)
3204 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3205
3206 break;
3207 default:
3208 dev_warn(adev->dev,
3209 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n",
3210 adev->asic_type);
3211 fallthrough;
3212
3213 case CHIP_CARRIZO:
3214 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3215 PIPE_CONFIG(ADDR_SURF_P2) |
3216 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
3217 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3218 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3219 PIPE_CONFIG(ADDR_SURF_P2) |
3220 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
3221 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3222 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3223 PIPE_CONFIG(ADDR_SURF_P2) |
3224 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
3225 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3226 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3227 PIPE_CONFIG(ADDR_SURF_P2) |
3228 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
3229 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3230 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3231 PIPE_CONFIG(ADDR_SURF_P2) |
3232 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3233 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3234 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3235 PIPE_CONFIG(ADDR_SURF_P2) |
3236 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3238 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3239 PIPE_CONFIG(ADDR_SURF_P2) |
3240 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) |
3241 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
3242 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
3243 PIPE_CONFIG(ADDR_SURF_P2));
3244 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3245 PIPE_CONFIG(ADDR_SURF_P2) |
3246 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3247 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3248 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3249 PIPE_CONFIG(ADDR_SURF_P2) |
3250 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3251 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3252 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3253 PIPE_CONFIG(ADDR_SURF_P2) |
3254 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
3255 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3256 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3257 PIPE_CONFIG(ADDR_SURF_P2) |
3258 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3259 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3260 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3261 PIPE_CONFIG(ADDR_SURF_P2) |
3262 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3263 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3264 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) |
3265 PIPE_CONFIG(ADDR_SURF_P2) |
3266 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3267 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3268 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3269 PIPE_CONFIG(ADDR_SURF_P2) |
3270 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3271 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3272 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3273 PIPE_CONFIG(ADDR_SURF_P2) |
3274 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3276 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) |
3277 PIPE_CONFIG(ADDR_SURF_P2) |
3278 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3279 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3280 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3281 PIPE_CONFIG(ADDR_SURF_P2) |
3282 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3283 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3284 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) |
3285 PIPE_CONFIG(ADDR_SURF_P2) |
3286 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3288 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) |
3289 PIPE_CONFIG(ADDR_SURF_P2) |
3290 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3291 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3292 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) |
3293 PIPE_CONFIG(ADDR_SURF_P2) |
3294 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
3295 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3296 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) |
3297 PIPE_CONFIG(ADDR_SURF_P2) |
3298 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3299 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3300 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) |
3301 PIPE_CONFIG(ADDR_SURF_P2) |
3302 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) |
3303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1));
3304 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
3305 PIPE_CONFIG(ADDR_SURF_P2) |
3306 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3307 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3308 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
3309 PIPE_CONFIG(ADDR_SURF_P2) |
3310 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3311 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
3312 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
3313 PIPE_CONFIG(ADDR_SURF_P2) |
3314 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
3315 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8));
3316
3317 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3318 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3319 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3320 NUM_BANKS(ADDR_SURF_8_BANK));
3321 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3322 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3323 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3324 NUM_BANKS(ADDR_SURF_8_BANK));
3325 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3326 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3327 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3328 NUM_BANKS(ADDR_SURF_8_BANK));
3329 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3330 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3331 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3332 NUM_BANKS(ADDR_SURF_8_BANK));
3333 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3336 NUM_BANKS(ADDR_SURF_8_BANK));
3337 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3338 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3339 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3340 NUM_BANKS(ADDR_SURF_8_BANK));
3341 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3342 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3343 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3344 NUM_BANKS(ADDR_SURF_8_BANK));
3345 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3346 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
3347 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3348 NUM_BANKS(ADDR_SURF_16_BANK));
3349 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
3350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3352 NUM_BANKS(ADDR_SURF_16_BANK));
3353 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3354 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
3355 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3356 NUM_BANKS(ADDR_SURF_16_BANK));
3357 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
3358 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3359 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3360 NUM_BANKS(ADDR_SURF_16_BANK));
3361 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
3363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3364 NUM_BANKS(ADDR_SURF_16_BANK));
3365 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3366 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3367 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
3368 NUM_BANKS(ADDR_SURF_16_BANK));
3369 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
3370 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
3371 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3372 NUM_BANKS(ADDR_SURF_8_BANK));
3373
3374 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
3375 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 &&
3376 reg_offset != 23)
3377 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]);
3378
3379 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++)
3380 if (reg_offset != 7)
3381 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]);
3382
3383 break;
3384 }
3385 }
3386
gfx_v8_0_select_se_sh(struct amdgpu_device * adev,u32 se_num,u32 sh_num,u32 instance,int xcc_id)3387 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
3388 u32 se_num, u32 sh_num, u32 instance,
3389 int xcc_id)
3390 {
3391 u32 data;
3392
3393 if (instance == 0xffffffff)
3394 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
3395 else
3396 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
3397
3398 if (se_num == 0xffffffff)
3399 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
3400 else
3401 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
3402
3403 if (sh_num == 0xffffffff)
3404 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
3405 else
3406 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
3407
3408 WREG32(mmGRBM_GFX_INDEX, data);
3409 }
3410
gfx_v8_0_select_me_pipe_q(struct amdgpu_device * adev,u32 me,u32 pipe,u32 q,u32 vm,u32 xcc_id)3411 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev,
3412 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id)
3413 {
3414 vi_srbm_select(adev, me, pipe, q, vm);
3415 }
3416
gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device * adev)3417 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev)
3418 {
3419 u32 data, mask;
3420
3421 data = RREG32(mmCC_RB_BACKEND_DISABLE) |
3422 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3423
3424 data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE);
3425
3426 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
3427 adev->gfx.config.max_sh_per_se);
3428
3429 return (~data) & mask;
3430 }
3431
3432 static void
gfx_v8_0_raster_config(struct amdgpu_device * adev,u32 * rconf,u32 * rconf1)3433 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1)
3434 {
3435 switch (adev->asic_type) {
3436 case CHIP_FIJI:
3437 case CHIP_VEGAM:
3438 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) |
3439 RB_XSEL2(1) | PKR_MAP(2) |
3440 PKR_XSEL(1) | PKR_YSEL(1) |
3441 SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3);
3442 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) |
3443 SE_PAIR_YSEL(2);
3444 break;
3445 case CHIP_TONGA:
3446 case CHIP_POLARIS10:
3447 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3448 SE_XSEL(1) | SE_YSEL(1);
3449 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) |
3450 SE_PAIR_YSEL(2);
3451 break;
3452 case CHIP_TOPAZ:
3453 case CHIP_CARRIZO:
3454 *rconf |= RB_MAP_PKR0(2);
3455 *rconf1 |= 0x0;
3456 break;
3457 case CHIP_POLARIS11:
3458 case CHIP_POLARIS12:
3459 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) |
3460 SE_XSEL(1) | SE_YSEL(1);
3461 *rconf1 |= 0x0;
3462 break;
3463 case CHIP_STONEY:
3464 *rconf |= 0x0;
3465 *rconf1 |= 0x0;
3466 break;
3467 default:
3468 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type);
3469 break;
3470 }
3471 }
3472
3473 static void
gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device * adev,u32 raster_config,u32 raster_config_1,unsigned rb_mask,unsigned num_rb)3474 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev,
3475 u32 raster_config, u32 raster_config_1,
3476 unsigned rb_mask, unsigned num_rb)
3477 {
3478 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1);
3479 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1);
3480 unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2);
3481 unsigned rb_per_se = num_rb / num_se;
3482 unsigned se_mask[4];
3483 unsigned se;
3484
3485 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask;
3486 se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask;
3487 se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask;
3488 se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask;
3489
3490 WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4));
3491 WARN_ON(!(sh_per_se == 1 || sh_per_se == 2));
3492 WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2));
3493
3494 if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) ||
3495 (!se_mask[2] && !se_mask[3]))) {
3496 raster_config_1 &= ~SE_PAIR_MAP_MASK;
3497
3498 if (!se_mask[0] && !se_mask[1]) {
3499 raster_config_1 |=
3500 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3);
3501 } else {
3502 raster_config_1 |=
3503 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0);
3504 }
3505 }
3506
3507 for (se = 0; se < num_se; se++) {
3508 unsigned raster_config_se = raster_config;
3509 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se);
3510 unsigned pkr1_mask = pkr0_mask << rb_per_pkr;
3511 int idx = (se / 2) * 2;
3512
3513 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) {
3514 raster_config_se &= ~SE_MAP_MASK;
3515
3516 if (!se_mask[idx]) {
3517 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3);
3518 } else {
3519 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0);
3520 }
3521 }
3522
3523 pkr0_mask &= rb_mask;
3524 pkr1_mask &= rb_mask;
3525 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) {
3526 raster_config_se &= ~PKR_MAP_MASK;
3527
3528 if (!pkr0_mask) {
3529 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3);
3530 } else {
3531 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0);
3532 }
3533 }
3534
3535 if (rb_per_se >= 2) {
3536 unsigned rb0_mask = 1 << (se * rb_per_se);
3537 unsigned rb1_mask = rb0_mask << 1;
3538
3539 rb0_mask &= rb_mask;
3540 rb1_mask &= rb_mask;
3541 if (!rb0_mask || !rb1_mask) {
3542 raster_config_se &= ~RB_MAP_PKR0_MASK;
3543
3544 if (!rb0_mask) {
3545 raster_config_se |=
3546 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3);
3547 } else {
3548 raster_config_se |=
3549 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0);
3550 }
3551 }
3552
3553 if (rb_per_se > 2) {
3554 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr);
3555 rb1_mask = rb0_mask << 1;
3556 rb0_mask &= rb_mask;
3557 rb1_mask &= rb_mask;
3558 if (!rb0_mask || !rb1_mask) {
3559 raster_config_se &= ~RB_MAP_PKR1_MASK;
3560
3561 if (!rb0_mask) {
3562 raster_config_se |=
3563 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3);
3564 } else {
3565 raster_config_se |=
3566 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0);
3567 }
3568 }
3569 }
3570 }
3571
3572 /* GRBM_GFX_INDEX has a different offset on VI */
3573 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff, 0);
3574 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se);
3575 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3576 }
3577
3578 /* GRBM_GFX_INDEX has a different offset on VI */
3579 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3580 }
3581
gfx_v8_0_setup_rb(struct amdgpu_device * adev)3582 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
3583 {
3584 int i, j;
3585 u32 data;
3586 u32 raster_config = 0, raster_config_1 = 0;
3587 u32 active_rbs = 0;
3588 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
3589 adev->gfx.config.max_sh_per_se;
3590 unsigned num_rb_pipes;
3591
3592 mutex_lock(&adev->grbm_idx_mutex);
3593 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3594 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3595 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);
3596 data = gfx_v8_0_get_rb_active_bitmap(adev);
3597 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
3598 rb_bitmap_width_per_sh);
3599 }
3600 }
3601 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3602
3603 adev->gfx.config.backend_enable_mask = active_rbs;
3604 adev->gfx.config.num_rbs = hweight32(active_rbs);
3605
3606 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se *
3607 adev->gfx.config.max_shader_engines, 16);
3608
3609 gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1);
3610
3611 if (!adev->gfx.config.backend_enable_mask ||
3612 adev->gfx.config.num_rbs >= num_rb_pipes) {
3613 WREG32(mmPA_SC_RASTER_CONFIG, raster_config);
3614 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1);
3615 } else {
3616 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1,
3617 adev->gfx.config.backend_enable_mask,
3618 num_rb_pipes);
3619 }
3620
3621 /* cache the values for userspace */
3622 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3623 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3624 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);
3625 adev->gfx.config.rb_config[i][j].rb_backend_disable =
3626 RREG32(mmCC_RB_BACKEND_DISABLE);
3627 adev->gfx.config.rb_config[i][j].user_rb_backend_disable =
3628 RREG32(mmGC_USER_RB_BACKEND_DISABLE);
3629 adev->gfx.config.rb_config[i][j].raster_config =
3630 RREG32(mmPA_SC_RASTER_CONFIG);
3631 adev->gfx.config.rb_config[i][j].raster_config_1 =
3632 RREG32(mmPA_SC_RASTER_CONFIG_1);
3633 }
3634 }
3635 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3636 mutex_unlock(&adev->grbm_idx_mutex);
3637 }
3638
3639 #define DEFAULT_SH_MEM_BASES (0x6000)
3640 /**
3641 * gfx_v8_0_init_compute_vmid - gart enable
3642 *
3643 * @adev: amdgpu_device pointer
3644 *
3645 * Initialize compute vmid sh_mem registers
3646 *
3647 */
gfx_v8_0_init_compute_vmid(struct amdgpu_device * adev)3648 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev)
3649 {
3650 int i;
3651 uint32_t sh_mem_config;
3652 uint32_t sh_mem_bases;
3653
3654 /*
3655 * Configure apertures:
3656 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
3657 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
3658 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
3659 */
3660 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
3661
3662 sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 <<
3663 SH_MEM_CONFIG__ADDRESS_MODE__SHIFT |
3664 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
3665 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT |
3666 MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT |
3667 SH_MEM_CONFIG__PRIVATE_ATC_MASK;
3668
3669 mutex_lock(&adev->srbm_mutex);
3670 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
3671 vi_srbm_select(adev, 0, 0, 0, i);
3672 /* CP and shaders */
3673 WREG32(mmSH_MEM_CONFIG, sh_mem_config);
3674 WREG32(mmSH_MEM_APE1_BASE, 1);
3675 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3676 WREG32(mmSH_MEM_BASES, sh_mem_bases);
3677 }
3678 vi_srbm_select(adev, 0, 0, 0, 0);
3679 mutex_unlock(&adev->srbm_mutex);
3680
3681 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
3682 access. These should be enabled by FW for target VMIDs. */
3683 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) {
3684 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0);
3685 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0);
3686 WREG32(amdgpu_gds_reg_offset[i].gws, 0);
3687 WREG32(amdgpu_gds_reg_offset[i].oa, 0);
3688 }
3689 }
3690
gfx_v8_0_init_gds_vmid(struct amdgpu_device * adev)3691 static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev)
3692 {
3693 int vmid;
3694
3695 /*
3696 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
3697 * access. Compute VMIDs should be enabled by FW for target VMIDs,
3698 * the driver can enable them for graphics. VMID0 should maintain
3699 * access so that HWS firmware can save/restore entries.
3700 */
3701 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
3702 WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0);
3703 WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0);
3704 WREG32(amdgpu_gds_reg_offset[vmid].gws, 0);
3705 WREG32(amdgpu_gds_reg_offset[vmid].oa, 0);
3706 }
3707 }
3708
gfx_v8_0_config_init(struct amdgpu_device * adev)3709 static void gfx_v8_0_config_init(struct amdgpu_device *adev)
3710 {
3711 switch (adev->asic_type) {
3712 default:
3713 adev->gfx.config.double_offchip_lds_buf = 1;
3714 break;
3715 case CHIP_CARRIZO:
3716 case CHIP_STONEY:
3717 adev->gfx.config.double_offchip_lds_buf = 0;
3718 break;
3719 }
3720 }
3721
gfx_v8_0_constants_init(struct amdgpu_device * adev)3722 static void gfx_v8_0_constants_init(struct amdgpu_device *adev)
3723 {
3724 u32 tmp, sh_static_mem_cfg;
3725 int i;
3726
3727 WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF);
3728 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3729 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config);
3730 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config);
3731
3732 gfx_v8_0_tiling_mode_table_init(adev);
3733 gfx_v8_0_setup_rb(adev);
3734 gfx_v8_0_get_cu_info(adev);
3735 gfx_v8_0_config_init(adev);
3736
3737 /* XXX SH_MEM regs */
3738 /* where to put LDS, scratch, GPUVM in FSA64 space */
3739 sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG,
3740 SWIZZLE_ENABLE, 1);
3741 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3742 ELEMENT_SIZE, 1);
3743 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG,
3744 INDEX_STRIDE, 3);
3745 WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg);
3746
3747 mutex_lock(&adev->srbm_mutex);
3748 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) {
3749 vi_srbm_select(adev, 0, 0, 0, i);
3750 /* CP and shaders */
3751 if (i == 0) {
3752 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC);
3753 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3754 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3755 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3756 WREG32(mmSH_MEM_CONFIG, tmp);
3757 WREG32(mmSH_MEM_BASES, 0);
3758 } else {
3759 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC);
3760 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC);
3761 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE,
3762 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
3763 WREG32(mmSH_MEM_CONFIG, tmp);
3764 tmp = adev->gmc.shared_aperture_start >> 48;
3765 WREG32(mmSH_MEM_BASES, tmp);
3766 }
3767
3768 WREG32(mmSH_MEM_APE1_BASE, 1);
3769 WREG32(mmSH_MEM_APE1_LIMIT, 0);
3770 }
3771 vi_srbm_select(adev, 0, 0, 0, 0);
3772 mutex_unlock(&adev->srbm_mutex);
3773
3774 gfx_v8_0_init_compute_vmid(adev);
3775 gfx_v8_0_init_gds_vmid(adev);
3776
3777 mutex_lock(&adev->grbm_idx_mutex);
3778 /*
3779 * making sure that the following register writes will be broadcasted
3780 * to all the shaders
3781 */
3782 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3783
3784 WREG32(mmPA_SC_FIFO_SIZE,
3785 (adev->gfx.config.sc_prim_fifo_size_frontend <<
3786 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) |
3787 (adev->gfx.config.sc_prim_fifo_size_backend <<
3788 PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) |
3789 (adev->gfx.config.sc_hiz_tile_fifo_size <<
3790 PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) |
3791 (adev->gfx.config.sc_earlyz_tile_fifo_size <<
3792 PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT));
3793
3794 tmp = RREG32(mmSPI_ARB_PRIORITY);
3795 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2);
3796 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2);
3797 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2);
3798 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2);
3799 WREG32(mmSPI_ARB_PRIORITY, tmp);
3800
3801 mutex_unlock(&adev->grbm_idx_mutex);
3802
3803 }
3804
gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device * adev)3805 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
3806 {
3807 u32 i, j, k;
3808 u32 mask;
3809
3810 mutex_lock(&adev->grbm_idx_mutex);
3811 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
3812 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
3813 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);
3814 for (k = 0; k < adev->usec_timeout; k++) {
3815 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
3816 break;
3817 udelay(1);
3818 }
3819 if (k == adev->usec_timeout) {
3820 gfx_v8_0_select_se_sh(adev, 0xffffffff,
3821 0xffffffff, 0xffffffff, 0);
3822 mutex_unlock(&adev->grbm_idx_mutex);
3823 drm_info(adev_to_drm(adev), "Timeout wait for RLC serdes %u,%u\n",
3824 i, j);
3825 return;
3826 }
3827 }
3828 }
3829 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
3830 mutex_unlock(&adev->grbm_idx_mutex);
3831
3832 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
3833 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
3834 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
3835 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
3836 for (k = 0; k < adev->usec_timeout; k++) {
3837 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
3838 break;
3839 udelay(1);
3840 }
3841 }
3842
gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device * adev,bool enable)3843 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
3844 bool enable)
3845 {
3846 u32 tmp = RREG32(mmCP_INT_CNTL_RING0);
3847
3848 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
3849 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
3850 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
3851 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
3852
3853 WREG32(mmCP_INT_CNTL_RING0, tmp);
3854 }
3855
gfx_v8_0_init_csb(struct amdgpu_device * adev)3856 static void gfx_v8_0_init_csb(struct amdgpu_device *adev)
3857 {
3858 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
3859 /* csib */
3860 WREG32(mmRLC_CSIB_ADDR_HI,
3861 adev->gfx.rlc.clear_state_gpu_addr >> 32);
3862 WREG32(mmRLC_CSIB_ADDR_LO,
3863 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
3864 WREG32(mmRLC_CSIB_LENGTH,
3865 adev->gfx.rlc.clear_state_size);
3866 }
3867
gfx_v8_0_parse_ind_reg_list(int * register_list_format,int ind_offset,int list_size,int * unique_indices,int * indices_count,int max_indices,int * ind_start_offsets,int * offset_count,int max_offset)3868 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format,
3869 int ind_offset,
3870 int list_size,
3871 int *unique_indices,
3872 int *indices_count,
3873 int max_indices,
3874 int *ind_start_offsets,
3875 int *offset_count,
3876 int max_offset)
3877 {
3878 int indices;
3879 bool new_entry = true;
3880
3881 for (; ind_offset < list_size; ind_offset++) {
3882
3883 if (new_entry) {
3884 new_entry = false;
3885 ind_start_offsets[*offset_count] = ind_offset;
3886 *offset_count = *offset_count + 1;
3887 BUG_ON(*offset_count >= max_offset);
3888 }
3889
3890 if (register_list_format[ind_offset] == 0xFFFFFFFF) {
3891 new_entry = true;
3892 continue;
3893 }
3894
3895 ind_offset += 2;
3896
3897 /* look for the matching indice */
3898 for (indices = 0;
3899 indices < *indices_count;
3900 indices++) {
3901 if (unique_indices[indices] ==
3902 register_list_format[ind_offset])
3903 break;
3904 }
3905
3906 if (indices >= *indices_count) {
3907 unique_indices[*indices_count] =
3908 register_list_format[ind_offset];
3909 indices = *indices_count;
3910 *indices_count = *indices_count + 1;
3911 BUG_ON(*indices_count >= max_indices);
3912 }
3913
3914 register_list_format[ind_offset] = indices;
3915 }
3916 }
3917
gfx_v8_0_init_save_restore_list(struct amdgpu_device * adev)3918 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev)
3919 {
3920 int i, temp, data;
3921 int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0};
3922 int indices_count = 0;
3923 int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
3924 int offset_count = 0;
3925
3926 int list_size;
3927 unsigned int *register_list_format =
3928 kmemdup(adev->gfx.rlc.register_list_format,
3929 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
3930 if (!register_list_format)
3931 return -ENOMEM;
3932
3933 gfx_v8_0_parse_ind_reg_list(register_list_format,
3934 RLC_FormatDirectRegListLength,
3935 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
3936 unique_indices,
3937 &indices_count,
3938 ARRAY_SIZE(unique_indices),
3939 indirect_start_offsets,
3940 &offset_count,
3941 ARRAY_SIZE(indirect_start_offsets));
3942
3943 /* save and restore list */
3944 WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1);
3945
3946 WREG32(mmRLC_SRM_ARAM_ADDR, 0);
3947 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
3948 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]);
3949
3950 /* indirect list */
3951 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start);
3952 for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++)
3953 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]);
3954
3955 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
3956 list_size = list_size >> 1;
3957 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size);
3958 WREG32(mmRLC_GPM_SCRATCH_DATA, list_size);
3959
3960 /* starting offsets starts */
3961 WREG32(mmRLC_GPM_SCRATCH_ADDR,
3962 adev->gfx.rlc.starting_offsets_start);
3963 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
3964 WREG32(mmRLC_GPM_SCRATCH_DATA,
3965 indirect_start_offsets[i]);
3966
3967 /* unique indices */
3968 temp = mmRLC_SRM_INDEX_CNTL_ADDR_0;
3969 data = mmRLC_SRM_INDEX_CNTL_DATA_0;
3970 for (i = 0; i < ARRAY_SIZE(unique_indices); i++) {
3971 if (unique_indices[i] != 0) {
3972 WREG32(temp + i, unique_indices[i] & 0x3FFFF);
3973 WREG32(data + i, unique_indices[i] >> 20);
3974 }
3975 }
3976 kfree(register_list_format);
3977
3978 return 0;
3979 }
3980
gfx_v8_0_enable_save_restore_machine(struct amdgpu_device * adev)3981 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
3982 {
3983 WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1);
3984 }
3985
gfx_v8_0_init_power_gating(struct amdgpu_device * adev)3986 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
3987 {
3988 uint32_t data;
3989
3990 WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60);
3991
3992 data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10);
3993 data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10);
3994 data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10);
3995 data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10);
3996 WREG32(mmRLC_PG_DELAY, data);
3997
3998 WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3);
3999 WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0);
4000
4001 }
4002
cz_enable_sck_slow_down_on_power_up(struct amdgpu_device * adev,bool enable)4003 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
4004 bool enable)
4005 {
4006 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0);
4007 }
4008
cz_enable_sck_slow_down_on_power_down(struct amdgpu_device * adev,bool enable)4009 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
4010 bool enable)
4011 {
4012 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0);
4013 }
4014
cz_enable_cp_power_gating(struct amdgpu_device * adev,bool enable)4015 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
4016 {
4017 WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1);
4018 }
4019
gfx_v8_0_init_pg(struct amdgpu_device * adev)4020 static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
4021 {
4022 if ((adev->asic_type == CHIP_CARRIZO) ||
4023 (adev->asic_type == CHIP_STONEY)) {
4024 gfx_v8_0_init_csb(adev);
4025 gfx_v8_0_init_save_restore_list(adev);
4026 gfx_v8_0_enable_save_restore_machine(adev);
4027 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
4028 gfx_v8_0_init_power_gating(adev);
4029 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
4030 } else if ((adev->asic_type == CHIP_POLARIS11) ||
4031 (adev->asic_type == CHIP_POLARIS12) ||
4032 (adev->asic_type == CHIP_VEGAM)) {
4033 gfx_v8_0_init_csb(adev);
4034 gfx_v8_0_init_save_restore_list(adev);
4035 gfx_v8_0_enable_save_restore_machine(adev);
4036 gfx_v8_0_init_power_gating(adev);
4037 }
4038
4039 }
4040
gfx_v8_0_rlc_stop(struct amdgpu_device * adev)4041 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev)
4042 {
4043 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0);
4044
4045 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4046 gfx_v8_0_wait_for_rlc_serdes(adev);
4047 }
4048
gfx_v8_0_rlc_reset(struct amdgpu_device * adev)4049 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev)
4050 {
4051 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4052 udelay(50);
4053
4054 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
4055 udelay(50);
4056 }
4057
gfx_v8_0_rlc_start(struct amdgpu_device * adev)4058 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev)
4059 {
4060 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1);
4061
4062 /* carrizo do enable cp interrupt after cp inited */
4063 if (!(adev->flags & AMD_IS_APU))
4064 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4065
4066 udelay(50);
4067 }
4068
gfx_v8_0_rlc_resume(struct amdgpu_device * adev)4069 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev)
4070 {
4071 if (amdgpu_sriov_vf(adev)) {
4072 gfx_v8_0_init_csb(adev);
4073 return 0;
4074 }
4075
4076 adev->gfx.rlc.funcs->stop(adev);
4077 adev->gfx.rlc.funcs->reset(adev);
4078 gfx_v8_0_init_pg(adev);
4079 adev->gfx.rlc.funcs->start(adev);
4080
4081 return 0;
4082 }
4083
gfx_v8_0_cp_gfx_enable(struct amdgpu_device * adev,bool enable)4084 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
4085 {
4086 u32 tmp = RREG32(mmCP_ME_CNTL);
4087
4088 if (enable) {
4089 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0);
4090 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0);
4091 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0);
4092 } else {
4093 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1);
4094 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1);
4095 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1);
4096 }
4097 WREG32(mmCP_ME_CNTL, tmp);
4098 udelay(50);
4099 }
4100
gfx_v8_0_get_csb_size(struct amdgpu_device * adev)4101 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev)
4102 {
4103 u32 count = 0;
4104 const struct cs_section_def *sect = NULL;
4105 const struct cs_extent_def *ext = NULL;
4106
4107 /* begin clear state */
4108 count += 2;
4109 /* context control state */
4110 count += 3;
4111
4112 for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4113 for (ext = sect->section; ext->extent != NULL; ++ext) {
4114 if (sect->id == SECT_CONTEXT)
4115 count += 2 + ext->reg_count;
4116 else
4117 return 0;
4118 }
4119 }
4120 /* pa_sc_raster_config/pa_sc_raster_config1 */
4121 count += 4;
4122 /* end clear state */
4123 count += 2;
4124 /* clear state */
4125 count += 2;
4126
4127 return count;
4128 }
4129
gfx_v8_0_cp_gfx_start(struct amdgpu_device * adev)4130 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev)
4131 {
4132 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
4133 const struct cs_section_def *sect = NULL;
4134 const struct cs_extent_def *ext = NULL;
4135 int r, i;
4136
4137 /* init the CP */
4138 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
4139 WREG32(mmCP_ENDIAN_SWAP, 0);
4140 WREG32(mmCP_DEVICE_ID, 1);
4141
4142 gfx_v8_0_cp_gfx_enable(adev, true);
4143
4144 r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4);
4145 if (r) {
4146 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r);
4147 return r;
4148 }
4149
4150 /* clear state buffer */
4151 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4152 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
4153
4154 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
4155 amdgpu_ring_write(ring, 0x80000000);
4156 amdgpu_ring_write(ring, 0x80000000);
4157
4158 for (sect = vi_cs_data; sect->section != NULL; ++sect) {
4159 for (ext = sect->section; ext->extent != NULL; ++ext) {
4160 if (sect->id == SECT_CONTEXT) {
4161 amdgpu_ring_write(ring,
4162 PACKET3(PACKET3_SET_CONTEXT_REG,
4163 ext->reg_count));
4164 amdgpu_ring_write(ring,
4165 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
4166 for (i = 0; i < ext->reg_count; i++)
4167 amdgpu_ring_write(ring, ext->extent[i]);
4168 }
4169 }
4170 }
4171
4172 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
4173 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
4174 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config);
4175 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1);
4176
4177 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
4178 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
4179
4180 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
4181 amdgpu_ring_write(ring, 0);
4182
4183 /* init the CE partitions */
4184 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
4185 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
4186 amdgpu_ring_write(ring, 0x8000);
4187 amdgpu_ring_write(ring, 0x8000);
4188
4189 amdgpu_ring_commit(ring);
4190
4191 return 0;
4192 }
gfx_v8_0_set_cpg_door_bell(struct amdgpu_device * adev,struct amdgpu_ring * ring)4193 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring)
4194 {
4195 u32 tmp;
4196 /* no gfx doorbells on iceland */
4197 if (adev->asic_type == CHIP_TOPAZ)
4198 return;
4199
4200 tmp = RREG32(mmCP_RB_DOORBELL_CONTROL);
4201
4202 if (ring->use_doorbell) {
4203 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4204 DOORBELL_OFFSET, ring->doorbell_index);
4205 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4206 DOORBELL_HIT, 0);
4207 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
4208 DOORBELL_EN, 1);
4209 } else {
4210 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
4211 }
4212
4213 WREG32(mmCP_RB_DOORBELL_CONTROL, tmp);
4214
4215 if (adev->flags & AMD_IS_APU)
4216 return;
4217
4218 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
4219 DOORBELL_RANGE_LOWER,
4220 adev->doorbell_index.gfx_ring0);
4221 WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
4222
4223 WREG32(mmCP_RB_DOORBELL_RANGE_UPPER,
4224 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
4225 }
4226
gfx_v8_0_cp_gfx_resume(struct amdgpu_device * adev)4227 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev)
4228 {
4229 struct amdgpu_ring *ring;
4230 u32 tmp;
4231 u32 rb_bufsz;
4232 u64 rb_addr, rptr_addr, wptr_gpu_addr;
4233
4234 /* Set the write pointer delay */
4235 WREG32(mmCP_RB_WPTR_DELAY, 0);
4236
4237 /* set the RB to use vmid 0 */
4238 WREG32(mmCP_RB_VMID, 0);
4239
4240 /* Set ring buffer size */
4241 ring = &adev->gfx.gfx_ring[0];
4242 rb_bufsz = order_base_2(ring->ring_size / 8);
4243 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
4244 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
4245 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3);
4246 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1);
4247 #ifdef __BIG_ENDIAN
4248 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
4249 #endif
4250 WREG32(mmCP_RB0_CNTL, tmp);
4251
4252 /* Initialize the ring buffer's read and write pointers */
4253 WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK);
4254 ring->wptr = 0;
4255 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
4256
4257 /* set the wb address whether it's enabled or not */
4258 rptr_addr = ring->rptr_gpu_addr;
4259 WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
4260 WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF);
4261
4262 wptr_gpu_addr = ring->wptr_gpu_addr;
4263 WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
4264 WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
4265 mdelay(1);
4266 WREG32(mmCP_RB0_CNTL, tmp);
4267
4268 rb_addr = ring->gpu_addr >> 8;
4269 WREG32(mmCP_RB0_BASE, rb_addr);
4270 WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
4271
4272 gfx_v8_0_set_cpg_door_bell(adev, ring);
4273 /* start the ring */
4274 amdgpu_ring_clear_ring(ring);
4275 gfx_v8_0_cp_gfx_start(adev);
4276
4277 return 0;
4278 }
4279
gfx_v8_0_cp_compute_enable(struct amdgpu_device * adev,bool enable)4280 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
4281 {
4282 if (enable) {
4283 WREG32(mmCP_MEC_CNTL, 0);
4284 } else {
4285 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
4286 adev->gfx.kiq[0].ring.sched.ready = false;
4287 }
4288 udelay(50);
4289 }
4290
4291 /* KIQ functions */
gfx_v8_0_kiq_setting(struct amdgpu_ring * ring)4292 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring)
4293 {
4294 uint32_t tmp;
4295 struct amdgpu_device *adev = ring->adev;
4296
4297 /* tell RLC which is KIQ queue */
4298 tmp = RREG32(mmRLC_CP_SCHEDULERS);
4299 tmp &= 0xffffff00;
4300 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
4301 WREG32(mmRLC_CP_SCHEDULERS, tmp | 0x80);
4302 }
4303
gfx_v8_0_kiq_kcq_enable(struct amdgpu_device * adev)4304 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev)
4305 {
4306 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
4307 uint64_t queue_mask = 0;
4308 int r, i;
4309
4310 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) {
4311 if (!test_bit(i, adev->gfx.mec_bitmap[0].queue_bitmap))
4312 continue;
4313
4314 /* This situation may be hit in the future if a new HW
4315 * generation exposes more than 64 queues. If so, the
4316 * definition of queue_mask needs updating */
4317 if (WARN_ON(i >= (sizeof(queue_mask)*8))) {
4318 DRM_ERROR("Invalid KCQ enabled: %d\n", i);
4319 break;
4320 }
4321
4322 queue_mask |= (1ull << i);
4323 }
4324
4325 r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8);
4326 if (r) {
4327 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4328 return r;
4329 }
4330 /* set resources */
4331 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
4332 amdgpu_ring_write(kiq_ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */
4333 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */
4334 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */
4335 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
4336 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
4337 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
4338 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
4339 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4340 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4341 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
4342 uint64_t wptr_addr = ring->wptr_gpu_addr;
4343
4344 /* map queues */
4345 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
4346 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
4347 amdgpu_ring_write(kiq_ring,
4348 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
4349 amdgpu_ring_write(kiq_ring,
4350 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) |
4351 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
4352 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
4353 PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */
4354 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
4355 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
4356 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
4357 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
4358 }
4359
4360 amdgpu_ring_commit(kiq_ring);
4361
4362 return 0;
4363 }
4364
gfx_v8_0_deactivate_hqd(struct amdgpu_device * adev,u32 req)4365 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req)
4366 {
4367 int i, r = 0;
4368
4369 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) {
4370 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req);
4371 for (i = 0; i < adev->usec_timeout; i++) {
4372 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK))
4373 break;
4374 udelay(1);
4375 }
4376 if (i == adev->usec_timeout)
4377 r = -ETIMEDOUT;
4378 }
4379 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0);
4380 WREG32(mmCP_HQD_PQ_RPTR, 0);
4381 WREG32(mmCP_HQD_PQ_WPTR, 0);
4382
4383 return r;
4384 }
4385
gfx_v8_0_mqd_set_priority(struct amdgpu_ring * ring,struct vi_mqd * mqd)4386 static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd)
4387 {
4388 struct amdgpu_device *adev = ring->adev;
4389
4390 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
4391 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) {
4392 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
4393 mqd->cp_hqd_queue_priority =
4394 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
4395 }
4396 }
4397 }
4398
gfx_v8_0_mqd_init(struct amdgpu_ring * ring)4399 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
4400 {
4401 struct amdgpu_device *adev = ring->adev;
4402 struct vi_mqd *mqd = ring->mqd_ptr;
4403 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
4404 uint32_t tmp;
4405
4406 mqd->header = 0xC0310800;
4407 mqd->compute_pipelinestat_enable = 0x00000001;
4408 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
4409 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
4410 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
4411 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
4412 mqd->compute_misc_reserved = 0x00000003;
4413 mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr
4414 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4415 mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr
4416 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask));
4417 eop_base_addr = ring->eop_gpu_addr >> 8;
4418 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
4419 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
4420
4421 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4422 tmp = RREG32(mmCP_HQD_EOP_CONTROL);
4423 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
4424 (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1));
4425
4426 mqd->cp_hqd_eop_control = tmp;
4427
4428 /* enable doorbell? */
4429 tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL),
4430 CP_HQD_PQ_DOORBELL_CONTROL,
4431 DOORBELL_EN,
4432 ring->use_doorbell ? 1 : 0);
4433
4434 mqd->cp_hqd_pq_doorbell_control = tmp;
4435
4436 /* set the pointer to the MQD */
4437 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
4438 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
4439
4440 /* set MQD vmid to 0 */
4441 tmp = RREG32(mmCP_MQD_CONTROL);
4442 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
4443 mqd->cp_mqd_control = tmp;
4444
4445 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4446 hqd_gpu_addr = ring->gpu_addr >> 8;
4447 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
4448 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4449
4450 /* set up the HQD, this is similar to CP_RB0_CNTL */
4451 tmp = RREG32(mmCP_HQD_PQ_CONTROL);
4452 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
4453 (order_base_2(ring->ring_size / 4) - 1));
4454 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
4455 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1));
4456 #ifdef __BIG_ENDIAN
4457 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
4458 #endif
4459 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
4460 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
4461 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
4462 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
4463 mqd->cp_hqd_pq_control = tmp;
4464
4465 /* set the wb address whether it's enabled or not */
4466 wb_gpu_addr = ring->rptr_gpu_addr;
4467 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
4468 mqd->cp_hqd_pq_rptr_report_addr_hi =
4469 upper_32_bits(wb_gpu_addr) & 0xffff;
4470
4471 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
4472 wb_gpu_addr = ring->wptr_gpu_addr;
4473 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
4474 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4475
4476 tmp = 0;
4477 /* enable the doorbell if requested */
4478 if (ring->use_doorbell) {
4479 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL);
4480 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4481 DOORBELL_OFFSET, ring->doorbell_index);
4482
4483 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4484 DOORBELL_EN, 1);
4485 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4486 DOORBELL_SOURCE, 0);
4487 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
4488 DOORBELL_HIT, 0);
4489 }
4490
4491 mqd->cp_hqd_pq_doorbell_control = tmp;
4492
4493 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4494 ring->wptr = 0;
4495 mqd->cp_hqd_pq_wptr = ring->wptr;
4496 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
4497
4498 /* set the vmid for the queue */
4499 mqd->cp_hqd_vmid = 0;
4500
4501 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE);
4502 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
4503 mqd->cp_hqd_persistent_state = tmp;
4504
4505 /* set MTYPE */
4506 tmp = RREG32(mmCP_HQD_IB_CONTROL);
4507 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
4508 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3);
4509 mqd->cp_hqd_ib_control = tmp;
4510
4511 tmp = RREG32(mmCP_HQD_IQ_TIMER);
4512 tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3);
4513 mqd->cp_hqd_iq_timer = tmp;
4514
4515 tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL);
4516 tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3);
4517 mqd->cp_hqd_ctx_save_control = tmp;
4518
4519 /* defaults */
4520 mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR);
4521 mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR);
4522 mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO);
4523 mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI);
4524 mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET);
4525 mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE);
4526 mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET);
4527 mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE);
4528 mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS);
4529 mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR);
4530 mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM);
4531 mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES);
4532
4533 /* set static priority for a queue/ring */
4534 gfx_v8_0_mqd_set_priority(ring, mqd);
4535 tmp = RREG32(mmCP_HQD_QUANTUM);
4536 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
4537 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
4538 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10);
4539 mqd->cp_hqd_quantum = tmp;
4540
4541 /* map_queues packet doesn't need activate the queue,
4542 * so only kiq need set this field.
4543 */
4544 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
4545 mqd->cp_hqd_active = 1;
4546
4547 return 0;
4548 }
4549
gfx_v8_0_mqd_commit(struct amdgpu_device * adev,struct vi_mqd * mqd)4550 static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev,
4551 struct vi_mqd *mqd)
4552 {
4553 uint32_t mqd_reg;
4554 uint32_t *mqd_data;
4555
4556 /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */
4557 mqd_data = &mqd->cp_mqd_base_addr_lo;
4558
4559 /* disable wptr polling */
4560 WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0);
4561
4562 /* program all HQD registers */
4563 for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++)
4564 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4565
4566 /* Tonga errata: EOP RPTR/WPTR should be left unmodified.
4567 * This is safe since EOP RPTR==WPTR for any inactive HQD
4568 * on ASICs that do not support context-save.
4569 * EOP writes/reads can start anywhere in the ring.
4570 */
4571 if (adev->asic_type != CHIP_TONGA) {
4572 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr);
4573 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr);
4574 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem);
4575 }
4576
4577 for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++)
4578 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4579
4580 /* activate the HQD */
4581 for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++)
4582 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]);
4583
4584 return 0;
4585 }
4586
gfx_v8_0_kiq_init_queue(struct amdgpu_ring * ring)4587 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring)
4588 {
4589 struct amdgpu_device *adev = ring->adev;
4590 struct vi_mqd *mqd = ring->mqd_ptr;
4591
4592 gfx_v8_0_kiq_setting(ring);
4593
4594 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */
4595 /* reset MQD to a clean status */
4596 if (adev->gfx.kiq[0].mqd_backup)
4597 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation));
4598
4599 /* reset ring buffer */
4600 ring->wptr = 0;
4601 amdgpu_ring_clear_ring(ring);
4602 mutex_lock(&adev->srbm_mutex);
4603 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4604 gfx_v8_0_mqd_commit(adev, mqd);
4605 vi_srbm_select(adev, 0, 0, 0, 0);
4606 mutex_unlock(&adev->srbm_mutex);
4607 } else {
4608 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4609 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4610 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4611 if (amdgpu_sriov_vf(adev) && adev->in_suspend)
4612 amdgpu_ring_clear_ring(ring);
4613 mutex_lock(&adev->srbm_mutex);
4614 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4615 gfx_v8_0_mqd_init(ring);
4616 gfx_v8_0_mqd_commit(adev, mqd);
4617 vi_srbm_select(adev, 0, 0, 0, 0);
4618 mutex_unlock(&adev->srbm_mutex);
4619
4620 if (adev->gfx.kiq[0].mqd_backup)
4621 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation));
4622 }
4623
4624 return 0;
4625 }
4626
gfx_v8_0_kcq_init_queue(struct amdgpu_ring * ring)4627 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring)
4628 {
4629 struct amdgpu_device *adev = ring->adev;
4630 struct vi_mqd *mqd = ring->mqd_ptr;
4631 int mqd_idx = ring - &adev->gfx.compute_ring[0];
4632
4633 if (!amdgpu_in_reset(adev) && !adev->in_suspend) {
4634 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation));
4635 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
4636 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
4637 mutex_lock(&adev->srbm_mutex);
4638 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4639 gfx_v8_0_mqd_init(ring);
4640 vi_srbm_select(adev, 0, 0, 0, 0);
4641 mutex_unlock(&adev->srbm_mutex);
4642
4643 if (adev->gfx.mec.mqd_backup[mqd_idx])
4644 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation));
4645 } else {
4646 /* restore MQD to a clean status */
4647 if (adev->gfx.mec.mqd_backup[mqd_idx])
4648 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation));
4649 /* reset ring buffer */
4650 ring->wptr = 0;
4651 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0);
4652 amdgpu_ring_clear_ring(ring);
4653 }
4654 return 0;
4655 }
4656
gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device * adev)4657 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev)
4658 {
4659 if (adev->asic_type > CHIP_TONGA) {
4660 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2);
4661 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2);
4662 }
4663 /* enable doorbells */
4664 WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1);
4665 }
4666
gfx_v8_0_kiq_resume(struct amdgpu_device * adev)4667 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev)
4668 {
4669 gfx_v8_0_kiq_init_queue(&adev->gfx.kiq[0].ring);
4670 return 0;
4671 }
4672
gfx_v8_0_kcq_resume(struct amdgpu_device * adev)4673 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev)
4674 {
4675 int i, r;
4676
4677 gfx_v8_0_cp_compute_enable(adev, true);
4678
4679 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4680 r = gfx_v8_0_kcq_init_queue(&adev->gfx.compute_ring[i]);
4681 if (r)
4682 return r;
4683 }
4684
4685 gfx_v8_0_set_mec_doorbell_range(adev);
4686
4687 return gfx_v8_0_kiq_kcq_enable(adev);
4688 }
4689
gfx_v8_0_cp_test_all_rings(struct amdgpu_device * adev)4690 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev)
4691 {
4692 int r, i;
4693 struct amdgpu_ring *ring;
4694
4695 /* collect all the ring_tests here, gfx, kiq, compute */
4696 ring = &adev->gfx.gfx_ring[0];
4697 r = amdgpu_ring_test_helper(ring);
4698 if (r)
4699 return r;
4700
4701 ring = &adev->gfx.kiq[0].ring;
4702 r = amdgpu_ring_test_helper(ring);
4703 if (r)
4704 return r;
4705
4706 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4707 ring = &adev->gfx.compute_ring[i];
4708 amdgpu_ring_test_helper(ring);
4709 }
4710
4711 return 0;
4712 }
4713
gfx_v8_0_cp_resume(struct amdgpu_device * adev)4714 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev)
4715 {
4716 int r;
4717
4718 if (!(adev->flags & AMD_IS_APU))
4719 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
4720
4721 r = gfx_v8_0_kiq_resume(adev);
4722 if (r)
4723 return r;
4724
4725 r = gfx_v8_0_cp_gfx_resume(adev);
4726 if (r)
4727 return r;
4728
4729 r = gfx_v8_0_kcq_resume(adev);
4730 if (r)
4731 return r;
4732
4733 r = gfx_v8_0_cp_test_all_rings(adev);
4734 if (r)
4735 return r;
4736
4737 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
4738
4739 return 0;
4740 }
4741
gfx_v8_0_cp_enable(struct amdgpu_device * adev,bool enable)4742 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable)
4743 {
4744 gfx_v8_0_cp_gfx_enable(adev, enable);
4745 gfx_v8_0_cp_compute_enable(adev, enable);
4746 }
4747
gfx_v8_0_hw_init(struct amdgpu_ip_block * ip_block)4748 static int gfx_v8_0_hw_init(struct amdgpu_ip_block *ip_block)
4749 {
4750 int r;
4751 struct amdgpu_device *adev = ip_block->adev;
4752
4753 gfx_v8_0_init_golden_registers(adev);
4754 gfx_v8_0_constants_init(adev);
4755
4756 r = adev->gfx.rlc.funcs->resume(adev);
4757 if (r)
4758 return r;
4759
4760 r = gfx_v8_0_cp_resume(adev);
4761
4762 return r;
4763 }
4764
gfx_v8_0_kcq_disable(struct amdgpu_device * adev)4765 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev)
4766 {
4767 int r, i;
4768 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring;
4769
4770 r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings);
4771 if (r)
4772 DRM_ERROR("Failed to lock KIQ (%d).\n", r);
4773
4774 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4775 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4776
4777 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
4778 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
4779 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */
4780 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
4781 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) |
4782 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
4783 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
4784 amdgpu_ring_write(kiq_ring, 0);
4785 amdgpu_ring_write(kiq_ring, 0);
4786 amdgpu_ring_write(kiq_ring, 0);
4787 }
4788 /* Submit unmap queue packet */
4789 amdgpu_ring_commit(kiq_ring);
4790 /*
4791 * Ring test will do a basic scratch register change check. Just run
4792 * this to ensure that unmap queues that is submitted before got
4793 * processed successfully before returning.
4794 */
4795 r = amdgpu_ring_test_helper(kiq_ring);
4796 if (r)
4797 DRM_ERROR("KCQ disable failed\n");
4798
4799 return r;
4800 }
4801
gfx_v8_0_is_idle(struct amdgpu_ip_block * ip_block)4802 static bool gfx_v8_0_is_idle(struct amdgpu_ip_block *ip_block)
4803 {
4804 struct amdgpu_device *adev = ip_block->adev;
4805
4806 if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE)
4807 || RREG32(mmGRBM_STATUS2) != 0x8)
4808 return false;
4809 else
4810 return true;
4811 }
4812
gfx_v8_0_rlc_is_idle(void * handle)4813 static bool gfx_v8_0_rlc_is_idle(void *handle)
4814 {
4815 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4816
4817 if (RREG32(mmGRBM_STATUS2) != 0x8)
4818 return false;
4819 else
4820 return true;
4821 }
4822
gfx_v8_0_wait_for_rlc_idle(void * handle)4823 static int gfx_v8_0_wait_for_rlc_idle(void *handle)
4824 {
4825 unsigned int i;
4826 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4827
4828 for (i = 0; i < adev->usec_timeout; i++) {
4829 if (gfx_v8_0_rlc_is_idle(handle))
4830 return 0;
4831
4832 udelay(1);
4833 }
4834 return -ETIMEDOUT;
4835 }
4836
gfx_v8_0_wait_for_idle(struct amdgpu_ip_block * ip_block)4837 static int gfx_v8_0_wait_for_idle(struct amdgpu_ip_block *ip_block)
4838 {
4839 unsigned int i;
4840 struct amdgpu_device *adev = ip_block->adev;
4841
4842 for (i = 0; i < adev->usec_timeout; i++) {
4843 if (gfx_v8_0_is_idle(ip_block))
4844 return 0;
4845
4846 udelay(1);
4847 }
4848 return -ETIMEDOUT;
4849 }
4850
gfx_v8_0_hw_fini(struct amdgpu_ip_block * ip_block)4851 static int gfx_v8_0_hw_fini(struct amdgpu_ip_block *ip_block)
4852 {
4853 struct amdgpu_device *adev = ip_block->adev;
4854
4855 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
4856 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
4857
4858 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
4859
4860 amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0);
4861
4862 /* disable KCQ to avoid CPC touch memory not valid anymore */
4863 gfx_v8_0_kcq_disable(adev);
4864
4865 if (amdgpu_sriov_vf(adev)) {
4866 pr_debug("For SRIOV client, shouldn't do anything.\n");
4867 return 0;
4868 }
4869
4870 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
4871 if (!gfx_v8_0_wait_for_idle(ip_block))
4872 gfx_v8_0_cp_enable(adev, false);
4873 else
4874 pr_err("cp is busy, skip halt cp\n");
4875 if (!gfx_v8_0_wait_for_rlc_idle(adev))
4876 adev->gfx.rlc.funcs->stop(adev);
4877 else
4878 pr_err("rlc is busy, skip halt rlc\n");
4879 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
4880
4881 return 0;
4882 }
4883
gfx_v8_0_suspend(struct amdgpu_ip_block * ip_block)4884 static int gfx_v8_0_suspend(struct amdgpu_ip_block *ip_block)
4885 {
4886 return gfx_v8_0_hw_fini(ip_block);
4887 }
4888
gfx_v8_0_resume(struct amdgpu_ip_block * ip_block)4889 static int gfx_v8_0_resume(struct amdgpu_ip_block *ip_block)
4890 {
4891 return gfx_v8_0_hw_init(ip_block);
4892 }
4893
gfx_v8_0_check_soft_reset(struct amdgpu_ip_block * ip_block)4894 static bool gfx_v8_0_check_soft_reset(struct amdgpu_ip_block *ip_block)
4895 {
4896 struct amdgpu_device *adev = ip_block->adev;
4897 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4898 u32 tmp;
4899
4900 /* GRBM_STATUS */
4901 tmp = RREG32(mmGRBM_STATUS);
4902 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4903 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4904 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4905 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4906 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4907 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK |
4908 GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4909 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4910 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4911 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4912 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4913 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4914 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
4915 }
4916
4917 /* GRBM_STATUS2 */
4918 tmp = RREG32(mmGRBM_STATUS2);
4919 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4920 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4921 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4922
4923 if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) ||
4924 REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) ||
4925 REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) {
4926 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4927 SOFT_RESET_CPF, 1);
4928 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4929 SOFT_RESET_CPC, 1);
4930 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET,
4931 SOFT_RESET_CPG, 1);
4932 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET,
4933 SOFT_RESET_GRBM, 1);
4934 }
4935
4936 /* SRBM_STATUS */
4937 tmp = RREG32(mmSRBM_STATUS);
4938 if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING))
4939 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4940 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1);
4941 if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY))
4942 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
4943 SRBM_SOFT_RESET, SOFT_RESET_SEM, 1);
4944
4945 if (grbm_soft_reset || srbm_soft_reset) {
4946 adev->gfx.grbm_soft_reset = grbm_soft_reset;
4947 adev->gfx.srbm_soft_reset = srbm_soft_reset;
4948 return true;
4949 } else {
4950 adev->gfx.grbm_soft_reset = 0;
4951 adev->gfx.srbm_soft_reset = 0;
4952 return false;
4953 }
4954 }
4955
gfx_v8_0_pre_soft_reset(struct amdgpu_ip_block * ip_block)4956 static int gfx_v8_0_pre_soft_reset(struct amdgpu_ip_block *ip_block)
4957 {
4958 struct amdgpu_device *adev = ip_block->adev;
4959 u32 grbm_soft_reset = 0;
4960
4961 if ((!adev->gfx.grbm_soft_reset) &&
4962 (!adev->gfx.srbm_soft_reset))
4963 return 0;
4964
4965 grbm_soft_reset = adev->gfx.grbm_soft_reset;
4966
4967 /* stop the rlc */
4968 adev->gfx.rlc.funcs->stop(adev);
4969
4970 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
4971 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
4972 /* Disable GFX parsing/prefetching */
4973 gfx_v8_0_cp_gfx_enable(adev, false);
4974
4975 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
4976 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
4977 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
4978 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
4979 int i;
4980
4981 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
4982 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
4983
4984 mutex_lock(&adev->srbm_mutex);
4985 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
4986 gfx_v8_0_deactivate_hqd(adev, 2);
4987 vi_srbm_select(adev, 0, 0, 0, 0);
4988 mutex_unlock(&adev->srbm_mutex);
4989 }
4990 /* Disable MEC parsing/prefetching */
4991 gfx_v8_0_cp_compute_enable(adev, false);
4992 }
4993
4994 return 0;
4995 }
4996
gfx_v8_0_soft_reset(struct amdgpu_ip_block * ip_block)4997 static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block)
4998 {
4999 struct amdgpu_device *adev = ip_block->adev;
5000 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
5001 u32 tmp;
5002
5003 if ((!adev->gfx.grbm_soft_reset) &&
5004 (!adev->gfx.srbm_soft_reset))
5005 return 0;
5006
5007 grbm_soft_reset = adev->gfx.grbm_soft_reset;
5008 srbm_soft_reset = adev->gfx.srbm_soft_reset;
5009
5010 if (grbm_soft_reset || srbm_soft_reset) {
5011 tmp = RREG32(mmGMCON_DEBUG);
5012 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1);
5013 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1);
5014 WREG32(mmGMCON_DEBUG, tmp);
5015 udelay(50);
5016 }
5017
5018 if (grbm_soft_reset) {
5019 tmp = RREG32(mmGRBM_SOFT_RESET);
5020 tmp |= grbm_soft_reset;
5021 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
5022 WREG32(mmGRBM_SOFT_RESET, tmp);
5023 tmp = RREG32(mmGRBM_SOFT_RESET);
5024
5025 udelay(50);
5026
5027 tmp &= ~grbm_soft_reset;
5028 WREG32(mmGRBM_SOFT_RESET, tmp);
5029 tmp = RREG32(mmGRBM_SOFT_RESET);
5030 }
5031
5032 if (srbm_soft_reset) {
5033 tmp = RREG32(mmSRBM_SOFT_RESET);
5034 tmp |= srbm_soft_reset;
5035 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
5036 WREG32(mmSRBM_SOFT_RESET, tmp);
5037 tmp = RREG32(mmSRBM_SOFT_RESET);
5038
5039 udelay(50);
5040
5041 tmp &= ~srbm_soft_reset;
5042 WREG32(mmSRBM_SOFT_RESET, tmp);
5043 tmp = RREG32(mmSRBM_SOFT_RESET);
5044 }
5045
5046 if (grbm_soft_reset || srbm_soft_reset) {
5047 tmp = RREG32(mmGMCON_DEBUG);
5048 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0);
5049 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0);
5050 WREG32(mmGMCON_DEBUG, tmp);
5051 }
5052
5053 /* Wait a little for things to settle down */
5054 udelay(50);
5055
5056 return 0;
5057 }
5058
gfx_v8_0_post_soft_reset(struct amdgpu_ip_block * ip_block)5059 static int gfx_v8_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
5060 {
5061 struct amdgpu_device *adev = ip_block->adev;
5062 u32 grbm_soft_reset = 0;
5063
5064 if ((!adev->gfx.grbm_soft_reset) &&
5065 (!adev->gfx.srbm_soft_reset))
5066 return 0;
5067
5068 grbm_soft_reset = adev->gfx.grbm_soft_reset;
5069
5070 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5071 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) ||
5072 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) ||
5073 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) {
5074 int i;
5075
5076 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5077 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i];
5078
5079 mutex_lock(&adev->srbm_mutex);
5080 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
5081 gfx_v8_0_deactivate_hqd(adev, 2);
5082 vi_srbm_select(adev, 0, 0, 0, 0);
5083 mutex_unlock(&adev->srbm_mutex);
5084 }
5085 gfx_v8_0_kiq_resume(adev);
5086 gfx_v8_0_kcq_resume(adev);
5087 }
5088
5089 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) ||
5090 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX))
5091 gfx_v8_0_cp_gfx_resume(adev);
5092
5093 gfx_v8_0_cp_test_all_rings(adev);
5094
5095 adev->gfx.rlc.funcs->start(adev);
5096
5097 return 0;
5098 }
5099
5100 /**
5101 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot
5102 *
5103 * @adev: amdgpu_device pointer
5104 *
5105 * Fetches a GPU clock counter snapshot.
5106 * Returns the 64 bit clock counter snapshot.
5107 */
gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device * adev)5108 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
5109 {
5110 uint64_t clock;
5111
5112 mutex_lock(&adev->gfx.gpu_clock_mutex);
5113 WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5114 clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) |
5115 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5116 mutex_unlock(&adev->gfx.gpu_clock_mutex);
5117 return clock;
5118 }
5119
gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring * ring,uint32_t vmid,uint32_t gds_base,uint32_t gds_size,uint32_t gws_base,uint32_t gws_size,uint32_t oa_base,uint32_t oa_size)5120 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
5121 uint32_t vmid,
5122 uint32_t gds_base, uint32_t gds_size,
5123 uint32_t gws_base, uint32_t gws_size,
5124 uint32_t oa_base, uint32_t oa_size)
5125 {
5126 /* GDS Base */
5127 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5128 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5129 WRITE_DATA_DST_SEL(0)));
5130 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base);
5131 amdgpu_ring_write(ring, 0);
5132 amdgpu_ring_write(ring, gds_base);
5133
5134 /* GDS Size */
5135 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5136 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5137 WRITE_DATA_DST_SEL(0)));
5138 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size);
5139 amdgpu_ring_write(ring, 0);
5140 amdgpu_ring_write(ring, gds_size);
5141
5142 /* GWS */
5143 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5144 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5145 WRITE_DATA_DST_SEL(0)));
5146 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws);
5147 amdgpu_ring_write(ring, 0);
5148 amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
5149
5150 /* OA */
5151 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5152 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5153 WRITE_DATA_DST_SEL(0)));
5154 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa);
5155 amdgpu_ring_write(ring, 0);
5156 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
5157 }
5158
wave_read_ind(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t address)5159 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
5160 {
5161 WREG32(mmSQ_IND_INDEX,
5162 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5163 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5164 (address << SQ_IND_INDEX__INDEX__SHIFT) |
5165 (SQ_IND_INDEX__FORCE_READ_MASK));
5166 return RREG32(mmSQ_IND_DATA);
5167 }
5168
wave_read_regs(struct amdgpu_device * adev,uint32_t simd,uint32_t wave,uint32_t thread,uint32_t regno,uint32_t num,uint32_t * out)5169 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
5170 uint32_t wave, uint32_t thread,
5171 uint32_t regno, uint32_t num, uint32_t *out)
5172 {
5173 WREG32(mmSQ_IND_INDEX,
5174 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
5175 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
5176 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
5177 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
5178 (SQ_IND_INDEX__FORCE_READ_MASK) |
5179 (SQ_IND_INDEX__AUTO_INCR_MASK));
5180 while (num--)
5181 *(out++) = RREG32(mmSQ_IND_DATA);
5182 }
5183
gfx_v8_0_read_wave_data(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t * dst,int * no_fields)5184 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
5185 {
5186 /* type 0 wave data */
5187 dst[(*no_fields)++] = 0;
5188 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
5189 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
5190 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
5191 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
5192 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
5193 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
5194 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
5195 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
5196 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
5197 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
5198 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
5199 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
5200 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO);
5201 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI);
5202 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO);
5203 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI);
5204 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
5205 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
5206 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE);
5207 }
5208
gfx_v8_0_read_wave_sgprs(struct amdgpu_device * adev,uint32_t xcc_id,uint32_t simd,uint32_t wave,uint32_t start,uint32_t size,uint32_t * dst)5209 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd,
5210 uint32_t wave, uint32_t start,
5211 uint32_t size, uint32_t *dst)
5212 {
5213 wave_read_regs(
5214 adev, simd, wave, 0,
5215 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
5216 }
5217
5218 /**
5219 * gfx_v8_0_get_hdp_flush_mask - get the reference and mask for HDP flush
5220 *
5221 * @ring: amdgpu_ring structure holding ring information
5222 * @ref_and_mask: pointer to store the reference and mask
5223 * @reg_mem_engine: pointer to store the register memory engine
5224 *
5225 * Calculates the reference and mask for HDP flush based on the ring type and me.
5226 */
gfx_v8_0_get_hdp_flush_mask(struct amdgpu_ring * ring,uint32_t * ref_and_mask,uint32_t * reg_mem_engine)5227 static void gfx_v8_0_get_hdp_flush_mask(struct amdgpu_ring *ring,
5228 uint32_t *ref_and_mask, uint32_t *reg_mem_engine)
5229 {
5230 if (!ring || !ref_and_mask || !reg_mem_engine) {
5231 DRM_INFO("%s:invalid params\n", __func__);
5232 return;
5233 }
5234
5235 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) ||
5236 (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) {
5237 switch (ring->me) {
5238 case 1:
5239 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe;
5240 break;
5241 case 2:
5242 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe;
5243 break;
5244 default:
5245 return;
5246 }
5247 *reg_mem_engine = 0;
5248 } else {
5249 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK;
5250 *reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */
5251 }
5252 }
5253
5254 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
5255 .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
5256 .select_se_sh = &gfx_v8_0_select_se_sh,
5257 .read_wave_data = &gfx_v8_0_read_wave_data,
5258 .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs,
5259 .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q,
5260 .get_hdp_flush_mask = &gfx_v8_0_get_hdp_flush_mask,
5261 };
5262
gfx_v8_0_early_init(struct amdgpu_ip_block * ip_block)5263 static int gfx_v8_0_early_init(struct amdgpu_ip_block *ip_block)
5264 {
5265 struct amdgpu_device *adev = ip_block->adev;
5266
5267 adev->gfx.xcc_mask = 1;
5268 adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;
5269 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev),
5270 AMDGPU_MAX_COMPUTE_RINGS);
5271 adev->gfx.funcs = &gfx_v8_0_gfx_funcs;
5272 gfx_v8_0_set_ring_funcs(adev);
5273 gfx_v8_0_set_irq_funcs(adev);
5274 gfx_v8_0_set_gds_init(adev);
5275 gfx_v8_0_set_rlc_funcs(adev);
5276
5277 return 0;
5278 }
5279
gfx_v8_0_late_init(struct amdgpu_ip_block * ip_block)5280 static int gfx_v8_0_late_init(struct amdgpu_ip_block *ip_block)
5281 {
5282 struct amdgpu_device *adev = ip_block->adev;
5283 int r;
5284
5285 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
5286 if (r)
5287 return r;
5288
5289 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
5290 if (r)
5291 return r;
5292
5293 /* requires IBs so do in late init after IB pool is initialized */
5294 r = gfx_v8_0_do_edc_gpr_workarounds(adev);
5295 if (r)
5296 return r;
5297
5298 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0);
5299 if (r) {
5300 DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r);
5301 return r;
5302 }
5303
5304 r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0);
5305 if (r) {
5306 DRM_ERROR(
5307 "amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n",
5308 r);
5309 return r;
5310 }
5311
5312 return 0;
5313 }
5314
gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device * adev,bool enable)5315 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
5316 bool enable)
5317 {
5318 if ((adev->asic_type == CHIP_POLARIS11) ||
5319 (adev->asic_type == CHIP_POLARIS12) ||
5320 (adev->asic_type == CHIP_VEGAM))
5321 /* Send msg to SMU via Powerplay */
5322 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable, 0);
5323
5324 WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0);
5325 }
5326
gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device * adev,bool enable)5327 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
5328 bool enable)
5329 {
5330 WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0);
5331 }
5332
polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device * adev,bool enable)5333 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
5334 bool enable)
5335 {
5336 WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0);
5337 }
5338
cz_enable_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)5339 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
5340 bool enable)
5341 {
5342 WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0);
5343 }
5344
cz_enable_gfx_pipeline_power_gating(struct amdgpu_device * adev,bool enable)5345 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
5346 bool enable)
5347 {
5348 WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0);
5349
5350 /* Read any GFX register to wake up GFX. */
5351 if (!enable)
5352 RREG32(mmDB_RENDER_CONTROL);
5353 }
5354
cz_update_gfx_cg_power_gating(struct amdgpu_device * adev,bool enable)5355 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
5356 bool enable)
5357 {
5358 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
5359 cz_enable_gfx_cg_power_gating(adev, true);
5360 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
5361 cz_enable_gfx_pipeline_power_gating(adev, true);
5362 } else {
5363 cz_enable_gfx_cg_power_gating(adev, false);
5364 cz_enable_gfx_pipeline_power_gating(adev, false);
5365 }
5366 }
5367
gfx_v8_0_set_powergating_state(struct amdgpu_ip_block * ip_block,enum amd_powergating_state state)5368 static int gfx_v8_0_set_powergating_state(struct amdgpu_ip_block *ip_block,
5369 enum amd_powergating_state state)
5370 {
5371 struct amdgpu_device *adev = ip_block->adev;
5372 bool enable = (state == AMD_PG_STATE_GATE);
5373
5374 if (amdgpu_sriov_vf(adev))
5375 return 0;
5376
5377 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5378 AMD_PG_SUPPORT_RLC_SMU_HS |
5379 AMD_PG_SUPPORT_CP |
5380 AMD_PG_SUPPORT_GFX_DMG))
5381 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5382 switch (adev->asic_type) {
5383 case CHIP_CARRIZO:
5384 case CHIP_STONEY:
5385
5386 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5387 cz_enable_sck_slow_down_on_power_up(adev, true);
5388 cz_enable_sck_slow_down_on_power_down(adev, true);
5389 } else {
5390 cz_enable_sck_slow_down_on_power_up(adev, false);
5391 cz_enable_sck_slow_down_on_power_down(adev, false);
5392 }
5393 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5394 cz_enable_cp_power_gating(adev, true);
5395 else
5396 cz_enable_cp_power_gating(adev, false);
5397
5398 cz_update_gfx_cg_power_gating(adev, enable);
5399
5400 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5401 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5402 else
5403 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5404
5405 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5406 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5407 else
5408 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5409 break;
5410 case CHIP_POLARIS11:
5411 case CHIP_POLARIS12:
5412 case CHIP_VEGAM:
5413 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
5414 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
5415 else
5416 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
5417
5418 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
5419 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
5420 else
5421 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
5422
5423 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)
5424 polaris11_enable_gfx_quick_mg_power_gating(adev, true);
5425 else
5426 polaris11_enable_gfx_quick_mg_power_gating(adev, false);
5427 break;
5428 default:
5429 break;
5430 }
5431 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG |
5432 AMD_PG_SUPPORT_RLC_SMU_HS |
5433 AMD_PG_SUPPORT_CP |
5434 AMD_PG_SUPPORT_GFX_DMG))
5435 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5436 return 0;
5437 }
5438
gfx_v8_0_get_clockgating_state(struct amdgpu_ip_block * ip_block,u64 * flags)5439 static void gfx_v8_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags)
5440 {
5441 struct amdgpu_device *adev = ip_block->adev;
5442 int data;
5443
5444 if (amdgpu_sriov_vf(adev))
5445 *flags = 0;
5446
5447 /* AMD_CG_SUPPORT_GFX_MGCG */
5448 data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5449 if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK))
5450 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5451
5452 /* AMD_CG_SUPPORT_GFX_CGLG */
5453 data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5454 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5455 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5456
5457 /* AMD_CG_SUPPORT_GFX_CGLS */
5458 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5459 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5460
5461 /* AMD_CG_SUPPORT_GFX_CGTS */
5462 data = RREG32(mmCGTS_SM_CTRL_REG);
5463 if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK))
5464 *flags |= AMD_CG_SUPPORT_GFX_CGTS;
5465
5466 /* AMD_CG_SUPPORT_GFX_CGTS_LS */
5467 if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK))
5468 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS;
5469
5470 /* AMD_CG_SUPPORT_GFX_RLC_LS */
5471 data = RREG32(mmRLC_MEM_SLP_CNTL);
5472 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5473 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5474
5475 /* AMD_CG_SUPPORT_GFX_CP_LS */
5476 data = RREG32(mmCP_MEM_SLP_CNTL);
5477 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5478 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5479 }
5480
gfx_v8_0_send_serdes_cmd(struct amdgpu_device * adev,uint32_t reg_addr,uint32_t cmd)5481 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
5482 uint32_t reg_addr, uint32_t cmd)
5483 {
5484 uint32_t data;
5485
5486 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
5487
5488 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5489 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5490
5491 data = RREG32(mmRLC_SERDES_WR_CTRL);
5492 if (adev->asic_type == CHIP_STONEY)
5493 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5494 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5495 RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5496 RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5497 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5498 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5499 RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5500 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5501 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5502 else
5503 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK |
5504 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK |
5505 RLC_SERDES_WR_CTRL__P1_SELECT_MASK |
5506 RLC_SERDES_WR_CTRL__P2_SELECT_MASK |
5507 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK |
5508 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK |
5509 RLC_SERDES_WR_CTRL__POWER_UP_MASK |
5510 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK |
5511 RLC_SERDES_WR_CTRL__BPM_DATA_MASK |
5512 RLC_SERDES_WR_CTRL__REG_ADDR_MASK |
5513 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK);
5514 data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK |
5515 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) |
5516 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) |
5517 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT));
5518
5519 WREG32(mmRLC_SERDES_WR_CTRL, data);
5520 }
5521
5522 #define MSG_ENTER_RLC_SAFE_MODE 1
5523 #define MSG_EXIT_RLC_SAFE_MODE 0
5524 #define RLC_GPR_REG2__REQ_MASK 0x00000001
5525 #define RLC_GPR_REG2__REQ__SHIFT 0
5526 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001
5527 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e
5528
gfx_v8_0_is_rlc_enabled(struct amdgpu_device * adev)5529 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev)
5530 {
5531 uint32_t rlc_setting;
5532
5533 rlc_setting = RREG32(mmRLC_CNTL);
5534 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
5535 return false;
5536
5537 return true;
5538 }
5539
gfx_v8_0_set_safe_mode(struct amdgpu_device * adev,int xcc_id)5540 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id)
5541 {
5542 uint32_t data;
5543 unsigned i;
5544 data = RREG32(mmRLC_CNTL);
5545 data |= RLC_SAFE_MODE__CMD_MASK;
5546 data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5547 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
5548 WREG32(mmRLC_SAFE_MODE, data);
5549
5550 /* wait for RLC_SAFE_MODE */
5551 for (i = 0; i < adev->usec_timeout; i++) {
5552 if ((RREG32(mmRLC_GPM_STAT) &
5553 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5554 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) ==
5555 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK |
5556 RLC_GPM_STAT__GFX_POWER_STATUS_MASK))
5557 break;
5558 udelay(1);
5559 }
5560 for (i = 0; i < adev->usec_timeout; i++) {
5561 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5562 break;
5563 udelay(1);
5564 }
5565 }
5566
gfx_v8_0_unset_safe_mode(struct amdgpu_device * adev,int xcc_id)5567 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id)
5568 {
5569 uint32_t data;
5570 unsigned i;
5571
5572 data = RREG32(mmRLC_CNTL);
5573 data |= RLC_SAFE_MODE__CMD_MASK;
5574 data &= ~RLC_SAFE_MODE__MESSAGE_MASK;
5575 WREG32(mmRLC_SAFE_MODE, data);
5576
5577 for (i = 0; i < adev->usec_timeout; i++) {
5578 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
5579 break;
5580 udelay(1);
5581 }
5582 }
5583
gfx_v8_0_update_spm_vmid(struct amdgpu_device * adev,int xcc_id,struct amdgpu_ring * ring,unsigned vmid)5584 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id,
5585 struct amdgpu_ring *ring, unsigned vmid)
5586 {
5587 u32 data;
5588
5589 amdgpu_gfx_off_ctrl(adev, false);
5590
5591 if (amdgpu_sriov_is_pp_one_vf(adev))
5592 data = RREG32_NO_KIQ(mmRLC_SPM_VMID);
5593 else
5594 data = RREG32(mmRLC_SPM_VMID);
5595
5596 data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK;
5597 data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT;
5598
5599 if (amdgpu_sriov_is_pp_one_vf(adev))
5600 WREG32_NO_KIQ(mmRLC_SPM_VMID, data);
5601 else
5602 WREG32(mmRLC_SPM_VMID, data);
5603
5604 amdgpu_gfx_off_ctrl(adev, true);
5605 }
5606
5607 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = {
5608 .is_rlc_enabled = gfx_v8_0_is_rlc_enabled,
5609 .set_safe_mode = gfx_v8_0_set_safe_mode,
5610 .unset_safe_mode = gfx_v8_0_unset_safe_mode,
5611 .init = gfx_v8_0_rlc_init,
5612 .get_csb_size = gfx_v8_0_get_csb_size,
5613 .get_csb_buffer = gfx_v8_0_get_csb_buffer,
5614 .get_cp_table_num = gfx_v8_0_cp_jump_table_num,
5615 .resume = gfx_v8_0_rlc_resume,
5616 .stop = gfx_v8_0_rlc_stop,
5617 .reset = gfx_v8_0_rlc_reset,
5618 .start = gfx_v8_0_rlc_start,
5619 .update_spm_vmid = gfx_v8_0_update_spm_vmid
5620 };
5621
gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device * adev,bool enable)5622 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
5623 bool enable)
5624 {
5625 uint32_t temp, data;
5626
5627 /* It is disabled by HW by default */
5628 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
5629 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5630 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS)
5631 /* 1 - RLC memory Light sleep */
5632 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1);
5633
5634 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS)
5635 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1);
5636 }
5637
5638 /* 3 - RLC_CGTT_MGCG_OVERRIDE */
5639 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5640 if (adev->flags & AMD_IS_APU)
5641 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5642 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5643 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK);
5644 else
5645 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5646 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5647 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5648 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5649
5650 if (temp != data)
5651 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5652
5653 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5654 gfx_v8_0_wait_for_rlc_serdes(adev);
5655
5656 /* 5 - clear mgcg override */
5657 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5658
5659 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) {
5660 /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */
5661 temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5662 data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK);
5663 data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT);
5664 data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK;
5665 data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK;
5666 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) &&
5667 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS))
5668 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK;
5669 data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK;
5670 data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT);
5671 if (temp != data)
5672 WREG32(mmCGTS_SM_CTRL_REG, data);
5673 }
5674 udelay(50);
5675
5676 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5677 gfx_v8_0_wait_for_rlc_serdes(adev);
5678 } else {
5679 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */
5680 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5681 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK |
5682 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK |
5683 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK |
5684 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK);
5685 if (temp != data)
5686 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data);
5687
5688 /* 2 - disable MGLS in RLC */
5689 data = RREG32(mmRLC_MEM_SLP_CNTL);
5690 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
5691 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
5692 WREG32(mmRLC_MEM_SLP_CNTL, data);
5693 }
5694
5695 /* 3 - disable MGLS in CP */
5696 data = RREG32(mmCP_MEM_SLP_CNTL);
5697 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
5698 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
5699 WREG32(mmCP_MEM_SLP_CNTL, data);
5700 }
5701
5702 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */
5703 temp = data = RREG32(mmCGTS_SM_CTRL_REG);
5704 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK |
5705 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK);
5706 if (temp != data)
5707 WREG32(mmCGTS_SM_CTRL_REG, data);
5708
5709 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5710 gfx_v8_0_wait_for_rlc_serdes(adev);
5711
5712 /* 6 - set mgcg override */
5713 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5714
5715 udelay(50);
5716
5717 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5718 gfx_v8_0_wait_for_rlc_serdes(adev);
5719 }
5720 }
5721
gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device * adev,bool enable)5722 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
5723 bool enable)
5724 {
5725 uint32_t temp, temp1, data, data1;
5726
5727 temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL);
5728
5729 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
5730 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5731 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK;
5732 if (temp1 != data1)
5733 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5734
5735 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5736 gfx_v8_0_wait_for_rlc_serdes(adev);
5737
5738 /* 2 - clear cgcg override */
5739 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD);
5740
5741 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5742 gfx_v8_0_wait_for_rlc_serdes(adev);
5743
5744 /* 3 - write cmd to set CGLS */
5745 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD);
5746
5747 /* 4 - enable cgcg */
5748 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
5749
5750 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5751 /* enable cgls*/
5752 data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5753
5754 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5755 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK;
5756
5757 if (temp1 != data1)
5758 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5759 } else {
5760 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
5761 }
5762
5763 if (temp != data)
5764 WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5765
5766 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/
5767 * Cmp_busy/GFX_Idle interrupts
5768 */
5769 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5770 } else {
5771 /* disable cntx_empty_int_enable & GFX Idle interrupt */
5772 gfx_v8_0_enable_gui_idle_interrupt(adev, false);
5773
5774 /* TEST CGCG */
5775 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE);
5776 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK |
5777 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK);
5778 if (temp1 != data1)
5779 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1);
5780
5781 /* read gfx register to wake up cgcg */
5782 RREG32(mmCB_CGTT_SCLK_CTRL);
5783 RREG32(mmCB_CGTT_SCLK_CTRL);
5784 RREG32(mmCB_CGTT_SCLK_CTRL);
5785 RREG32(mmCB_CGTT_SCLK_CTRL);
5786
5787 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5788 gfx_v8_0_wait_for_rlc_serdes(adev);
5789
5790 /* write cmd to Set CGCG Override */
5791 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD);
5792
5793 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */
5794 gfx_v8_0_wait_for_rlc_serdes(adev);
5795
5796 /* write cmd to Clear CGLS */
5797 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD);
5798
5799 /* disable cgcg, cgls should be disabled too. */
5800 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK |
5801 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
5802 if (temp != data)
5803 WREG32(mmRLC_CGCG_CGLS_CTRL, data);
5804 /* enable interrupts again for PG */
5805 gfx_v8_0_enable_gui_idle_interrupt(adev, true);
5806 }
5807
5808 gfx_v8_0_wait_for_rlc_serdes(adev);
5809 }
gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device * adev,bool enable)5810 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
5811 bool enable)
5812 {
5813 amdgpu_gfx_rlc_enter_safe_mode(adev, 0);
5814
5815 if (enable) {
5816 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS)
5817 * === MGCG + MGLS + TS(CG/LS) ===
5818 */
5819 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5820 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5821 } else {
5822 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS)
5823 * === CGCG + CGLS ===
5824 */
5825 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable);
5826 gfx_v8_0_update_medium_grain_clock_gating(adev, enable);
5827 }
5828
5829 amdgpu_gfx_rlc_exit_safe_mode(adev, 0);
5830 return 0;
5831 }
5832
gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device * adev,enum amd_clockgating_state state)5833 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev,
5834 enum amd_clockgating_state state)
5835 {
5836 uint32_t msg_id, pp_state = 0;
5837 uint32_t pp_support_state = 0;
5838
5839 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5840 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5841 pp_support_state = PP_STATE_SUPPORT_LS;
5842 pp_state = PP_STATE_LS;
5843 }
5844 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5845 pp_support_state |= PP_STATE_SUPPORT_CG;
5846 pp_state |= PP_STATE_CG;
5847 }
5848 if (state == AMD_CG_STATE_UNGATE)
5849 pp_state = 0;
5850
5851 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5852 PP_BLOCK_GFX_CG,
5853 pp_support_state,
5854 pp_state);
5855 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5856 }
5857
5858 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5859 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5860 pp_support_state = PP_STATE_SUPPORT_LS;
5861 pp_state = PP_STATE_LS;
5862 }
5863
5864 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5865 pp_support_state |= PP_STATE_SUPPORT_CG;
5866 pp_state |= PP_STATE_CG;
5867 }
5868
5869 if (state == AMD_CG_STATE_UNGATE)
5870 pp_state = 0;
5871
5872 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5873 PP_BLOCK_GFX_MG,
5874 pp_support_state,
5875 pp_state);
5876 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5877 }
5878
5879 return 0;
5880 }
5881
gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device * adev,enum amd_clockgating_state state)5882 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev,
5883 enum amd_clockgating_state state)
5884 {
5885
5886 uint32_t msg_id, pp_state = 0;
5887 uint32_t pp_support_state = 0;
5888
5889 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) {
5890 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) {
5891 pp_support_state = PP_STATE_SUPPORT_LS;
5892 pp_state = PP_STATE_LS;
5893 }
5894 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) {
5895 pp_support_state |= PP_STATE_SUPPORT_CG;
5896 pp_state |= PP_STATE_CG;
5897 }
5898 if (state == AMD_CG_STATE_UNGATE)
5899 pp_state = 0;
5900
5901 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5902 PP_BLOCK_GFX_CG,
5903 pp_support_state,
5904 pp_state);
5905 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5906 }
5907
5908 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) {
5909 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) {
5910 pp_support_state = PP_STATE_SUPPORT_LS;
5911 pp_state = PP_STATE_LS;
5912 }
5913 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) {
5914 pp_support_state |= PP_STATE_SUPPORT_CG;
5915 pp_state |= PP_STATE_CG;
5916 }
5917 if (state == AMD_CG_STATE_UNGATE)
5918 pp_state = 0;
5919
5920 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5921 PP_BLOCK_GFX_3D,
5922 pp_support_state,
5923 pp_state);
5924 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5925 }
5926
5927 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) {
5928 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
5929 pp_support_state = PP_STATE_SUPPORT_LS;
5930 pp_state = PP_STATE_LS;
5931 }
5932
5933 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) {
5934 pp_support_state |= PP_STATE_SUPPORT_CG;
5935 pp_state |= PP_STATE_CG;
5936 }
5937
5938 if (state == AMD_CG_STATE_UNGATE)
5939 pp_state = 0;
5940
5941 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5942 PP_BLOCK_GFX_MG,
5943 pp_support_state,
5944 pp_state);
5945 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5946 }
5947
5948 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
5949 pp_support_state = PP_STATE_SUPPORT_LS;
5950
5951 if (state == AMD_CG_STATE_UNGATE)
5952 pp_state = 0;
5953 else
5954 pp_state = PP_STATE_LS;
5955
5956 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5957 PP_BLOCK_GFX_RLC,
5958 pp_support_state,
5959 pp_state);
5960 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5961 }
5962
5963 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
5964 pp_support_state = PP_STATE_SUPPORT_LS;
5965
5966 if (state == AMD_CG_STATE_UNGATE)
5967 pp_state = 0;
5968 else
5969 pp_state = PP_STATE_LS;
5970 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX,
5971 PP_BLOCK_GFX_CP,
5972 pp_support_state,
5973 pp_state);
5974 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id);
5975 }
5976
5977 return 0;
5978 }
5979
gfx_v8_0_set_clockgating_state(struct amdgpu_ip_block * ip_block,enum amd_clockgating_state state)5980 static int gfx_v8_0_set_clockgating_state(struct amdgpu_ip_block *ip_block,
5981 enum amd_clockgating_state state)
5982 {
5983 struct amdgpu_device *adev = ip_block->adev;
5984
5985 if (amdgpu_sriov_vf(adev))
5986 return 0;
5987
5988 switch (adev->asic_type) {
5989 case CHIP_FIJI:
5990 case CHIP_CARRIZO:
5991 case CHIP_STONEY:
5992 gfx_v8_0_update_gfx_clock_gating(adev,
5993 state == AMD_CG_STATE_GATE);
5994 break;
5995 case CHIP_TONGA:
5996 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state);
5997 break;
5998 case CHIP_POLARIS10:
5999 case CHIP_POLARIS11:
6000 case CHIP_POLARIS12:
6001 case CHIP_VEGAM:
6002 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state);
6003 break;
6004 default:
6005 break;
6006 }
6007 return 0;
6008 }
6009
gfx_v8_0_ring_get_rptr(struct amdgpu_ring * ring)6010 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring)
6011 {
6012 return *ring->rptr_cpu_addr;
6013 }
6014
gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring * ring)6015 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
6016 {
6017 struct amdgpu_device *adev = ring->adev;
6018
6019 if (ring->use_doorbell)
6020 /* XXX check if swapping is necessary on BE */
6021 return *ring->wptr_cpu_addr;
6022 else
6023 return RREG32(mmCP_RB0_WPTR);
6024 }
6025
gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring * ring)6026 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
6027 {
6028 struct amdgpu_device *adev = ring->adev;
6029
6030 if (ring->use_doorbell) {
6031 /* XXX check if swapping is necessary on BE */
6032 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
6033 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6034 } else {
6035 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
6036 (void)RREG32(mmCP_RB0_WPTR);
6037 }
6038 }
6039
gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring * ring)6040 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
6041 {
6042 u32 ref_and_mask, reg_mem_engine;
6043 struct amdgpu_device *adev = ring->adev;
6044
6045 if (!adev->gfx.funcs->get_hdp_flush_mask) {
6046 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__);
6047 return;
6048 }
6049
6050 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine);
6051 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6052 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
6053 WAIT_REG_MEM_FUNCTION(3) | /* == */
6054 reg_mem_engine));
6055 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ);
6056 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE);
6057 amdgpu_ring_write(ring, ref_and_mask);
6058 amdgpu_ring_write(ring, ref_and_mask);
6059 amdgpu_ring_write(ring, 0x20); /* poll interval */
6060 }
6061
gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring * ring)6062 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring)
6063 {
6064 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6065 amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) |
6066 EVENT_INDEX(4));
6067
6068 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0));
6069 amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) |
6070 EVENT_INDEX(0));
6071 }
6072
gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)6073 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
6074 struct amdgpu_job *job,
6075 struct amdgpu_ib *ib,
6076 uint32_t flags)
6077 {
6078 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6079 u32 header, control = 0;
6080
6081 if (ib->flags & AMDGPU_IB_FLAG_CE)
6082 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
6083 else
6084 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
6085
6086 control |= ib->length_dw | (vmid << 24);
6087
6088 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
6089 control |= INDIRECT_BUFFER_PRE_ENB(1);
6090
6091 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
6092 gfx_v8_0_ring_emit_de_meta(ring);
6093 }
6094
6095 amdgpu_ring_write(ring, header);
6096 amdgpu_ring_write(ring,
6097 #ifdef __BIG_ENDIAN
6098 (2 << 0) |
6099 #endif
6100 (ib->gpu_addr & 0xFFFFFFFC));
6101 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6102 amdgpu_ring_write(ring, control);
6103 }
6104
gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring * ring,struct amdgpu_job * job,struct amdgpu_ib * ib,uint32_t flags)6105 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
6106 struct amdgpu_job *job,
6107 struct amdgpu_ib *ib,
6108 uint32_t flags)
6109 {
6110 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
6111 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
6112
6113 /* Currently, there is a high possibility to get wave ID mismatch
6114 * between ME and GDS, leading to a hw deadlock, because ME generates
6115 * different wave IDs than the GDS expects. This situation happens
6116 * randomly when at least 5 compute pipes use GDS ordered append.
6117 * The wave IDs generated by ME are also wrong after suspend/resume.
6118 * Those are probably bugs somewhere else in the kernel driver.
6119 *
6120 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
6121 * GDS to 0 for this ring (me/pipe).
6122 */
6123 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
6124 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
6125 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START);
6126 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
6127 }
6128
6129 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
6130 amdgpu_ring_write(ring,
6131 #ifdef __BIG_ENDIAN
6132 (2 << 0) |
6133 #endif
6134 (ib->gpu_addr & 0xFFFFFFFC));
6135 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
6136 amdgpu_ring_write(ring, control);
6137 }
6138
gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)6139 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr,
6140 u64 seq, unsigned flags)
6141 {
6142 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6143 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6144 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC;
6145
6146 /* Workaround for cache flush problems. First send a dummy EOP
6147 * event down the pipe with seq one below.
6148 */
6149 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6150 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6151 EOP_TC_ACTION_EN |
6152 EOP_TC_WB_ACTION_EN |
6153 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6154 EVENT_INDEX(5)));
6155 amdgpu_ring_write(ring, addr & 0xfffffffc);
6156 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6157 DATA_SEL(1) | INT_SEL(0));
6158 amdgpu_ring_write(ring, lower_32_bits(seq - 1));
6159 amdgpu_ring_write(ring, upper_32_bits(seq - 1));
6160
6161 /* Then send the real EOP event down the pipe:
6162 * EVENT_WRITE_EOP - flush caches, send int */
6163 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
6164 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6165 EOP_TC_ACTION_EN |
6166 EOP_TC_WB_ACTION_EN |
6167 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6168 EVENT_INDEX(5) |
6169 (exec ? EOP_EXEC : 0)));
6170 amdgpu_ring_write(ring, addr & 0xfffffffc);
6171 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) |
6172 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6173 amdgpu_ring_write(ring, lower_32_bits(seq));
6174 amdgpu_ring_write(ring, upper_32_bits(seq));
6175
6176 }
6177
gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring * ring)6178 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
6179 {
6180 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6181 uint32_t seq = ring->fence_drv.sync_seq;
6182 uint64_t addr = ring->fence_drv.gpu_addr;
6183
6184 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6185 amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */
6186 WAIT_REG_MEM_FUNCTION(3) | /* equal */
6187 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */
6188 amdgpu_ring_write(ring, addr & 0xfffffffc);
6189 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff);
6190 amdgpu_ring_write(ring, seq);
6191 amdgpu_ring_write(ring, 0xffffffff);
6192 amdgpu_ring_write(ring, 4); /* poll interval */
6193 }
6194
gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring * ring,unsigned vmid,uint64_t pd_addr)6195 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
6196 unsigned vmid, uint64_t pd_addr)
6197 {
6198 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
6199
6200 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
6201
6202 /* wait for the invalidate to complete */
6203 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
6204 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */
6205 WAIT_REG_MEM_FUNCTION(0) | /* always */
6206 WAIT_REG_MEM_ENGINE(0))); /* me */
6207 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST);
6208 amdgpu_ring_write(ring, 0);
6209 amdgpu_ring_write(ring, 0); /* ref */
6210 amdgpu_ring_write(ring, 0); /* mask */
6211 amdgpu_ring_write(ring, 0x20); /* poll interval */
6212
6213 /* compute doesn't have PFP */
6214 if (usepfp) {
6215 /* sync PFP to ME, otherwise we might get invalid PFP reads */
6216 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
6217 amdgpu_ring_write(ring, 0x0);
6218 }
6219 }
6220
gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring * ring)6221 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
6222 {
6223 return *ring->wptr_cpu_addr;
6224 }
6225
gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring * ring)6226 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
6227 {
6228 struct amdgpu_device *adev = ring->adev;
6229
6230 /* XXX check if swapping is necessary on BE */
6231 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr);
6232 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr));
6233 }
6234
gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned flags)6235 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring,
6236 u64 addr, u64 seq,
6237 unsigned flags)
6238 {
6239 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
6240 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
6241
6242 /* RELEASE_MEM - flush caches, send int */
6243 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
6244 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN |
6245 EOP_TC_ACTION_EN |
6246 EOP_TC_WB_ACTION_EN |
6247 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
6248 EVENT_INDEX(5)));
6249 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
6250 amdgpu_ring_write(ring, addr & 0xfffffffc);
6251 amdgpu_ring_write(ring, upper_32_bits(addr));
6252 amdgpu_ring_write(ring, lower_32_bits(seq));
6253 amdgpu_ring_write(ring, upper_32_bits(seq));
6254 }
6255
gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring * ring,u64 addr,u64 seq,unsigned int flags)6256 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
6257 u64 seq, unsigned int flags)
6258 {
6259 /* write fence seq to the "addr" */
6260 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6261 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6262 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
6263 amdgpu_ring_write(ring, lower_32_bits(addr));
6264 amdgpu_ring_write(ring, upper_32_bits(addr));
6265 amdgpu_ring_write(ring, lower_32_bits(seq));
6266
6267 if (flags & AMDGPU_FENCE_FLAG_INT) {
6268 /* set register to trigger INT */
6269 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6270 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
6271 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
6272 amdgpu_ring_write(ring, mmCPC_INT_STATUS);
6273 amdgpu_ring_write(ring, 0);
6274 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
6275 }
6276 }
6277
gfx_v8_ring_emit_sb(struct amdgpu_ring * ring)6278 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
6279 {
6280 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
6281 amdgpu_ring_write(ring, 0);
6282 }
6283
gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring * ring,uint32_t flags)6284 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
6285 {
6286 uint32_t dw2 = 0;
6287
6288 if (amdgpu_sriov_vf(ring->adev))
6289 gfx_v8_0_ring_emit_ce_meta(ring);
6290
6291 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
6292 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
6293 gfx_v8_0_ring_emit_vgt_flush(ring);
6294 /* set load_global_config & load_global_uconfig */
6295 dw2 |= 0x8001;
6296 /* set load_cs_sh_regs */
6297 dw2 |= 0x01000000;
6298 /* set load_per_context_state & load_gfx_sh_regs for GFX */
6299 dw2 |= 0x10002;
6300
6301 /* set load_ce_ram if preamble presented */
6302 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
6303 dw2 |= 0x10000000;
6304 } else {
6305 /* still load_ce_ram if this is the first time preamble presented
6306 * although there is no context switch happens.
6307 */
6308 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
6309 dw2 |= 0x10000000;
6310 }
6311
6312 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6313 amdgpu_ring_write(ring, dw2);
6314 amdgpu_ring_write(ring, 0);
6315 }
6316
gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring * ring,uint64_t addr)6317 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
6318 uint64_t addr)
6319 {
6320 unsigned ret;
6321
6322 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
6323 amdgpu_ring_write(ring, lower_32_bits(addr));
6324 amdgpu_ring_write(ring, upper_32_bits(addr));
6325 /* discard following DWs if *cond_exec_gpu_addr==0 */
6326 amdgpu_ring_write(ring, 0);
6327 ret = ring->wptr & ring->buf_mask;
6328 /* patch dummy value later */
6329 amdgpu_ring_write(ring, 0);
6330 return ret;
6331 }
6332
gfx_v8_0_ring_emit_rreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t reg_val_offs)6333 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
6334 uint32_t reg_val_offs)
6335 {
6336 struct amdgpu_device *adev = ring->adev;
6337
6338 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
6339 amdgpu_ring_write(ring, 0 | /* src: register*/
6340 (5 << 8) | /* dst: memory */
6341 (1 << 20)); /* write confirm */
6342 amdgpu_ring_write(ring, reg);
6343 amdgpu_ring_write(ring, 0);
6344 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
6345 reg_val_offs * 4));
6346 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
6347 reg_val_offs * 4));
6348 }
6349
gfx_v8_0_ring_emit_wreg(struct amdgpu_ring * ring,uint32_t reg,uint32_t val)6350 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
6351 uint32_t val)
6352 {
6353 uint32_t cmd;
6354
6355 switch (ring->funcs->type) {
6356 case AMDGPU_RING_TYPE_GFX:
6357 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
6358 break;
6359 case AMDGPU_RING_TYPE_KIQ:
6360 cmd = 1 << 16; /* no inc addr */
6361 break;
6362 default:
6363 cmd = WR_CONFIRM;
6364 break;
6365 }
6366
6367 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
6368 amdgpu_ring_write(ring, cmd);
6369 amdgpu_ring_write(ring, reg);
6370 amdgpu_ring_write(ring, 0);
6371 amdgpu_ring_write(ring, val);
6372 }
6373
gfx_v8_0_ring_soft_recovery(struct amdgpu_ring * ring,unsigned vmid)6374 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
6375 {
6376 struct amdgpu_device *adev = ring->adev;
6377 uint32_t value = 0;
6378
6379 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
6380 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
6381 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
6382 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
6383 WREG32(mmSQ_CMD, value);
6384 }
6385
gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device * adev,enum amdgpu_interrupt_state state)6386 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
6387 enum amdgpu_interrupt_state state)
6388 {
6389 WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE,
6390 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6391 }
6392
gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device * adev,int me,int pipe,enum amdgpu_interrupt_state state)6393 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
6394 int me, int pipe,
6395 enum amdgpu_interrupt_state state)
6396 {
6397 u32 mec_int_cntl, mec_int_cntl_reg;
6398
6399 /*
6400 * amdgpu controls only the first MEC. That's why this function only
6401 * handles the setting of interrupts for this specific MEC. All other
6402 * pipes' interrupts are set by amdkfd.
6403 */
6404
6405 if (me == 1) {
6406 switch (pipe) {
6407 case 0:
6408 mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL;
6409 break;
6410 case 1:
6411 mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL;
6412 break;
6413 case 2:
6414 mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL;
6415 break;
6416 case 3:
6417 mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL;
6418 break;
6419 default:
6420 DRM_DEBUG("invalid pipe %d\n", pipe);
6421 return;
6422 }
6423 } else {
6424 DRM_DEBUG("invalid me %d\n", me);
6425 return;
6426 }
6427
6428 switch (state) {
6429 case AMDGPU_IRQ_STATE_DISABLE:
6430 mec_int_cntl = RREG32(mec_int_cntl_reg);
6431 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6432 WREG32(mec_int_cntl_reg, mec_int_cntl);
6433 break;
6434 case AMDGPU_IRQ_STATE_ENABLE:
6435 mec_int_cntl = RREG32(mec_int_cntl_reg);
6436 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK;
6437 WREG32(mec_int_cntl_reg, mec_int_cntl);
6438 break;
6439 default:
6440 break;
6441 }
6442 }
6443
gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6444 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
6445 struct amdgpu_irq_src *source,
6446 unsigned type,
6447 enum amdgpu_interrupt_state state)
6448 {
6449 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE,
6450 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6451
6452 return 0;
6453 }
6454
gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned type,enum amdgpu_interrupt_state state)6455 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
6456 struct amdgpu_irq_src *source,
6457 unsigned type,
6458 enum amdgpu_interrupt_state state)
6459 {
6460 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE,
6461 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1);
6462
6463 return 0;
6464 }
6465
gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned type,enum amdgpu_interrupt_state state)6466 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev,
6467 struct amdgpu_irq_src *src,
6468 unsigned type,
6469 enum amdgpu_interrupt_state state)
6470 {
6471 switch (type) {
6472 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
6473 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state);
6474 break;
6475 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
6476 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
6477 break;
6478 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
6479 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
6480 break;
6481 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
6482 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
6483 break;
6484 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
6485 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
6486 break;
6487 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
6488 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
6489 break;
6490 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
6491 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
6492 break;
6493 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
6494 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
6495 break;
6496 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
6497 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
6498 break;
6499 default:
6500 break;
6501 }
6502 return 0;
6503 }
6504
gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6505 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev,
6506 struct amdgpu_irq_src *source,
6507 unsigned int type,
6508 enum amdgpu_interrupt_state state)
6509 {
6510 int enable_flag;
6511
6512 switch (state) {
6513 case AMDGPU_IRQ_STATE_DISABLE:
6514 enable_flag = 0;
6515 break;
6516
6517 case AMDGPU_IRQ_STATE_ENABLE:
6518 enable_flag = 1;
6519 break;
6520
6521 default:
6522 return -EINVAL;
6523 }
6524
6525 WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6526 WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6527 WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6528 WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6529 WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag);
6530 WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6531 enable_flag);
6532 WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6533 enable_flag);
6534 WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6535 enable_flag);
6536 WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6537 enable_flag);
6538 WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6539 enable_flag);
6540 WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6541 enable_flag);
6542 WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6543 enable_flag);
6544 WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE,
6545 enable_flag);
6546
6547 return 0;
6548 }
6549
gfx_v8_0_set_sq_int_state(struct amdgpu_device * adev,struct amdgpu_irq_src * source,unsigned int type,enum amdgpu_interrupt_state state)6550 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev,
6551 struct amdgpu_irq_src *source,
6552 unsigned int type,
6553 enum amdgpu_interrupt_state state)
6554 {
6555 int enable_flag;
6556
6557 switch (state) {
6558 case AMDGPU_IRQ_STATE_DISABLE:
6559 enable_flag = 1;
6560 break;
6561
6562 case AMDGPU_IRQ_STATE_ENABLE:
6563 enable_flag = 0;
6564 break;
6565
6566 default:
6567 return -EINVAL;
6568 }
6569
6570 WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL,
6571 enable_flag);
6572
6573 return 0;
6574 }
6575
gfx_v8_0_eop_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6576 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev,
6577 struct amdgpu_irq_src *source,
6578 struct amdgpu_iv_entry *entry)
6579 {
6580 int i;
6581 u8 me_id, pipe_id, queue_id;
6582 struct amdgpu_ring *ring;
6583
6584 DRM_DEBUG("IH: CP EOP\n");
6585 me_id = (entry->ring_id & 0x0c) >> 2;
6586 pipe_id = (entry->ring_id & 0x03) >> 0;
6587 queue_id = (entry->ring_id & 0x70) >> 4;
6588
6589 switch (me_id) {
6590 case 0:
6591 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
6592 break;
6593 case 1:
6594 case 2:
6595 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6596 ring = &adev->gfx.compute_ring[i];
6597 /* Per-queue interrupt is supported for MEC starting from VI.
6598 * The interrupt can only be enabled/disabled per pipe instead of per queue.
6599 */
6600 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
6601 amdgpu_fence_process(ring);
6602 }
6603 break;
6604 }
6605 return 0;
6606 }
6607
gfx_v8_0_fault(struct amdgpu_device * adev,struct amdgpu_iv_entry * entry)6608 static void gfx_v8_0_fault(struct amdgpu_device *adev,
6609 struct amdgpu_iv_entry *entry)
6610 {
6611 u8 me_id, pipe_id, queue_id;
6612 struct amdgpu_ring *ring;
6613 int i;
6614
6615 me_id = (entry->ring_id & 0x0c) >> 2;
6616 pipe_id = (entry->ring_id & 0x03) >> 0;
6617 queue_id = (entry->ring_id & 0x70) >> 4;
6618
6619 switch (me_id) {
6620 case 0:
6621 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
6622 break;
6623 case 1:
6624 case 2:
6625 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
6626 ring = &adev->gfx.compute_ring[i];
6627 if (ring->me == me_id && ring->pipe == pipe_id &&
6628 ring->queue == queue_id)
6629 drm_sched_fault(&ring->sched);
6630 }
6631 break;
6632 }
6633 }
6634
gfx_v8_0_priv_reg_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6635 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev,
6636 struct amdgpu_irq_src *source,
6637 struct amdgpu_iv_entry *entry)
6638 {
6639 DRM_ERROR("Illegal register access in command stream\n");
6640 gfx_v8_0_fault(adev, entry);
6641 return 0;
6642 }
6643
gfx_v8_0_priv_inst_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6644 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev,
6645 struct amdgpu_irq_src *source,
6646 struct amdgpu_iv_entry *entry)
6647 {
6648 DRM_ERROR("Illegal instruction in command stream\n");
6649 gfx_v8_0_fault(adev, entry);
6650 return 0;
6651 }
6652
gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6653 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev,
6654 struct amdgpu_irq_src *source,
6655 struct amdgpu_iv_entry *entry)
6656 {
6657 DRM_ERROR("CP EDC/ECC error detected.");
6658 return 0;
6659 }
6660
gfx_v8_0_parse_sq_irq(struct amdgpu_device * adev,unsigned ih_data,bool from_wq)6661 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data,
6662 bool from_wq)
6663 {
6664 u32 enc, se_id, sh_id, cu_id;
6665 char type[20];
6666 int sq_edc_source = -1;
6667
6668 enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING);
6669 se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID);
6670
6671 switch (enc) {
6672 case 0:
6673 drm_info(adev_to_drm(adev), "SQ general purpose intr detected:"
6674 "se_id %d, immed_overflow %d, host_reg_overflow %d,"
6675 "host_cmd_overflow %d, cmd_timestamp %d,"
6676 "reg_timestamp %d, thread_trace_buff_full %d,"
6677 "wlt %d, thread_trace %d.\n",
6678 se_id,
6679 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW),
6680 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW),
6681 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW),
6682 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP),
6683 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP),
6684 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL),
6685 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT),
6686 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE)
6687 );
6688 break;
6689 case 1:
6690 case 2:
6691
6692 cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID);
6693 sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID);
6694
6695 /*
6696 * This function can be called either directly from ISR
6697 * or from BH in which case we can access SQ_EDC_INFO
6698 * instance
6699 */
6700 if (from_wq) {
6701 mutex_lock(&adev->grbm_idx_mutex);
6702 gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id, 0);
6703
6704 sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE);
6705
6706 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
6707 mutex_unlock(&adev->grbm_idx_mutex);
6708 }
6709
6710 if (enc == 1)
6711 sprintf(type, "instruction intr");
6712 else
6713 sprintf(type, "EDC/ECC error");
6714
6715 drm_info(adev_to_drm(adev),
6716 "SQ %s detected: "
6717 "se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d "
6718 "trap %s, sq_ed_info.source %s.\n",
6719 type, se_id, sh_id, cu_id,
6720 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID),
6721 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID),
6722 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID),
6723 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false",
6724 (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable"
6725 );
6726 break;
6727 default:
6728 DRM_ERROR("SQ invalid encoding type\n.");
6729 }
6730 }
6731
gfx_v8_0_sq_irq_work_func(struct work_struct * work)6732 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work)
6733 {
6734
6735 struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work);
6736 struct sq_work *sq_work = container_of(work, struct sq_work, work);
6737
6738 gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true);
6739 }
6740
gfx_v8_0_sq_irq(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)6741 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev,
6742 struct amdgpu_irq_src *source,
6743 struct amdgpu_iv_entry *entry)
6744 {
6745 unsigned ih_data = entry->src_data[0];
6746
6747 /*
6748 * Try to submit work so SQ_EDC_INFO can be accessed from
6749 * BH. If previous work submission hasn't finished yet
6750 * just print whatever info is possible directly from the ISR.
6751 */
6752 if (work_pending(&adev->gfx.sq_work.work)) {
6753 gfx_v8_0_parse_sq_irq(adev, ih_data, false);
6754 } else {
6755 adev->gfx.sq_work.ih_data = ih_data;
6756 schedule_work(&adev->gfx.sq_work.work);
6757 }
6758
6759 return 0;
6760 }
6761
gfx_v8_0_emit_mem_sync(struct amdgpu_ring * ring)6762 static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring)
6763 {
6764 amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
6765 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
6766 PACKET3_TC_ACTION_ENA |
6767 PACKET3_SH_KCACHE_ACTION_ENA |
6768 PACKET3_SH_ICACHE_ACTION_ENA |
6769 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */
6770 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
6771 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6772 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */
6773 }
6774
gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring * ring)6775 static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring)
6776 {
6777 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5));
6778 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
6779 PACKET3_TC_ACTION_ENA |
6780 PACKET3_SH_KCACHE_ACTION_ENA |
6781 PACKET3_SH_ICACHE_ACTION_ENA |
6782 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */
6783 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */
6784 amdgpu_ring_write(ring, 0xff); /* CP_COHER_SIZE_HI */
6785 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */
6786 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */
6787 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */
6788 }
6789
6790
6791 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */
6792 #define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT 0x0000007f
gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring * ring,uint32_t pipe,bool enable)6793 static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring,
6794 uint32_t pipe, bool enable)
6795 {
6796 uint32_t val;
6797 uint32_t wcl_cs_reg;
6798
6799 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT;
6800
6801 switch (pipe) {
6802 case 0:
6803 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0;
6804 break;
6805 case 1:
6806 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1;
6807 break;
6808 case 2:
6809 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2;
6810 break;
6811 case 3:
6812 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3;
6813 break;
6814 default:
6815 DRM_DEBUG("invalid pipe %d\n", pipe);
6816 return;
6817 }
6818
6819 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val);
6820
6821 }
6822
6823 #define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT 0x07ffffff
gfx_v8_0_emit_wave_limit(struct amdgpu_ring * ring,bool enable)6824 static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable)
6825 {
6826 struct amdgpu_device *adev = ring->adev;
6827 uint32_t val;
6828 int i;
6829
6830 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit
6831 * number of gfx waves. Setting 5 bit will make sure gfx only gets
6832 * around 25% of gpu resources.
6833 */
6834 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT;
6835 amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val);
6836
6837 /* Restrict waves for normal/low priority compute queues as well
6838 * to get best QoS for high priority compute jobs.
6839 *
6840 * amdgpu controls only 1st ME(0-3 CS pipes).
6841 */
6842 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) {
6843 if (i != ring->pipe)
6844 gfx_v8_0_emit_wave_limit_cs(ring, i, enable);
6845
6846 }
6847
6848 }
6849
6850 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = {
6851 .name = "gfx_v8_0",
6852 .early_init = gfx_v8_0_early_init,
6853 .late_init = gfx_v8_0_late_init,
6854 .sw_init = gfx_v8_0_sw_init,
6855 .sw_fini = gfx_v8_0_sw_fini,
6856 .hw_init = gfx_v8_0_hw_init,
6857 .hw_fini = gfx_v8_0_hw_fini,
6858 .suspend = gfx_v8_0_suspend,
6859 .resume = gfx_v8_0_resume,
6860 .is_idle = gfx_v8_0_is_idle,
6861 .wait_for_idle = gfx_v8_0_wait_for_idle,
6862 .check_soft_reset = gfx_v8_0_check_soft_reset,
6863 .pre_soft_reset = gfx_v8_0_pre_soft_reset,
6864 .soft_reset = gfx_v8_0_soft_reset,
6865 .post_soft_reset = gfx_v8_0_post_soft_reset,
6866 .set_clockgating_state = gfx_v8_0_set_clockgating_state,
6867 .set_powergating_state = gfx_v8_0_set_powergating_state,
6868 .get_clockgating_state = gfx_v8_0_get_clockgating_state,
6869 };
6870
6871 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = {
6872 .type = AMDGPU_RING_TYPE_GFX,
6873 .align_mask = 0xff,
6874 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6875 .support_64bit_ptrs = false,
6876 .get_rptr = gfx_v8_0_ring_get_rptr,
6877 .get_wptr = gfx_v8_0_ring_get_wptr_gfx,
6878 .set_wptr = gfx_v8_0_ring_set_wptr_gfx,
6879 .emit_frame_size = /* maximum 215dw if count 16 IBs in */
6880 5 + /* COND_EXEC */
6881 7 + /* PIPELINE_SYNC */
6882 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */
6883 12 + /* FENCE for VM_FLUSH */
6884 20 + /* GDS switch */
6885 4 + /* double SWITCH_BUFFER,
6886 the first COND_EXEC jump to the place just
6887 prior to this double SWITCH_BUFFER */
6888 5 + /* COND_EXEC */
6889 7 + /* HDP_flush */
6890 4 + /* VGT_flush */
6891 14 + /* CE_META */
6892 31 + /* DE_META */
6893 3 + /* CNTX_CTRL */
6894 5 + /* HDP_INVL */
6895 12 + 12 + /* FENCE x2 */
6896 2 + /* SWITCH_BUFFER */
6897 5, /* SURFACE_SYNC */
6898 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */
6899 .emit_ib = gfx_v8_0_ring_emit_ib_gfx,
6900 .emit_fence = gfx_v8_0_ring_emit_fence_gfx,
6901 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6902 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6903 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6904 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6905 .test_ring = gfx_v8_0_ring_test_ring,
6906 .test_ib = gfx_v8_0_ring_test_ib,
6907 .insert_nop = amdgpu_ring_insert_nop,
6908 .pad_ib = amdgpu_ring_generic_pad_ib,
6909 .emit_switch_buffer = gfx_v8_ring_emit_sb,
6910 .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl,
6911 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec,
6912 .emit_wreg = gfx_v8_0_ring_emit_wreg,
6913 .soft_recovery = gfx_v8_0_ring_soft_recovery,
6914 .emit_mem_sync = gfx_v8_0_emit_mem_sync,
6915 };
6916
6917 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = {
6918 .type = AMDGPU_RING_TYPE_COMPUTE,
6919 .align_mask = 0xff,
6920 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6921 .support_64bit_ptrs = false,
6922 .get_rptr = gfx_v8_0_ring_get_rptr,
6923 .get_wptr = gfx_v8_0_ring_get_wptr_compute,
6924 .set_wptr = gfx_v8_0_ring_set_wptr_compute,
6925 .emit_frame_size =
6926 20 + /* gfx_v8_0_ring_emit_gds_switch */
6927 7 + /* gfx_v8_0_ring_emit_hdp_flush */
6928 5 + /* hdp_invalidate */
6929 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6930 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */
6931 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */
6932 7 + /* gfx_v8_0_emit_mem_sync_compute */
6933 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */
6934 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */
6935 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */
6936 .emit_ib = gfx_v8_0_ring_emit_ib_compute,
6937 .emit_fence = gfx_v8_0_ring_emit_fence_compute,
6938 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync,
6939 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush,
6940 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch,
6941 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6942 .test_ring = gfx_v8_0_ring_test_ring,
6943 .test_ib = gfx_v8_0_ring_test_ib,
6944 .insert_nop = amdgpu_ring_insert_nop,
6945 .pad_ib = amdgpu_ring_generic_pad_ib,
6946 .emit_wreg = gfx_v8_0_ring_emit_wreg,
6947 .soft_recovery = gfx_v8_0_ring_soft_recovery,
6948 .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute,
6949 .emit_wave_limit = gfx_v8_0_emit_wave_limit,
6950 };
6951
6952 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = {
6953 .type = AMDGPU_RING_TYPE_KIQ,
6954 .align_mask = 0xff,
6955 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6956 .support_64bit_ptrs = false,
6957 .get_rptr = gfx_v8_0_ring_get_rptr,
6958 .get_wptr = gfx_v8_0_ring_get_wptr_compute,
6959 .set_wptr = gfx_v8_0_ring_set_wptr_compute,
6960 .emit_frame_size =
6961 20 + /* gfx_v8_0_ring_emit_gds_switch */
6962 7 + /* gfx_v8_0_ring_emit_hdp_flush */
6963 5 + /* hdp_invalidate */
6964 7 + /* gfx_v8_0_ring_emit_pipeline_sync */
6965 17 + /* gfx_v8_0_ring_emit_vm_flush */
6966 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6967 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */
6968 .emit_fence = gfx_v8_0_ring_emit_fence_kiq,
6969 .test_ring = gfx_v8_0_ring_test_ring,
6970 .insert_nop = amdgpu_ring_insert_nop,
6971 .pad_ib = amdgpu_ring_generic_pad_ib,
6972 .emit_rreg = gfx_v8_0_ring_emit_rreg,
6973 .emit_wreg = gfx_v8_0_ring_emit_wreg,
6974 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush,
6975 };
6976
gfx_v8_0_set_ring_funcs(struct amdgpu_device * adev)6977 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev)
6978 {
6979 int i;
6980
6981 adev->gfx.kiq[0].ring.funcs = &gfx_v8_0_ring_funcs_kiq;
6982
6983 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6984 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx;
6985
6986 for (i = 0; i < adev->gfx.num_compute_rings; i++)
6987 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute;
6988 }
6989
6990 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = {
6991 .set = gfx_v8_0_set_eop_interrupt_state,
6992 .process = gfx_v8_0_eop_irq,
6993 };
6994
6995 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = {
6996 .set = gfx_v8_0_set_priv_reg_fault_state,
6997 .process = gfx_v8_0_priv_reg_irq,
6998 };
6999
7000 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = {
7001 .set = gfx_v8_0_set_priv_inst_fault_state,
7002 .process = gfx_v8_0_priv_inst_irq,
7003 };
7004
7005 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = {
7006 .set = gfx_v8_0_set_cp_ecc_int_state,
7007 .process = gfx_v8_0_cp_ecc_error_irq,
7008 };
7009
7010 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = {
7011 .set = gfx_v8_0_set_sq_int_state,
7012 .process = gfx_v8_0_sq_irq,
7013 };
7014
gfx_v8_0_set_irq_funcs(struct amdgpu_device * adev)7015 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev)
7016 {
7017 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
7018 adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs;
7019
7020 adev->gfx.priv_reg_irq.num_types = 1;
7021 adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs;
7022
7023 adev->gfx.priv_inst_irq.num_types = 1;
7024 adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs;
7025
7026 adev->gfx.cp_ecc_error_irq.num_types = 1;
7027 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs;
7028
7029 adev->gfx.sq_irq.num_types = 1;
7030 adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs;
7031 }
7032
gfx_v8_0_set_rlc_funcs(struct amdgpu_device * adev)7033 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
7034 {
7035 adev->gfx.rlc.funcs = &iceland_rlc_funcs;
7036 }
7037
gfx_v8_0_set_gds_init(struct amdgpu_device * adev)7038 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
7039 {
7040 /* init asci gds info */
7041 adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE);
7042 adev->gds.gws_size = 64;
7043 adev->gds.oa_size = 16;
7044 adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID);
7045 }
7046
gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device * adev,u32 bitmap)7047 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
7048 u32 bitmap)
7049 {
7050 u32 data;
7051
7052 if (!bitmap)
7053 return;
7054
7055 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
7056 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
7057
7058 WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
7059 }
7060
gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device * adev)7061 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
7062 {
7063 u32 data, mask;
7064
7065 data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) |
7066 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG);
7067
7068 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
7069
7070 return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask;
7071 }
7072
gfx_v8_0_get_cu_info(struct amdgpu_device * adev)7073 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
7074 {
7075 int i, j, k, counter, active_cu_number = 0;
7076 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
7077 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
7078 unsigned disable_masks[4 * 2];
7079 u32 ao_cu_num;
7080
7081 memset(cu_info, 0, sizeof(*cu_info));
7082
7083 if (adev->flags & AMD_IS_APU)
7084 ao_cu_num = 2;
7085 else
7086 ao_cu_num = adev->gfx.config.max_cu_per_sh;
7087
7088 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 4, 2);
7089
7090 mutex_lock(&adev->grbm_idx_mutex);
7091 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
7092 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
7093 mask = 1;
7094 ao_bitmap = 0;
7095 counter = 0;
7096 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0);
7097 if (i < 4 && j < 2)
7098 gfx_v8_0_set_user_cu_inactive_bitmap(
7099 adev, disable_masks[i * 2 + j]);
7100 bitmap = gfx_v8_0_get_cu_active_bitmap(adev);
7101 cu_info->bitmap[0][i][j] = bitmap;
7102
7103 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
7104 if (bitmap & mask) {
7105 if (counter < ao_cu_num)
7106 ao_bitmap |= mask;
7107 counter ++;
7108 }
7109 mask <<= 1;
7110 }
7111 active_cu_number += counter;
7112 if (i < 2 && j < 2)
7113 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
7114 cu_info->ao_cu_bitmap[i][j] = ao_bitmap;
7115 }
7116 }
7117 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
7118 mutex_unlock(&adev->grbm_idx_mutex);
7119
7120 cu_info->number = active_cu_number;
7121 cu_info->ao_cu_mask = ao_cu_mask;
7122 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
7123 cu_info->max_waves_per_simd = 10;
7124 cu_info->max_scratch_slots_per_cu = 32;
7125 cu_info->wave_front_size = 64;
7126 cu_info->lds_size = 64;
7127 }
7128
7129 const struct amdgpu_ip_block_version gfx_v8_0_ip_block =
7130 {
7131 .type = AMD_IP_BLOCK_TYPE_GFX,
7132 .major = 8,
7133 .minor = 0,
7134 .rev = 0,
7135 .funcs = &gfx_v8_0_ip_funcs,
7136 };
7137
7138 const struct amdgpu_ip_block_version gfx_v8_1_ip_block =
7139 {
7140 .type = AMD_IP_BLOCK_TYPE_GFX,
7141 .major = 8,
7142 .minor = 1,
7143 .rev = 0,
7144 .funcs = &gfx_v8_0_ip_funcs,
7145 };
7146
gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring * ring)7147 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
7148 {
7149 uint64_t ce_payload_addr;
7150 int cnt_ce;
7151 union {
7152 struct vi_ce_ib_state regular;
7153 struct vi_ce_ib_state_chained_ib chained;
7154 } ce_payload = {};
7155
7156 if (ring->adev->virt.chained_ib_support) {
7157 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7158 offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
7159 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
7160 } else {
7161 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
7162 offsetof(struct vi_gfx_meta_data, ce_payload);
7163 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
7164 }
7165
7166 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce));
7167 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
7168 WRITE_DATA_DST_SEL(8) |
7169 WR_CONFIRM) |
7170 WRITE_DATA_CACHE_POLICY(0));
7171 amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr));
7172 amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr));
7173 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
7174 }
7175
gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring * ring)7176 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
7177 {
7178 uint64_t de_payload_addr, gds_addr, csa_addr;
7179 int cnt_de;
7180 union {
7181 struct vi_de_ib_state regular;
7182 struct vi_de_ib_state_chained_ib chained;
7183 } de_payload = {};
7184
7185 csa_addr = amdgpu_csa_vaddr(ring->adev);
7186 gds_addr = csa_addr + 4096;
7187 if (ring->adev->virt.chained_ib_support) {
7188 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
7189 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr);
7190 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload);
7191 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2;
7192 } else {
7193 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr);
7194 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr);
7195 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload);
7196 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2;
7197 }
7198
7199 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de));
7200 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
7201 WRITE_DATA_DST_SEL(8) |
7202 WR_CONFIRM) |
7203 WRITE_DATA_CACHE_POLICY(0));
7204 amdgpu_ring_write(ring, lower_32_bits(de_payload_addr));
7205 amdgpu_ring_write(ring, upper_32_bits(de_payload_addr));
7206 amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2);
7207 }
7208