1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "vi.h" 33 #include "vi_structs.h" 34 #include "vid.h" 35 #include "amdgpu_ucode.h" 36 #include "amdgpu_atombios.h" 37 #include "atombios_i2c.h" 38 #include "clearstate_vi.h" 39 40 #include "gmc/gmc_8_2_d.h" 41 #include "gmc/gmc_8_2_sh_mask.h" 42 43 #include "oss/oss_3_0_d.h" 44 #include "oss/oss_3_0_sh_mask.h" 45 46 #include "bif/bif_5_0_d.h" 47 #include "bif/bif_5_0_sh_mask.h" 48 #include "gca/gfx_8_0_d.h" 49 #include "gca/gfx_8_0_enum.h" 50 #include "gca/gfx_8_0_sh_mask.h" 51 52 #include "dce/dce_10_0_d.h" 53 #include "dce/dce_10_0_sh_mask.h" 54 55 #include "smu/smu_7_1_3_d.h" 56 57 #include "ivsrcid/ivsrcid_vislands30.h" 58 59 #define GFX8_NUM_GFX_RINGS 1 60 #define GFX8_MEC_HPD_SIZE 4096 61 62 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001 63 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001 64 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002 65 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003 66 67 #define ARRAY_MODE(x) ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT) 68 #define PIPE_CONFIG(x) ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT) 69 #define TILE_SPLIT(x) ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT) 70 #define MICRO_TILE_MODE_NEW(x) ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT) 71 #define SAMPLE_SPLIT(x) ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT) 72 #define BANK_WIDTH(x) ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT) 73 #define BANK_HEIGHT(x) ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT) 74 #define MACRO_TILE_ASPECT(x) ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT) 75 #define NUM_BANKS(x) ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT) 76 77 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK 0x00000001L 78 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK 0x00000002L 79 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK 0x00000004L 80 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK 0x00000008L 81 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK 0x00000010L 82 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK 0x00000020L 83 84 /* BPM SERDES CMD */ 85 #define SET_BPM_SERDES_CMD 1 86 #define CLE_BPM_SERDES_CMD 0 87 88 /* BPM Register Address*/ 89 enum { 90 BPM_REG_CGLS_EN = 0, /* Enable/Disable CGLS */ 91 BPM_REG_CGLS_ON, /* ON/OFF CGLS: shall be controlled by RLC FW */ 92 BPM_REG_CGCG_OVERRIDE, /* Set/Clear CGCG Override */ 93 BPM_REG_MGCG_OVERRIDE, /* Set/Clear MGCG Override */ 94 BPM_REG_FGCG_OVERRIDE, /* Set/Clear FGCG Override */ 95 BPM_REG_FGCG_MAX 96 }; 97 98 #define RLC_FormatDirectRegListLength 14 99 100 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin"); 101 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin"); 102 MODULE_FIRMWARE("amdgpu/carrizo_me.bin"); 103 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin"); 104 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin"); 105 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin"); 106 107 MODULE_FIRMWARE("amdgpu/stoney_ce.bin"); 108 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin"); 109 MODULE_FIRMWARE("amdgpu/stoney_me.bin"); 110 MODULE_FIRMWARE("amdgpu/stoney_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/tonga_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/tonga_me.bin"); 116 MODULE_FIRMWARE("amdgpu/tonga_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin"); 118 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin"); 119 120 MODULE_FIRMWARE("amdgpu/topaz_ce.bin"); 121 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin"); 122 MODULE_FIRMWARE("amdgpu/topaz_me.bin"); 123 MODULE_FIRMWARE("amdgpu/topaz_mec.bin"); 124 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin"); 125 126 MODULE_FIRMWARE("amdgpu/fiji_ce.bin"); 127 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin"); 128 MODULE_FIRMWARE("amdgpu/fiji_me.bin"); 129 MODULE_FIRMWARE("amdgpu/fiji_mec.bin"); 130 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin"); 131 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin"); 132 133 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin"); 134 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin"); 135 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin"); 136 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin"); 137 MODULE_FIRMWARE("amdgpu/polaris10_me.bin"); 138 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin"); 139 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin"); 140 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin"); 141 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin"); 142 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin"); 143 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin"); 144 145 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin"); 146 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin"); 147 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin"); 148 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin"); 149 MODULE_FIRMWARE("amdgpu/polaris11_me.bin"); 150 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin"); 151 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin"); 152 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin"); 153 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin"); 154 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin"); 155 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin"); 156 157 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin"); 158 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin"); 159 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin"); 160 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin"); 161 MODULE_FIRMWARE("amdgpu/polaris12_me.bin"); 162 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin"); 163 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin"); 164 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin"); 165 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin"); 166 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin"); 167 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin"); 168 169 MODULE_FIRMWARE("amdgpu/vegam_ce.bin"); 170 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin"); 171 MODULE_FIRMWARE("amdgpu/vegam_me.bin"); 172 MODULE_FIRMWARE("amdgpu/vegam_mec.bin"); 173 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin"); 174 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin"); 175 176 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] = 177 { 178 {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0}, 179 {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1}, 180 {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2}, 181 {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3}, 182 {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4}, 183 {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5}, 184 {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6}, 185 {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7}, 186 {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8}, 187 {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9}, 188 {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10}, 189 {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11}, 190 {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12}, 191 {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13}, 192 {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14}, 193 {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15} 194 }; 195 196 static const u32 golden_settings_tonga_a11[] = 197 { 198 mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208, 199 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 200 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 201 mmGB_GPU_ID, 0x0000000f, 0x00000000, 202 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 203 mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc, 204 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 205 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 206 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 207 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 208 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 209 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 210 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb, 211 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b, 212 mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876, 213 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 214 }; 215 216 static const u32 tonga_golden_common_all[] = 217 { 218 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 219 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 220 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 221 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 222 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 223 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 224 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 225 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 226 }; 227 228 static const u32 tonga_mgcg_cgcg_init[] = 229 { 230 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 231 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 232 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 233 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 234 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 235 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 236 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 237 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 238 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 239 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 240 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 241 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 242 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 243 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 244 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 245 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 246 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 247 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 248 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 249 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 250 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 251 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 252 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 253 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 254 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 255 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 256 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 257 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 258 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 259 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 260 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 261 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 262 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 263 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 264 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 265 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 266 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 267 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 268 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 269 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 270 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 271 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 272 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 273 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 274 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 275 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 276 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 277 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 278 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 279 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 280 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 281 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 282 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 283 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 284 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 285 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 286 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 287 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 288 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 289 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 290 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 291 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 292 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 293 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 294 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 295 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 296 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 297 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 298 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 299 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 300 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 301 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 302 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 303 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 304 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 305 }; 306 307 static const u32 golden_settings_vegam_a11[] = 308 { 309 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 310 mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000, 311 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 312 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 313 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 314 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 315 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a, 316 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e, 317 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 318 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 319 mmSQ_CONFIG, 0x07f80000, 0x01180000, 320 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 321 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 322 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 323 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 324 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054, 325 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 326 }; 327 328 static const u32 vegam_golden_common_all[] = 329 { 330 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 331 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 332 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 333 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 334 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 335 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 336 }; 337 338 static const u32 golden_settings_polaris11_a11[] = 339 { 340 mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208, 341 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 342 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 343 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 344 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 345 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 346 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 347 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 348 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 349 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 350 mmSQ_CONFIG, 0x07f80000, 0x01180000, 351 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 352 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 353 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3, 354 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 355 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210, 356 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 357 }; 358 359 static const u32 polaris11_golden_common_all[] = 360 { 361 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 362 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002, 363 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 364 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 365 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 366 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 367 }; 368 369 static const u32 golden_settings_polaris10_a11[] = 370 { 371 mmATC_MISC_CG, 0x000c0fc0, 0x000c0200, 372 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 373 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 374 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 375 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 376 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 377 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 378 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 379 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a, 380 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 381 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 382 mmSQ_CONFIG, 0x07f80000, 0x07180000, 383 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 384 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 385 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 386 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 387 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 388 }; 389 390 static const u32 polaris10_golden_common_all[] = 391 { 392 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 393 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 394 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 395 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 396 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 397 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 398 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 399 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 400 }; 401 402 static const u32 fiji_golden_common_all[] = 403 { 404 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 405 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a, 406 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e, 407 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 408 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 409 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 410 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 411 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 412 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 413 mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, 414 }; 415 416 static const u32 golden_settings_fiji_a10[] = 417 { 418 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 419 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 420 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 421 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 422 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 423 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 424 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 425 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 426 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 427 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff, 428 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 429 }; 430 431 static const u32 fiji_mgcg_cgcg_init[] = 432 { 433 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 434 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 435 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 436 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 437 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 438 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 439 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 440 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 441 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 442 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 443 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 444 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 445 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 446 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 447 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 448 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 449 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 450 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 451 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 452 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 453 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 454 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 455 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 456 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 457 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 458 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 459 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 460 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 461 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 462 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 463 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 464 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 465 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 466 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 467 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 468 }; 469 470 static const u32 golden_settings_iceland_a11[] = 471 { 472 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 473 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 474 mmDB_DEBUG3, 0xc0000000, 0xc0000000, 475 mmGB_GPU_ID, 0x0000000f, 0x00000000, 476 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 477 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 478 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002, 479 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 480 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 481 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 482 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 483 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 484 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 485 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1, 486 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 487 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010, 488 }; 489 490 static const u32 iceland_golden_common_all[] = 491 { 492 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 493 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 494 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 495 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 496 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 497 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 498 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 499 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 500 }; 501 502 static const u32 iceland_mgcg_cgcg_init[] = 503 { 504 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 505 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 506 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 507 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 508 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100, 509 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100, 510 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100, 511 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 512 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 513 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 514 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 515 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 516 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 517 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 518 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 519 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 520 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 521 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 522 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 523 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 524 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 525 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 526 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100, 527 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 528 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 529 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 530 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 531 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 532 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 533 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 534 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 535 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 536 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 537 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 538 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 539 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 540 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 541 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 542 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 543 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 544 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 545 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 546 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 547 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 548 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 549 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 550 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 551 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 552 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 553 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 554 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 555 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 556 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 557 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 558 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 559 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 560 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 561 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 562 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 563 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 564 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 565 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 566 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 567 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 568 }; 569 570 static const u32 cz_golden_settings_a11[] = 571 { 572 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 573 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 574 mmGB_GPU_ID, 0x0000000f, 0x00000000, 575 mmPA_SC_ENHANCE, 0xffffffff, 0x00000001, 576 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 577 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 578 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 579 mmTA_CNTL_AUX, 0x000f000f, 0x00010000, 580 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 581 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 582 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3, 583 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302 584 }; 585 586 static const u32 cz_golden_common_all[] = 587 { 588 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 589 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 590 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 591 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 592 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 593 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 594 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 595 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 596 }; 597 598 static const u32 cz_mgcg_cgcg_init[] = 599 { 600 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 601 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 602 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 603 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 604 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 605 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 606 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100, 607 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 608 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 609 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 610 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 611 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 612 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 613 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 614 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 615 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 616 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 617 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 618 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 619 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 620 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 621 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 622 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 623 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 624 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 625 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 626 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 627 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 628 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 629 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 630 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 631 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 632 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 633 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 634 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 635 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 636 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 637 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 638 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 639 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 640 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 641 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 642 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 643 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 644 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 645 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 646 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 647 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 648 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 649 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 650 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 651 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 652 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 653 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 654 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 655 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 656 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 657 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 658 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 659 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 660 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 661 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 662 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 663 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 664 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 665 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 666 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 667 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 668 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 669 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 670 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 671 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 672 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 673 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 674 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 675 }; 676 677 static const u32 stoney_golden_settings_a11[] = 678 { 679 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 680 mmGB_GPU_ID, 0x0000000f, 0x00000000, 681 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 682 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 683 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 684 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 685 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 686 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 687 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1, 688 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010, 689 }; 690 691 static const u32 stoney_golden_common_all[] = 692 { 693 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 694 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000, 695 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 696 mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001, 697 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 698 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 699 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 700 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 701 }; 702 703 static const u32 stoney_mgcg_cgcg_init[] = 704 { 705 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 706 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 707 mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 708 mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 709 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200, 710 }; 711 712 713 static const char * const sq_edc_source_names[] = { 714 "SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred", 715 "SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch", 716 "SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return", 717 "SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR", 718 "SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS", 719 "SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS", 720 "SQ_EDC_INFO_SOURCE_TA: EDC source is TA", 721 }; 722 723 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev); 724 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev); 725 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev); 726 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev); 727 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev); 728 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev); 729 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring); 730 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring); 731 732 #define CG_ACLK_CNTL__ACLK_DIVIDER_MASK 0x0000007fL 733 #define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT 0x00000000L 734 735 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev) 736 { 737 uint32_t data; 738 739 switch (adev->asic_type) { 740 case CHIP_TOPAZ: 741 amdgpu_device_program_register_sequence(adev, 742 iceland_mgcg_cgcg_init, 743 ARRAY_SIZE(iceland_mgcg_cgcg_init)); 744 amdgpu_device_program_register_sequence(adev, 745 golden_settings_iceland_a11, 746 ARRAY_SIZE(golden_settings_iceland_a11)); 747 amdgpu_device_program_register_sequence(adev, 748 iceland_golden_common_all, 749 ARRAY_SIZE(iceland_golden_common_all)); 750 break; 751 case CHIP_FIJI: 752 amdgpu_device_program_register_sequence(adev, 753 fiji_mgcg_cgcg_init, 754 ARRAY_SIZE(fiji_mgcg_cgcg_init)); 755 amdgpu_device_program_register_sequence(adev, 756 golden_settings_fiji_a10, 757 ARRAY_SIZE(golden_settings_fiji_a10)); 758 amdgpu_device_program_register_sequence(adev, 759 fiji_golden_common_all, 760 ARRAY_SIZE(fiji_golden_common_all)); 761 break; 762 763 case CHIP_TONGA: 764 amdgpu_device_program_register_sequence(adev, 765 tonga_mgcg_cgcg_init, 766 ARRAY_SIZE(tonga_mgcg_cgcg_init)); 767 amdgpu_device_program_register_sequence(adev, 768 golden_settings_tonga_a11, 769 ARRAY_SIZE(golden_settings_tonga_a11)); 770 amdgpu_device_program_register_sequence(adev, 771 tonga_golden_common_all, 772 ARRAY_SIZE(tonga_golden_common_all)); 773 break; 774 case CHIP_VEGAM: 775 amdgpu_device_program_register_sequence(adev, 776 golden_settings_vegam_a11, 777 ARRAY_SIZE(golden_settings_vegam_a11)); 778 amdgpu_device_program_register_sequence(adev, 779 vegam_golden_common_all, 780 ARRAY_SIZE(vegam_golden_common_all)); 781 break; 782 case CHIP_POLARIS11: 783 case CHIP_POLARIS12: 784 amdgpu_device_program_register_sequence(adev, 785 golden_settings_polaris11_a11, 786 ARRAY_SIZE(golden_settings_polaris11_a11)); 787 amdgpu_device_program_register_sequence(adev, 788 polaris11_golden_common_all, 789 ARRAY_SIZE(polaris11_golden_common_all)); 790 break; 791 case CHIP_POLARIS10: 792 amdgpu_device_program_register_sequence(adev, 793 golden_settings_polaris10_a11, 794 ARRAY_SIZE(golden_settings_polaris10_a11)); 795 amdgpu_device_program_register_sequence(adev, 796 polaris10_golden_common_all, 797 ARRAY_SIZE(polaris10_golden_common_all)); 798 data = RREG32_SMC(ixCG_ACLK_CNTL); 799 data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK; 800 data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT; 801 WREG32_SMC(ixCG_ACLK_CNTL, data); 802 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) && 803 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) || 804 (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) || 805 (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) { 806 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD); 807 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0); 808 } 809 break; 810 case CHIP_CARRIZO: 811 amdgpu_device_program_register_sequence(adev, 812 cz_mgcg_cgcg_init, 813 ARRAY_SIZE(cz_mgcg_cgcg_init)); 814 amdgpu_device_program_register_sequence(adev, 815 cz_golden_settings_a11, 816 ARRAY_SIZE(cz_golden_settings_a11)); 817 amdgpu_device_program_register_sequence(adev, 818 cz_golden_common_all, 819 ARRAY_SIZE(cz_golden_common_all)); 820 break; 821 case CHIP_STONEY: 822 amdgpu_device_program_register_sequence(adev, 823 stoney_mgcg_cgcg_init, 824 ARRAY_SIZE(stoney_mgcg_cgcg_init)); 825 amdgpu_device_program_register_sequence(adev, 826 stoney_golden_settings_a11, 827 ARRAY_SIZE(stoney_golden_settings_a11)); 828 amdgpu_device_program_register_sequence(adev, 829 stoney_golden_common_all, 830 ARRAY_SIZE(stoney_golden_common_all)); 831 break; 832 default: 833 break; 834 } 835 } 836 837 static void gfx_v8_0_scratch_init(struct amdgpu_device *adev) 838 { 839 adev->gfx.scratch.num_reg = 8; 840 adev->gfx.scratch.reg_base = mmSCRATCH_REG0; 841 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 842 } 843 844 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring) 845 { 846 struct amdgpu_device *adev = ring->adev; 847 uint32_t scratch; 848 uint32_t tmp = 0; 849 unsigned i; 850 int r; 851 852 r = amdgpu_gfx_scratch_get(adev, &scratch); 853 if (r) 854 return r; 855 856 WREG32(scratch, 0xCAFEDEAD); 857 r = amdgpu_ring_alloc(ring, 3); 858 if (r) 859 goto error_free_scratch; 860 861 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 862 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 863 amdgpu_ring_write(ring, 0xDEADBEEF); 864 amdgpu_ring_commit(ring); 865 866 for (i = 0; i < adev->usec_timeout; i++) { 867 tmp = RREG32(scratch); 868 if (tmp == 0xDEADBEEF) 869 break; 870 udelay(1); 871 } 872 873 if (i >= adev->usec_timeout) 874 r = -ETIMEDOUT; 875 876 error_free_scratch: 877 amdgpu_gfx_scratch_free(adev, scratch); 878 return r; 879 } 880 881 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 882 { 883 struct amdgpu_device *adev = ring->adev; 884 struct amdgpu_ib ib; 885 struct dma_fence *f = NULL; 886 887 unsigned int index; 888 uint64_t gpu_addr; 889 uint32_t tmp; 890 long r; 891 892 r = amdgpu_device_wb_get(adev, &index); 893 if (r) 894 return r; 895 896 gpu_addr = adev->wb.gpu_addr + (index * 4); 897 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 898 memset(&ib, 0, sizeof(ib)); 899 r = amdgpu_ib_get(adev, NULL, 16, 900 AMDGPU_IB_POOL_DIRECT, &ib); 901 if (r) 902 goto err1; 903 904 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 905 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 906 ib.ptr[2] = lower_32_bits(gpu_addr); 907 ib.ptr[3] = upper_32_bits(gpu_addr); 908 ib.ptr[4] = 0xDEADBEEF; 909 ib.length_dw = 5; 910 911 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 912 if (r) 913 goto err2; 914 915 r = dma_fence_wait_timeout(f, false, timeout); 916 if (r == 0) { 917 r = -ETIMEDOUT; 918 goto err2; 919 } else if (r < 0) { 920 goto err2; 921 } 922 923 tmp = adev->wb.wb[index]; 924 if (tmp == 0xDEADBEEF) 925 r = 0; 926 else 927 r = -EINVAL; 928 929 err2: 930 amdgpu_ib_free(adev, &ib, NULL); 931 dma_fence_put(f); 932 err1: 933 amdgpu_device_wb_free(adev, index); 934 return r; 935 } 936 937 938 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev) 939 { 940 release_firmware(adev->gfx.pfp_fw); 941 adev->gfx.pfp_fw = NULL; 942 release_firmware(adev->gfx.me_fw); 943 adev->gfx.me_fw = NULL; 944 release_firmware(adev->gfx.ce_fw); 945 adev->gfx.ce_fw = NULL; 946 release_firmware(adev->gfx.rlc_fw); 947 adev->gfx.rlc_fw = NULL; 948 release_firmware(adev->gfx.mec_fw); 949 adev->gfx.mec_fw = NULL; 950 if ((adev->asic_type != CHIP_STONEY) && 951 (adev->asic_type != CHIP_TOPAZ)) 952 release_firmware(adev->gfx.mec2_fw); 953 adev->gfx.mec2_fw = NULL; 954 955 kfree(adev->gfx.rlc.register_list_format); 956 } 957 958 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev) 959 { 960 const char *chip_name; 961 char fw_name[30]; 962 int err; 963 struct amdgpu_firmware_info *info = NULL; 964 const struct common_firmware_header *header = NULL; 965 const struct gfx_firmware_header_v1_0 *cp_hdr; 966 const struct rlc_firmware_header_v2_0 *rlc_hdr; 967 unsigned int *tmp = NULL, i; 968 969 DRM_DEBUG("\n"); 970 971 switch (adev->asic_type) { 972 case CHIP_TOPAZ: 973 chip_name = "topaz"; 974 break; 975 case CHIP_TONGA: 976 chip_name = "tonga"; 977 break; 978 case CHIP_CARRIZO: 979 chip_name = "carrizo"; 980 break; 981 case CHIP_FIJI: 982 chip_name = "fiji"; 983 break; 984 case CHIP_STONEY: 985 chip_name = "stoney"; 986 break; 987 case CHIP_POLARIS10: 988 chip_name = "polaris10"; 989 break; 990 case CHIP_POLARIS11: 991 chip_name = "polaris11"; 992 break; 993 case CHIP_POLARIS12: 994 chip_name = "polaris12"; 995 break; 996 case CHIP_VEGAM: 997 chip_name = "vegam"; 998 break; 999 default: 1000 BUG(); 1001 } 1002 1003 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1004 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp_2.bin", chip_name); 1005 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1006 if (err == -ENOENT) { 1007 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1008 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1009 } 1010 } else { 1011 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1012 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1013 } 1014 if (err) 1015 goto out; 1016 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1017 if (err) 1018 goto out; 1019 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1020 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1021 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1022 1023 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1024 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me_2.bin", chip_name); 1025 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1026 if (err == -ENOENT) { 1027 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1028 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1029 } 1030 } else { 1031 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1032 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1033 } 1034 if (err) 1035 goto out; 1036 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1037 if (err) 1038 goto out; 1039 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1040 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1041 1042 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1043 1044 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1045 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce_2.bin", chip_name); 1046 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1047 if (err == -ENOENT) { 1048 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1049 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1050 } 1051 } else { 1052 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1053 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1054 } 1055 if (err) 1056 goto out; 1057 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1058 if (err) 1059 goto out; 1060 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1061 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1062 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1063 1064 /* 1065 * Support for MCBP/Virtualization in combination with chained IBs is 1066 * formal released on feature version #46 1067 */ 1068 if (adev->gfx.ce_feature_version >= 46 && 1069 adev->gfx.pfp_feature_version >= 46) { 1070 adev->virt.chained_ib_support = true; 1071 DRM_INFO("Chained IB support enabled!\n"); 1072 } else 1073 adev->virt.chained_ib_support = false; 1074 1075 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1076 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1077 if (err) 1078 goto out; 1079 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1080 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1081 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1082 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1083 1084 adev->gfx.rlc.save_and_restore_offset = 1085 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1086 adev->gfx.rlc.clear_state_descriptor_offset = 1087 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1088 adev->gfx.rlc.avail_scratch_ram_locations = 1089 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1090 adev->gfx.rlc.reg_restore_list_size = 1091 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1092 adev->gfx.rlc.reg_list_format_start = 1093 le32_to_cpu(rlc_hdr->reg_list_format_start); 1094 adev->gfx.rlc.reg_list_format_separate_start = 1095 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1096 adev->gfx.rlc.starting_offsets_start = 1097 le32_to_cpu(rlc_hdr->starting_offsets_start); 1098 adev->gfx.rlc.reg_list_format_size_bytes = 1099 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1100 adev->gfx.rlc.reg_list_size_bytes = 1101 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1102 1103 adev->gfx.rlc.register_list_format = 1104 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1105 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1106 1107 if (!adev->gfx.rlc.register_list_format) { 1108 err = -ENOMEM; 1109 goto out; 1110 } 1111 1112 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1113 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1114 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1115 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1116 1117 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1118 1119 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1120 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1121 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1122 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1123 1124 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1125 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec_2.bin", chip_name); 1126 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1127 if (err == -ENOENT) { 1128 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1129 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1130 } 1131 } else { 1132 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1133 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1134 } 1135 if (err) 1136 goto out; 1137 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1138 if (err) 1139 goto out; 1140 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1141 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1142 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1143 1144 if ((adev->asic_type != CHIP_STONEY) && 1145 (adev->asic_type != CHIP_TOPAZ)) { 1146 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1147 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2_2.bin", chip_name); 1148 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1149 if (err == -ENOENT) { 1150 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1151 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1152 } 1153 } else { 1154 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1155 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1156 } 1157 if (!err) { 1158 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1159 if (err) 1160 goto out; 1161 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1162 adev->gfx.mec2_fw->data; 1163 adev->gfx.mec2_fw_version = 1164 le32_to_cpu(cp_hdr->header.ucode_version); 1165 adev->gfx.mec2_feature_version = 1166 le32_to_cpu(cp_hdr->ucode_feature_version); 1167 } else { 1168 err = 0; 1169 adev->gfx.mec2_fw = NULL; 1170 } 1171 } 1172 1173 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1174 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1175 info->fw = adev->gfx.pfp_fw; 1176 header = (const struct common_firmware_header *)info->fw->data; 1177 adev->firmware.fw_size += 1178 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1179 1180 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1181 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1182 info->fw = adev->gfx.me_fw; 1183 header = (const struct common_firmware_header *)info->fw->data; 1184 adev->firmware.fw_size += 1185 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1186 1187 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1188 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1189 info->fw = adev->gfx.ce_fw; 1190 header = (const struct common_firmware_header *)info->fw->data; 1191 adev->firmware.fw_size += 1192 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1193 1194 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1195 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1196 info->fw = adev->gfx.rlc_fw; 1197 header = (const struct common_firmware_header *)info->fw->data; 1198 adev->firmware.fw_size += 1199 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1200 1201 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1202 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1203 info->fw = adev->gfx.mec_fw; 1204 header = (const struct common_firmware_header *)info->fw->data; 1205 adev->firmware.fw_size += 1206 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1207 1208 /* we need account JT in */ 1209 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1210 adev->firmware.fw_size += 1211 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE); 1212 1213 if (amdgpu_sriov_vf(adev)) { 1214 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE]; 1215 info->ucode_id = AMDGPU_UCODE_ID_STORAGE; 1216 info->fw = adev->gfx.mec_fw; 1217 adev->firmware.fw_size += 1218 ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE); 1219 } 1220 1221 if (adev->gfx.mec2_fw) { 1222 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1223 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1224 info->fw = adev->gfx.mec2_fw; 1225 header = (const struct common_firmware_header *)info->fw->data; 1226 adev->firmware.fw_size += 1227 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1228 } 1229 1230 out: 1231 if (err) { 1232 dev_err(adev->dev, 1233 "gfx8: Failed to load firmware \"%s\"\n", 1234 fw_name); 1235 release_firmware(adev->gfx.pfp_fw); 1236 adev->gfx.pfp_fw = NULL; 1237 release_firmware(adev->gfx.me_fw); 1238 adev->gfx.me_fw = NULL; 1239 release_firmware(adev->gfx.ce_fw); 1240 adev->gfx.ce_fw = NULL; 1241 release_firmware(adev->gfx.rlc_fw); 1242 adev->gfx.rlc_fw = NULL; 1243 release_firmware(adev->gfx.mec_fw); 1244 adev->gfx.mec_fw = NULL; 1245 release_firmware(adev->gfx.mec2_fw); 1246 adev->gfx.mec2_fw = NULL; 1247 } 1248 return err; 1249 } 1250 1251 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev, 1252 volatile u32 *buffer) 1253 { 1254 u32 count = 0, i; 1255 const struct cs_section_def *sect = NULL; 1256 const struct cs_extent_def *ext = NULL; 1257 1258 if (adev->gfx.rlc.cs_data == NULL) 1259 return; 1260 if (buffer == NULL) 1261 return; 1262 1263 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1264 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1265 1266 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1267 buffer[count++] = cpu_to_le32(0x80000000); 1268 buffer[count++] = cpu_to_le32(0x80000000); 1269 1270 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1271 for (ext = sect->section; ext->extent != NULL; ++ext) { 1272 if (sect->id == SECT_CONTEXT) { 1273 buffer[count++] = 1274 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1275 buffer[count++] = cpu_to_le32(ext->reg_index - 1276 PACKET3_SET_CONTEXT_REG_START); 1277 for (i = 0; i < ext->reg_count; i++) 1278 buffer[count++] = cpu_to_le32(ext->extent[i]); 1279 } else { 1280 return; 1281 } 1282 } 1283 } 1284 1285 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 1286 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - 1287 PACKET3_SET_CONTEXT_REG_START); 1288 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config); 1289 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1); 1290 1291 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1292 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1293 1294 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1295 buffer[count++] = cpu_to_le32(0); 1296 } 1297 1298 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev) 1299 { 1300 if (adev->asic_type == CHIP_CARRIZO) 1301 return 5; 1302 else 1303 return 4; 1304 } 1305 1306 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev) 1307 { 1308 const struct cs_section_def *cs_data; 1309 int r; 1310 1311 adev->gfx.rlc.cs_data = vi_cs_data; 1312 1313 cs_data = adev->gfx.rlc.cs_data; 1314 1315 if (cs_data) { 1316 /* init clear state block */ 1317 r = amdgpu_gfx_rlc_init_csb(adev); 1318 if (r) 1319 return r; 1320 } 1321 1322 if ((adev->asic_type == CHIP_CARRIZO) || 1323 (adev->asic_type == CHIP_STONEY)) { 1324 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1325 r = amdgpu_gfx_rlc_init_cpt(adev); 1326 if (r) 1327 return r; 1328 } 1329 1330 /* init spm vmid with 0xf */ 1331 if (adev->gfx.rlc.funcs->update_spm_vmid) 1332 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1333 1334 return 0; 1335 } 1336 1337 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev) 1338 { 1339 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1340 } 1341 1342 static int gfx_v8_0_mec_init(struct amdgpu_device *adev) 1343 { 1344 int r; 1345 u32 *hpd; 1346 size_t mec_hpd_size; 1347 1348 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1349 1350 /* take ownership of the relevant compute queues */ 1351 amdgpu_gfx_compute_queue_acquire(adev); 1352 1353 mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE; 1354 if (mec_hpd_size) { 1355 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1356 AMDGPU_GEM_DOMAIN_VRAM, 1357 &adev->gfx.mec.hpd_eop_obj, 1358 &adev->gfx.mec.hpd_eop_gpu_addr, 1359 (void **)&hpd); 1360 if (r) { 1361 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1362 return r; 1363 } 1364 1365 memset(hpd, 0, mec_hpd_size); 1366 1367 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1368 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1369 } 1370 1371 return 0; 1372 } 1373 1374 static const u32 vgpr_init_compute_shader[] = 1375 { 1376 0x7e000209, 0x7e020208, 1377 0x7e040207, 0x7e060206, 1378 0x7e080205, 0x7e0a0204, 1379 0x7e0c0203, 0x7e0e0202, 1380 0x7e100201, 0x7e120200, 1381 0x7e140209, 0x7e160208, 1382 0x7e180207, 0x7e1a0206, 1383 0x7e1c0205, 0x7e1e0204, 1384 0x7e200203, 0x7e220202, 1385 0x7e240201, 0x7e260200, 1386 0x7e280209, 0x7e2a0208, 1387 0x7e2c0207, 0x7e2e0206, 1388 0x7e300205, 0x7e320204, 1389 0x7e340203, 0x7e360202, 1390 0x7e380201, 0x7e3a0200, 1391 0x7e3c0209, 0x7e3e0208, 1392 0x7e400207, 0x7e420206, 1393 0x7e440205, 0x7e460204, 1394 0x7e480203, 0x7e4a0202, 1395 0x7e4c0201, 0x7e4e0200, 1396 0x7e500209, 0x7e520208, 1397 0x7e540207, 0x7e560206, 1398 0x7e580205, 0x7e5a0204, 1399 0x7e5c0203, 0x7e5e0202, 1400 0x7e600201, 0x7e620200, 1401 0x7e640209, 0x7e660208, 1402 0x7e680207, 0x7e6a0206, 1403 0x7e6c0205, 0x7e6e0204, 1404 0x7e700203, 0x7e720202, 1405 0x7e740201, 0x7e760200, 1406 0x7e780209, 0x7e7a0208, 1407 0x7e7c0207, 0x7e7e0206, 1408 0xbf8a0000, 0xbf810000, 1409 }; 1410 1411 static const u32 sgpr_init_compute_shader[] = 1412 { 1413 0xbe8a0100, 0xbe8c0102, 1414 0xbe8e0104, 0xbe900106, 1415 0xbe920108, 0xbe940100, 1416 0xbe960102, 0xbe980104, 1417 0xbe9a0106, 0xbe9c0108, 1418 0xbe9e0100, 0xbea00102, 1419 0xbea20104, 0xbea40106, 1420 0xbea60108, 0xbea80100, 1421 0xbeaa0102, 0xbeac0104, 1422 0xbeae0106, 0xbeb00108, 1423 0xbeb20100, 0xbeb40102, 1424 0xbeb60104, 0xbeb80106, 1425 0xbeba0108, 0xbebc0100, 1426 0xbebe0102, 0xbec00104, 1427 0xbec20106, 0xbec40108, 1428 0xbec60100, 0xbec80102, 1429 0xbee60004, 0xbee70005, 1430 0xbeea0006, 0xbeeb0007, 1431 0xbee80008, 0xbee90009, 1432 0xbefc0000, 0xbf8a0000, 1433 0xbf810000, 0x00000000, 1434 }; 1435 1436 static const u32 vgpr_init_regs[] = 1437 { 1438 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff, 1439 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1440 mmCOMPUTE_NUM_THREAD_X, 256*4, 1441 mmCOMPUTE_NUM_THREAD_Y, 1, 1442 mmCOMPUTE_NUM_THREAD_Z, 1, 1443 mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */ 1444 mmCOMPUTE_PGM_RSRC2, 20, 1445 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1446 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1447 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1448 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1449 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1450 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1451 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1452 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1453 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1454 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1455 }; 1456 1457 static const u32 sgpr1_init_regs[] = 1458 { 1459 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f, 1460 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1461 mmCOMPUTE_NUM_THREAD_X, 256*5, 1462 mmCOMPUTE_NUM_THREAD_Y, 1, 1463 mmCOMPUTE_NUM_THREAD_Z, 1, 1464 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1465 mmCOMPUTE_PGM_RSRC2, 20, 1466 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1467 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1468 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1469 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1470 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1471 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1472 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1473 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1474 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1475 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1476 }; 1477 1478 static const u32 sgpr2_init_regs[] = 1479 { 1480 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0, 1481 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, 1482 mmCOMPUTE_NUM_THREAD_X, 256*5, 1483 mmCOMPUTE_NUM_THREAD_Y, 1, 1484 mmCOMPUTE_NUM_THREAD_Z, 1, 1485 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1486 mmCOMPUTE_PGM_RSRC2, 20, 1487 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1488 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1489 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1490 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1491 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1492 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1493 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1494 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1495 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1496 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1497 }; 1498 1499 static const u32 sec_ded_counter_registers[] = 1500 { 1501 mmCPC_EDC_ATC_CNT, 1502 mmCPC_EDC_SCRATCH_CNT, 1503 mmCPC_EDC_UCODE_CNT, 1504 mmCPF_EDC_ATC_CNT, 1505 mmCPF_EDC_ROQ_CNT, 1506 mmCPF_EDC_TAG_CNT, 1507 mmCPG_EDC_ATC_CNT, 1508 mmCPG_EDC_DMA_CNT, 1509 mmCPG_EDC_TAG_CNT, 1510 mmDC_EDC_CSINVOC_CNT, 1511 mmDC_EDC_RESTORE_CNT, 1512 mmDC_EDC_STATE_CNT, 1513 mmGDS_EDC_CNT, 1514 mmGDS_EDC_GRBM_CNT, 1515 mmGDS_EDC_OA_DED, 1516 mmSPI_EDC_CNT, 1517 mmSQC_ATC_EDC_GATCL1_CNT, 1518 mmSQC_EDC_CNT, 1519 mmSQ_EDC_DED_CNT, 1520 mmSQ_EDC_INFO, 1521 mmSQ_EDC_SEC_CNT, 1522 mmTCC_EDC_CNT, 1523 mmTCP_ATC_EDC_GATCL1_CNT, 1524 mmTCP_EDC_CNT, 1525 mmTD_EDC_CNT 1526 }; 1527 1528 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 1529 { 1530 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 1531 struct amdgpu_ib ib; 1532 struct dma_fence *f = NULL; 1533 int r, i; 1534 u32 tmp; 1535 unsigned total_size, vgpr_offset, sgpr_offset; 1536 u64 gpu_addr; 1537 1538 /* only supported on CZ */ 1539 if (adev->asic_type != CHIP_CARRIZO) 1540 return 0; 1541 1542 /* bail if the compute ring is not ready */ 1543 if (!ring->sched.ready) 1544 return 0; 1545 1546 tmp = RREG32(mmGB_EDC_MODE); 1547 WREG32(mmGB_EDC_MODE, 0); 1548 1549 total_size = 1550 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1551 total_size += 1552 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1553 total_size += 1554 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1555 total_size = ALIGN(total_size, 256); 1556 vgpr_offset = total_size; 1557 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256); 1558 sgpr_offset = total_size; 1559 total_size += sizeof(sgpr_init_compute_shader); 1560 1561 /* allocate an indirect buffer to put the commands in */ 1562 memset(&ib, 0, sizeof(ib)); 1563 r = amdgpu_ib_get(adev, NULL, total_size, 1564 AMDGPU_IB_POOL_DIRECT, &ib); 1565 if (r) { 1566 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 1567 return r; 1568 } 1569 1570 /* load the compute shaders */ 1571 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++) 1572 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i]; 1573 1574 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 1575 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 1576 1577 /* init the ib length to 0 */ 1578 ib.length_dw = 0; 1579 1580 /* VGPR */ 1581 /* write the register state for the compute dispatch */ 1582 for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) { 1583 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1584 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START; 1585 ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1]; 1586 } 1587 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1588 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 1589 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1590 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1591 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1592 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1593 1594 /* write dispatch packet */ 1595 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1596 ib.ptr[ib.length_dw++] = 8; /* x */ 1597 ib.ptr[ib.length_dw++] = 1; /* y */ 1598 ib.ptr[ib.length_dw++] = 1; /* z */ 1599 ib.ptr[ib.length_dw++] = 1600 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1601 1602 /* write CS partial flush packet */ 1603 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1604 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1605 1606 /* SGPR1 */ 1607 /* write the register state for the compute dispatch */ 1608 for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) { 1609 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1610 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START; 1611 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1]; 1612 } 1613 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1614 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 1615 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1616 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1617 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1618 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1619 1620 /* write dispatch packet */ 1621 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1622 ib.ptr[ib.length_dw++] = 8; /* x */ 1623 ib.ptr[ib.length_dw++] = 1; /* y */ 1624 ib.ptr[ib.length_dw++] = 1; /* z */ 1625 ib.ptr[ib.length_dw++] = 1626 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1627 1628 /* write CS partial flush packet */ 1629 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1630 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1631 1632 /* SGPR2 */ 1633 /* write the register state for the compute dispatch */ 1634 for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) { 1635 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1636 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START; 1637 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1]; 1638 } 1639 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1640 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 1641 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1642 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1643 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1644 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1645 1646 /* write dispatch packet */ 1647 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1648 ib.ptr[ib.length_dw++] = 8; /* x */ 1649 ib.ptr[ib.length_dw++] = 1; /* y */ 1650 ib.ptr[ib.length_dw++] = 1; /* z */ 1651 ib.ptr[ib.length_dw++] = 1652 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1653 1654 /* write CS partial flush packet */ 1655 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1656 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1657 1658 /* shedule the ib on the ring */ 1659 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1660 if (r) { 1661 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 1662 goto fail; 1663 } 1664 1665 /* wait for the GPU to finish processing the IB */ 1666 r = dma_fence_wait(f, false); 1667 if (r) { 1668 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 1669 goto fail; 1670 } 1671 1672 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2); 1673 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1); 1674 WREG32(mmGB_EDC_MODE, tmp); 1675 1676 tmp = RREG32(mmCC_GC_EDC_CONFIG); 1677 tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1; 1678 WREG32(mmCC_GC_EDC_CONFIG, tmp); 1679 1680 1681 /* read back registers to clear the counters */ 1682 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) 1683 RREG32(sec_ded_counter_registers[i]); 1684 1685 fail: 1686 amdgpu_ib_free(adev, &ib, NULL); 1687 dma_fence_put(f); 1688 1689 return r; 1690 } 1691 1692 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev) 1693 { 1694 u32 gb_addr_config; 1695 u32 mc_arb_ramcfg; 1696 u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map; 1697 u32 tmp; 1698 int ret; 1699 1700 switch (adev->asic_type) { 1701 case CHIP_TOPAZ: 1702 adev->gfx.config.max_shader_engines = 1; 1703 adev->gfx.config.max_tile_pipes = 2; 1704 adev->gfx.config.max_cu_per_sh = 6; 1705 adev->gfx.config.max_sh_per_se = 1; 1706 adev->gfx.config.max_backends_per_se = 2; 1707 adev->gfx.config.max_texture_channel_caches = 2; 1708 adev->gfx.config.max_gprs = 256; 1709 adev->gfx.config.max_gs_threads = 32; 1710 adev->gfx.config.max_hw_contexts = 8; 1711 1712 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1713 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1714 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1715 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1716 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN; 1717 break; 1718 case CHIP_FIJI: 1719 adev->gfx.config.max_shader_engines = 4; 1720 adev->gfx.config.max_tile_pipes = 16; 1721 adev->gfx.config.max_cu_per_sh = 16; 1722 adev->gfx.config.max_sh_per_se = 1; 1723 adev->gfx.config.max_backends_per_se = 4; 1724 adev->gfx.config.max_texture_channel_caches = 16; 1725 adev->gfx.config.max_gprs = 256; 1726 adev->gfx.config.max_gs_threads = 32; 1727 adev->gfx.config.max_hw_contexts = 8; 1728 1729 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1730 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1731 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1732 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1733 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1734 break; 1735 case CHIP_POLARIS11: 1736 case CHIP_POLARIS12: 1737 ret = amdgpu_atombios_get_gfx_info(adev); 1738 if (ret) 1739 return ret; 1740 adev->gfx.config.max_gprs = 256; 1741 adev->gfx.config.max_gs_threads = 32; 1742 adev->gfx.config.max_hw_contexts = 8; 1743 1744 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1745 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1746 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1747 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1748 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN; 1749 break; 1750 case CHIP_POLARIS10: 1751 case CHIP_VEGAM: 1752 ret = amdgpu_atombios_get_gfx_info(adev); 1753 if (ret) 1754 return ret; 1755 adev->gfx.config.max_gprs = 256; 1756 adev->gfx.config.max_gs_threads = 32; 1757 adev->gfx.config.max_hw_contexts = 8; 1758 1759 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1760 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1761 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1762 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1763 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1764 break; 1765 case CHIP_TONGA: 1766 adev->gfx.config.max_shader_engines = 4; 1767 adev->gfx.config.max_tile_pipes = 8; 1768 adev->gfx.config.max_cu_per_sh = 8; 1769 adev->gfx.config.max_sh_per_se = 1; 1770 adev->gfx.config.max_backends_per_se = 2; 1771 adev->gfx.config.max_texture_channel_caches = 8; 1772 adev->gfx.config.max_gprs = 256; 1773 adev->gfx.config.max_gs_threads = 32; 1774 adev->gfx.config.max_hw_contexts = 8; 1775 1776 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1777 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1778 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1779 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1780 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1781 break; 1782 case CHIP_CARRIZO: 1783 adev->gfx.config.max_shader_engines = 1; 1784 adev->gfx.config.max_tile_pipes = 2; 1785 adev->gfx.config.max_sh_per_se = 1; 1786 adev->gfx.config.max_backends_per_se = 2; 1787 adev->gfx.config.max_cu_per_sh = 8; 1788 adev->gfx.config.max_texture_channel_caches = 2; 1789 adev->gfx.config.max_gprs = 256; 1790 adev->gfx.config.max_gs_threads = 32; 1791 adev->gfx.config.max_hw_contexts = 8; 1792 1793 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1794 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1795 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1796 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1797 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1798 break; 1799 case CHIP_STONEY: 1800 adev->gfx.config.max_shader_engines = 1; 1801 adev->gfx.config.max_tile_pipes = 2; 1802 adev->gfx.config.max_sh_per_se = 1; 1803 adev->gfx.config.max_backends_per_se = 1; 1804 adev->gfx.config.max_cu_per_sh = 3; 1805 adev->gfx.config.max_texture_channel_caches = 2; 1806 adev->gfx.config.max_gprs = 256; 1807 adev->gfx.config.max_gs_threads = 16; 1808 adev->gfx.config.max_hw_contexts = 8; 1809 1810 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1811 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1812 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1813 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1814 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1815 break; 1816 default: 1817 adev->gfx.config.max_shader_engines = 2; 1818 adev->gfx.config.max_tile_pipes = 4; 1819 adev->gfx.config.max_cu_per_sh = 2; 1820 adev->gfx.config.max_sh_per_se = 1; 1821 adev->gfx.config.max_backends_per_se = 2; 1822 adev->gfx.config.max_texture_channel_caches = 4; 1823 adev->gfx.config.max_gprs = 256; 1824 adev->gfx.config.max_gs_threads = 32; 1825 adev->gfx.config.max_hw_contexts = 8; 1826 1827 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1828 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1829 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1830 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1831 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1832 break; 1833 } 1834 1835 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG); 1836 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg; 1837 1838 adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg, 1839 MC_ARB_RAMCFG, NOOFBANK); 1840 adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg, 1841 MC_ARB_RAMCFG, NOOFRANKS); 1842 1843 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; 1844 adev->gfx.config.mem_max_burst_length_bytes = 256; 1845 if (adev->flags & AMD_IS_APU) { 1846 /* Get memory bank mapping mode. */ 1847 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING); 1848 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1849 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1850 1851 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING); 1852 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1853 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1854 1855 /* Validate settings in case only one DIMM installed. */ 1856 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12)) 1857 dimm00_addr_map = 0; 1858 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12)) 1859 dimm01_addr_map = 0; 1860 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12)) 1861 dimm10_addr_map = 0; 1862 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12)) 1863 dimm11_addr_map = 0; 1864 1865 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */ 1866 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */ 1867 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11)) 1868 adev->gfx.config.mem_row_size_in_kb = 2; 1869 else 1870 adev->gfx.config.mem_row_size_in_kb = 1; 1871 } else { 1872 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS); 1873 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 1874 if (adev->gfx.config.mem_row_size_in_kb > 4) 1875 adev->gfx.config.mem_row_size_in_kb = 4; 1876 } 1877 1878 adev->gfx.config.shader_engine_tile_size = 32; 1879 adev->gfx.config.num_gpus = 1; 1880 adev->gfx.config.multi_gpu_tile_size = 64; 1881 1882 /* fix up row size */ 1883 switch (adev->gfx.config.mem_row_size_in_kb) { 1884 case 1: 1885 default: 1886 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0); 1887 break; 1888 case 2: 1889 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1); 1890 break; 1891 case 4: 1892 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2); 1893 break; 1894 } 1895 adev->gfx.config.gb_addr_config = gb_addr_config; 1896 1897 return 0; 1898 } 1899 1900 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1901 int mec, int pipe, int queue) 1902 { 1903 int r; 1904 unsigned irq_type; 1905 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 1906 unsigned int hw_prio; 1907 1908 ring = &adev->gfx.compute_ring[ring_id]; 1909 1910 /* mec0 is me1 */ 1911 ring->me = mec + 1; 1912 ring->pipe = pipe; 1913 ring->queue = queue; 1914 1915 ring->ring_obj = NULL; 1916 ring->use_doorbell = true; 1917 ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id; 1918 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1919 + (ring_id * GFX8_MEC_HPD_SIZE); 1920 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1921 1922 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1923 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1924 + ring->pipe; 1925 1926 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue) ? 1927 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_RING_PRIO_DEFAULT; 1928 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1929 r = amdgpu_ring_init(adev, ring, 1024, 1930 &adev->gfx.eop_irq, irq_type, hw_prio); 1931 if (r) 1932 return r; 1933 1934 1935 return 0; 1936 } 1937 1938 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work); 1939 1940 static int gfx_v8_0_sw_init(void *handle) 1941 { 1942 int i, j, k, r, ring_id; 1943 struct amdgpu_ring *ring; 1944 struct amdgpu_kiq *kiq; 1945 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1946 1947 switch (adev->asic_type) { 1948 case CHIP_TONGA: 1949 case CHIP_CARRIZO: 1950 case CHIP_FIJI: 1951 case CHIP_POLARIS10: 1952 case CHIP_POLARIS11: 1953 case CHIP_POLARIS12: 1954 case CHIP_VEGAM: 1955 adev->gfx.mec.num_mec = 2; 1956 break; 1957 case CHIP_TOPAZ: 1958 case CHIP_STONEY: 1959 default: 1960 adev->gfx.mec.num_mec = 1; 1961 break; 1962 } 1963 1964 adev->gfx.mec.num_pipe_per_mec = 4; 1965 adev->gfx.mec.num_queue_per_pipe = 8; 1966 1967 /* EOP Event */ 1968 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq); 1969 if (r) 1970 return r; 1971 1972 /* Privileged reg */ 1973 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT, 1974 &adev->gfx.priv_reg_irq); 1975 if (r) 1976 return r; 1977 1978 /* Privileged inst */ 1979 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT, 1980 &adev->gfx.priv_inst_irq); 1981 if (r) 1982 return r; 1983 1984 /* Add CP EDC/ECC irq */ 1985 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR, 1986 &adev->gfx.cp_ecc_error_irq); 1987 if (r) 1988 return r; 1989 1990 /* SQ interrupts. */ 1991 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG, 1992 &adev->gfx.sq_irq); 1993 if (r) { 1994 DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r); 1995 return r; 1996 } 1997 1998 INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func); 1999 2000 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2001 2002 gfx_v8_0_scratch_init(adev); 2003 2004 r = gfx_v8_0_init_microcode(adev); 2005 if (r) { 2006 DRM_ERROR("Failed to load gfx firmware!\n"); 2007 return r; 2008 } 2009 2010 r = adev->gfx.rlc.funcs->init(adev); 2011 if (r) { 2012 DRM_ERROR("Failed to init rlc BOs!\n"); 2013 return r; 2014 } 2015 2016 r = gfx_v8_0_mec_init(adev); 2017 if (r) { 2018 DRM_ERROR("Failed to init MEC BOs!\n"); 2019 return r; 2020 } 2021 2022 /* set up the gfx ring */ 2023 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2024 ring = &adev->gfx.gfx_ring[i]; 2025 ring->ring_obj = NULL; 2026 sprintf(ring->name, "gfx"); 2027 /* no gfx doorbells on iceland */ 2028 if (adev->asic_type != CHIP_TOPAZ) { 2029 ring->use_doorbell = true; 2030 ring->doorbell_index = adev->doorbell_index.gfx_ring0; 2031 } 2032 2033 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2034 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2035 AMDGPU_RING_PRIO_DEFAULT); 2036 if (r) 2037 return r; 2038 } 2039 2040 2041 /* set up the compute queues - allocate horizontally across pipes */ 2042 ring_id = 0; 2043 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2044 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2045 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2046 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2047 continue; 2048 2049 r = gfx_v8_0_compute_ring_init(adev, 2050 ring_id, 2051 i, k, j); 2052 if (r) 2053 return r; 2054 2055 ring_id++; 2056 } 2057 } 2058 } 2059 2060 r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE); 2061 if (r) { 2062 DRM_ERROR("Failed to init KIQ BOs!\n"); 2063 return r; 2064 } 2065 2066 kiq = &adev->gfx.kiq; 2067 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2068 if (r) 2069 return r; 2070 2071 /* create MQD for all compute queues as well as KIQ for SRIOV case */ 2072 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation)); 2073 if (r) 2074 return r; 2075 2076 adev->gfx.ce_ram_size = 0x8000; 2077 2078 r = gfx_v8_0_gpu_early_init(adev); 2079 if (r) 2080 return r; 2081 2082 return 0; 2083 } 2084 2085 static int gfx_v8_0_sw_fini(void *handle) 2086 { 2087 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2088 int i; 2089 2090 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2091 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2092 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2093 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2094 2095 amdgpu_gfx_mqd_sw_fini(adev); 2096 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2097 amdgpu_gfx_kiq_fini(adev); 2098 2099 gfx_v8_0_mec_fini(adev); 2100 amdgpu_gfx_rlc_fini(adev); 2101 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2102 &adev->gfx.rlc.clear_state_gpu_addr, 2103 (void **)&adev->gfx.rlc.cs_ptr); 2104 if ((adev->asic_type == CHIP_CARRIZO) || 2105 (adev->asic_type == CHIP_STONEY)) { 2106 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2107 &adev->gfx.rlc.cp_table_gpu_addr, 2108 (void **)&adev->gfx.rlc.cp_table_ptr); 2109 } 2110 gfx_v8_0_free_microcode(adev); 2111 2112 return 0; 2113 } 2114 2115 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev) 2116 { 2117 uint32_t *modearray, *mod2array; 2118 const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array); 2119 const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 2120 u32 reg_offset; 2121 2122 modearray = adev->gfx.config.tile_mode_array; 2123 mod2array = adev->gfx.config.macrotile_mode_array; 2124 2125 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2126 modearray[reg_offset] = 0; 2127 2128 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2129 mod2array[reg_offset] = 0; 2130 2131 switch (adev->asic_type) { 2132 case CHIP_TOPAZ: 2133 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2134 PIPE_CONFIG(ADDR_SURF_P2) | 2135 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2136 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2137 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2138 PIPE_CONFIG(ADDR_SURF_P2) | 2139 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2140 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2141 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2142 PIPE_CONFIG(ADDR_SURF_P2) | 2143 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2144 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2145 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2146 PIPE_CONFIG(ADDR_SURF_P2) | 2147 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2148 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2149 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2150 PIPE_CONFIG(ADDR_SURF_P2) | 2151 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2152 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2153 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2154 PIPE_CONFIG(ADDR_SURF_P2) | 2155 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2156 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2157 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2158 PIPE_CONFIG(ADDR_SURF_P2) | 2159 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2160 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2161 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2162 PIPE_CONFIG(ADDR_SURF_P2)); 2163 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2164 PIPE_CONFIG(ADDR_SURF_P2) | 2165 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2166 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2167 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2168 PIPE_CONFIG(ADDR_SURF_P2) | 2169 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2170 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2171 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2172 PIPE_CONFIG(ADDR_SURF_P2) | 2173 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2175 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2176 PIPE_CONFIG(ADDR_SURF_P2) | 2177 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2178 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2179 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2180 PIPE_CONFIG(ADDR_SURF_P2) | 2181 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2183 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2184 PIPE_CONFIG(ADDR_SURF_P2) | 2185 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2186 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2187 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2188 PIPE_CONFIG(ADDR_SURF_P2) | 2189 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2190 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2191 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2192 PIPE_CONFIG(ADDR_SURF_P2) | 2193 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2194 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2195 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2196 PIPE_CONFIG(ADDR_SURF_P2) | 2197 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2198 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2199 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2200 PIPE_CONFIG(ADDR_SURF_P2) | 2201 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2202 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2203 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2204 PIPE_CONFIG(ADDR_SURF_P2) | 2205 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2206 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2207 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2208 PIPE_CONFIG(ADDR_SURF_P2) | 2209 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2210 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2211 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2212 PIPE_CONFIG(ADDR_SURF_P2) | 2213 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2214 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2215 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2216 PIPE_CONFIG(ADDR_SURF_P2) | 2217 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2218 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2219 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2220 PIPE_CONFIG(ADDR_SURF_P2) | 2221 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2222 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2223 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2224 PIPE_CONFIG(ADDR_SURF_P2) | 2225 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2226 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2227 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2228 PIPE_CONFIG(ADDR_SURF_P2) | 2229 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2230 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2231 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2232 PIPE_CONFIG(ADDR_SURF_P2) | 2233 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2234 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2235 2236 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2237 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2238 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2239 NUM_BANKS(ADDR_SURF_8_BANK)); 2240 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2241 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2242 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2243 NUM_BANKS(ADDR_SURF_8_BANK)); 2244 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2247 NUM_BANKS(ADDR_SURF_8_BANK)); 2248 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2249 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2250 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2251 NUM_BANKS(ADDR_SURF_8_BANK)); 2252 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2255 NUM_BANKS(ADDR_SURF_8_BANK)); 2256 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2257 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2258 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2259 NUM_BANKS(ADDR_SURF_8_BANK)); 2260 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2261 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2262 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2263 NUM_BANKS(ADDR_SURF_8_BANK)); 2264 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2265 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2266 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2267 NUM_BANKS(ADDR_SURF_16_BANK)); 2268 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2269 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2270 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2271 NUM_BANKS(ADDR_SURF_16_BANK)); 2272 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2273 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2274 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2275 NUM_BANKS(ADDR_SURF_16_BANK)); 2276 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2277 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2278 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2279 NUM_BANKS(ADDR_SURF_16_BANK)); 2280 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2281 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2282 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2283 NUM_BANKS(ADDR_SURF_16_BANK)); 2284 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2285 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2286 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2287 NUM_BANKS(ADDR_SURF_16_BANK)); 2288 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2289 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2290 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2291 NUM_BANKS(ADDR_SURF_8_BANK)); 2292 2293 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2294 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 2295 reg_offset != 23) 2296 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2297 2298 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2299 if (reg_offset != 7) 2300 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2301 2302 break; 2303 case CHIP_FIJI: 2304 case CHIP_VEGAM: 2305 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2306 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2307 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2308 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2309 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2311 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2312 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2313 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2314 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2315 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2316 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2317 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2319 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2320 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2321 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2323 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2324 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2325 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2326 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2327 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2328 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2329 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2331 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2332 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2333 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2334 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2335 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2336 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2337 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2338 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2339 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2340 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2341 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2342 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2343 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2344 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2345 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2346 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2347 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2348 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2349 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2350 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2351 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2352 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2353 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2354 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2355 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2356 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2357 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2358 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2359 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2360 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2361 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2362 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2363 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2365 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2366 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2367 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2369 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2370 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2371 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2372 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2373 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2374 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2375 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2376 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2377 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2378 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2379 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2381 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2382 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2383 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2384 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2385 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2386 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2387 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2388 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2389 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2391 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2392 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2393 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2394 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2395 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2396 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2397 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2398 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2399 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2400 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2401 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2402 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2403 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2404 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2405 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2406 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2407 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2408 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2409 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2410 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2411 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2412 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2413 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2414 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2415 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2416 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2417 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2418 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2419 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2420 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2421 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2422 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2423 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2424 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2425 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2426 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2427 2428 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2429 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2430 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2431 NUM_BANKS(ADDR_SURF_8_BANK)); 2432 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2433 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2434 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2435 NUM_BANKS(ADDR_SURF_8_BANK)); 2436 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2437 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2438 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2439 NUM_BANKS(ADDR_SURF_8_BANK)); 2440 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2441 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2442 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2443 NUM_BANKS(ADDR_SURF_8_BANK)); 2444 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2447 NUM_BANKS(ADDR_SURF_8_BANK)); 2448 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2449 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2450 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2451 NUM_BANKS(ADDR_SURF_8_BANK)); 2452 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2453 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2454 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2455 NUM_BANKS(ADDR_SURF_8_BANK)); 2456 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2457 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2458 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2459 NUM_BANKS(ADDR_SURF_8_BANK)); 2460 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2461 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2462 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2463 NUM_BANKS(ADDR_SURF_8_BANK)); 2464 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2465 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2466 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2467 NUM_BANKS(ADDR_SURF_8_BANK)); 2468 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2469 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2470 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2471 NUM_BANKS(ADDR_SURF_8_BANK)); 2472 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2473 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2474 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2475 NUM_BANKS(ADDR_SURF_8_BANK)); 2476 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2477 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2478 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2479 NUM_BANKS(ADDR_SURF_8_BANK)); 2480 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2481 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2482 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2483 NUM_BANKS(ADDR_SURF_4_BANK)); 2484 2485 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2486 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2487 2488 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2489 if (reg_offset != 7) 2490 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2491 2492 break; 2493 case CHIP_TONGA: 2494 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2495 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2496 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2497 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2498 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2499 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2500 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2501 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2502 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2503 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2504 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2505 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2506 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2507 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2509 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2510 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2512 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2513 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2514 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2515 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2516 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2517 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2518 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2519 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2520 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2521 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2522 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2523 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2524 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2525 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2526 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2528 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2529 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2530 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2531 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2532 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2534 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2535 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2536 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2537 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2538 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2540 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2541 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2542 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2543 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2544 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2546 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2547 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2548 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2549 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2550 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2552 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2554 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2555 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2556 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2557 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2558 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2559 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2560 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2561 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2562 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2563 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2564 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2566 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2567 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2568 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2570 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2571 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2572 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2574 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2575 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2576 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2577 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2578 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2579 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2580 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2581 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2582 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2583 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2584 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2585 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2586 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2587 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2588 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2589 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2590 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2591 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2592 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2593 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2594 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2595 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2596 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2598 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2599 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2600 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2601 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2602 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2603 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2604 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2605 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2606 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2607 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2608 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2609 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2610 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2611 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2612 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2613 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2614 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2615 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2616 2617 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2618 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2619 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2620 NUM_BANKS(ADDR_SURF_16_BANK)); 2621 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2622 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2623 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2624 NUM_BANKS(ADDR_SURF_16_BANK)); 2625 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2626 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2627 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2628 NUM_BANKS(ADDR_SURF_16_BANK)); 2629 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2630 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2631 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2632 NUM_BANKS(ADDR_SURF_16_BANK)); 2633 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2634 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2635 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2636 NUM_BANKS(ADDR_SURF_16_BANK)); 2637 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2638 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2639 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2640 NUM_BANKS(ADDR_SURF_16_BANK)); 2641 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2642 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2643 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2644 NUM_BANKS(ADDR_SURF_16_BANK)); 2645 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2648 NUM_BANKS(ADDR_SURF_16_BANK)); 2649 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2650 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2651 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2652 NUM_BANKS(ADDR_SURF_16_BANK)); 2653 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2654 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2655 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2656 NUM_BANKS(ADDR_SURF_16_BANK)); 2657 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2658 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2659 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2660 NUM_BANKS(ADDR_SURF_16_BANK)); 2661 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2662 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2663 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2664 NUM_BANKS(ADDR_SURF_8_BANK)); 2665 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2666 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2667 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2668 NUM_BANKS(ADDR_SURF_4_BANK)); 2669 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2670 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2671 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2672 NUM_BANKS(ADDR_SURF_4_BANK)); 2673 2674 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2675 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2676 2677 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2678 if (reg_offset != 7) 2679 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2680 2681 break; 2682 case CHIP_POLARIS11: 2683 case CHIP_POLARIS12: 2684 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2685 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2686 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2687 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2688 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2689 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2690 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2691 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2692 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2693 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2694 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2695 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2696 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2697 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2698 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2699 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2700 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2701 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2702 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2703 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2704 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2705 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2706 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2707 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2708 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2709 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2710 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2711 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2712 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2713 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2714 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2715 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2716 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2717 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2718 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2719 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2720 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2721 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2722 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2723 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2724 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2725 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2726 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2727 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2728 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2729 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2730 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2731 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2732 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2733 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2734 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2735 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2736 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2737 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2738 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2739 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2740 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2741 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2742 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2743 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2744 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2745 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2746 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2747 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2748 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2749 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2750 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2752 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2753 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2754 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2755 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2756 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2757 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2758 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2759 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2760 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2761 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2762 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2764 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2765 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2766 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2767 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2768 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2769 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2770 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2771 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2772 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2773 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2774 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2775 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2776 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2777 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2778 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2779 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2780 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2781 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2782 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2783 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2784 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2785 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2786 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2787 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2788 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2789 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2790 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2791 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2792 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2793 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2794 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2795 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2796 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2797 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2798 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2799 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2800 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2801 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2802 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2803 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2804 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2805 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2806 2807 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2808 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2809 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2810 NUM_BANKS(ADDR_SURF_16_BANK)); 2811 2812 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2813 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2814 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2815 NUM_BANKS(ADDR_SURF_16_BANK)); 2816 2817 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2818 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2819 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2820 NUM_BANKS(ADDR_SURF_16_BANK)); 2821 2822 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2823 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2824 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2825 NUM_BANKS(ADDR_SURF_16_BANK)); 2826 2827 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2828 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2829 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2830 NUM_BANKS(ADDR_SURF_16_BANK)); 2831 2832 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2833 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2834 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2835 NUM_BANKS(ADDR_SURF_16_BANK)); 2836 2837 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2838 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2839 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2840 NUM_BANKS(ADDR_SURF_16_BANK)); 2841 2842 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2843 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2844 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2845 NUM_BANKS(ADDR_SURF_16_BANK)); 2846 2847 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2848 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2849 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2850 NUM_BANKS(ADDR_SURF_16_BANK)); 2851 2852 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2853 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2854 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2855 NUM_BANKS(ADDR_SURF_16_BANK)); 2856 2857 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2858 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2859 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2860 NUM_BANKS(ADDR_SURF_16_BANK)); 2861 2862 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2863 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2864 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2865 NUM_BANKS(ADDR_SURF_16_BANK)); 2866 2867 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2868 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2869 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2870 NUM_BANKS(ADDR_SURF_8_BANK)); 2871 2872 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2873 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2874 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2875 NUM_BANKS(ADDR_SURF_4_BANK)); 2876 2877 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2878 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2879 2880 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2881 if (reg_offset != 7) 2882 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2883 2884 break; 2885 case CHIP_POLARIS10: 2886 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2887 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2888 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2889 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2890 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2891 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2892 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2893 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2894 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2895 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2896 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2897 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2898 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2899 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2900 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2901 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2902 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2903 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2904 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2905 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2906 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2907 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2908 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2909 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2910 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2911 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2913 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2914 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2915 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2916 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2917 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2918 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2919 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2920 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2921 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2922 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2923 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2924 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2925 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2926 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2927 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2928 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2929 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2930 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2931 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2932 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2933 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2934 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2935 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2936 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2937 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2938 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2939 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2940 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2941 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2942 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2943 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2944 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2945 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2946 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2947 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2948 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2949 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2950 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2951 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2952 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2953 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2954 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2955 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2956 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2957 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2958 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2959 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2960 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2961 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2962 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2963 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2964 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2965 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2966 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2968 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2969 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2970 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2971 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2972 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2973 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2974 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2975 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2976 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2977 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2978 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2979 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2980 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2981 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2982 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2983 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2984 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2985 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2986 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2987 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2988 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2989 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2990 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2991 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2992 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2993 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2994 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2995 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2996 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2997 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2998 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2999 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3000 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3001 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 3002 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3003 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3004 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3005 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 3006 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3007 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3008 3009 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3010 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3011 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3012 NUM_BANKS(ADDR_SURF_16_BANK)); 3013 3014 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3015 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3016 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3017 NUM_BANKS(ADDR_SURF_16_BANK)); 3018 3019 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3020 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3021 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3022 NUM_BANKS(ADDR_SURF_16_BANK)); 3023 3024 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3025 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3026 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3027 NUM_BANKS(ADDR_SURF_16_BANK)); 3028 3029 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3030 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3031 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3032 NUM_BANKS(ADDR_SURF_16_BANK)); 3033 3034 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3035 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3036 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3037 NUM_BANKS(ADDR_SURF_16_BANK)); 3038 3039 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3040 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3041 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3042 NUM_BANKS(ADDR_SURF_16_BANK)); 3043 3044 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3045 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3046 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3047 NUM_BANKS(ADDR_SURF_16_BANK)); 3048 3049 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3050 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3051 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3052 NUM_BANKS(ADDR_SURF_16_BANK)); 3053 3054 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3055 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3056 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3057 NUM_BANKS(ADDR_SURF_16_BANK)); 3058 3059 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3060 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3061 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3062 NUM_BANKS(ADDR_SURF_16_BANK)); 3063 3064 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3065 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3066 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3067 NUM_BANKS(ADDR_SURF_8_BANK)); 3068 3069 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3070 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3071 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3072 NUM_BANKS(ADDR_SURF_4_BANK)); 3073 3074 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3075 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3076 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3077 NUM_BANKS(ADDR_SURF_4_BANK)); 3078 3079 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3080 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3081 3082 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3083 if (reg_offset != 7) 3084 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3085 3086 break; 3087 case CHIP_STONEY: 3088 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3089 PIPE_CONFIG(ADDR_SURF_P2) | 3090 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3091 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3092 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3093 PIPE_CONFIG(ADDR_SURF_P2) | 3094 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3095 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3096 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3097 PIPE_CONFIG(ADDR_SURF_P2) | 3098 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3099 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3100 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3101 PIPE_CONFIG(ADDR_SURF_P2) | 3102 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3103 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3104 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3105 PIPE_CONFIG(ADDR_SURF_P2) | 3106 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3107 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3108 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3109 PIPE_CONFIG(ADDR_SURF_P2) | 3110 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3111 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3112 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3113 PIPE_CONFIG(ADDR_SURF_P2) | 3114 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3115 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3116 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3117 PIPE_CONFIG(ADDR_SURF_P2)); 3118 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3119 PIPE_CONFIG(ADDR_SURF_P2) | 3120 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3121 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3122 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3123 PIPE_CONFIG(ADDR_SURF_P2) | 3124 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3125 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3126 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3127 PIPE_CONFIG(ADDR_SURF_P2) | 3128 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3129 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3130 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3131 PIPE_CONFIG(ADDR_SURF_P2) | 3132 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3133 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3134 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3135 PIPE_CONFIG(ADDR_SURF_P2) | 3136 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3137 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3138 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3139 PIPE_CONFIG(ADDR_SURF_P2) | 3140 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3141 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3142 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3143 PIPE_CONFIG(ADDR_SURF_P2) | 3144 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3145 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3146 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3147 PIPE_CONFIG(ADDR_SURF_P2) | 3148 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3149 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3150 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3151 PIPE_CONFIG(ADDR_SURF_P2) | 3152 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3153 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3154 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3155 PIPE_CONFIG(ADDR_SURF_P2) | 3156 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3157 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3158 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3159 PIPE_CONFIG(ADDR_SURF_P2) | 3160 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3161 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3162 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3163 PIPE_CONFIG(ADDR_SURF_P2) | 3164 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3165 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3166 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3167 PIPE_CONFIG(ADDR_SURF_P2) | 3168 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3169 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3170 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3171 PIPE_CONFIG(ADDR_SURF_P2) | 3172 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3173 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3174 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3175 PIPE_CONFIG(ADDR_SURF_P2) | 3176 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3177 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3178 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3179 PIPE_CONFIG(ADDR_SURF_P2) | 3180 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3181 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3182 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3183 PIPE_CONFIG(ADDR_SURF_P2) | 3184 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3185 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3186 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3187 PIPE_CONFIG(ADDR_SURF_P2) | 3188 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3189 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3190 3191 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3194 NUM_BANKS(ADDR_SURF_8_BANK)); 3195 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3196 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3197 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3198 NUM_BANKS(ADDR_SURF_8_BANK)); 3199 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3200 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3201 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3202 NUM_BANKS(ADDR_SURF_8_BANK)); 3203 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3206 NUM_BANKS(ADDR_SURF_8_BANK)); 3207 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3208 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3209 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3210 NUM_BANKS(ADDR_SURF_8_BANK)); 3211 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3212 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3213 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3214 NUM_BANKS(ADDR_SURF_8_BANK)); 3215 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3216 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3217 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3218 NUM_BANKS(ADDR_SURF_8_BANK)); 3219 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3220 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3221 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3222 NUM_BANKS(ADDR_SURF_16_BANK)); 3223 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3224 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3225 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3226 NUM_BANKS(ADDR_SURF_16_BANK)); 3227 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3228 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3229 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3230 NUM_BANKS(ADDR_SURF_16_BANK)); 3231 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3232 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3233 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3234 NUM_BANKS(ADDR_SURF_16_BANK)); 3235 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3236 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3237 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3238 NUM_BANKS(ADDR_SURF_16_BANK)); 3239 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3240 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3241 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3242 NUM_BANKS(ADDR_SURF_16_BANK)); 3243 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3244 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3245 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3246 NUM_BANKS(ADDR_SURF_8_BANK)); 3247 3248 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3249 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3250 reg_offset != 23) 3251 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3252 3253 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3254 if (reg_offset != 7) 3255 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3256 3257 break; 3258 default: 3259 dev_warn(adev->dev, 3260 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n", 3261 adev->asic_type); 3262 fallthrough; 3263 3264 case CHIP_CARRIZO: 3265 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3266 PIPE_CONFIG(ADDR_SURF_P2) | 3267 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3268 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3269 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3270 PIPE_CONFIG(ADDR_SURF_P2) | 3271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3272 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3273 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3274 PIPE_CONFIG(ADDR_SURF_P2) | 3275 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3276 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3277 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3278 PIPE_CONFIG(ADDR_SURF_P2) | 3279 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3280 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3281 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3282 PIPE_CONFIG(ADDR_SURF_P2) | 3283 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3284 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3285 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3286 PIPE_CONFIG(ADDR_SURF_P2) | 3287 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3288 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3289 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3290 PIPE_CONFIG(ADDR_SURF_P2) | 3291 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3292 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3293 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3294 PIPE_CONFIG(ADDR_SURF_P2)); 3295 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3296 PIPE_CONFIG(ADDR_SURF_P2) | 3297 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3299 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3300 PIPE_CONFIG(ADDR_SURF_P2) | 3301 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3302 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3303 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3304 PIPE_CONFIG(ADDR_SURF_P2) | 3305 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3306 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3307 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3308 PIPE_CONFIG(ADDR_SURF_P2) | 3309 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3310 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3311 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3312 PIPE_CONFIG(ADDR_SURF_P2) | 3313 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3314 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3315 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3316 PIPE_CONFIG(ADDR_SURF_P2) | 3317 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3318 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3319 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3320 PIPE_CONFIG(ADDR_SURF_P2) | 3321 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3322 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3323 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3324 PIPE_CONFIG(ADDR_SURF_P2) | 3325 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3326 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3327 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3328 PIPE_CONFIG(ADDR_SURF_P2) | 3329 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3330 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3331 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3332 PIPE_CONFIG(ADDR_SURF_P2) | 3333 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3334 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3335 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3336 PIPE_CONFIG(ADDR_SURF_P2) | 3337 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3338 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3339 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3340 PIPE_CONFIG(ADDR_SURF_P2) | 3341 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3342 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3343 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3344 PIPE_CONFIG(ADDR_SURF_P2) | 3345 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3346 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3347 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3348 PIPE_CONFIG(ADDR_SURF_P2) | 3349 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3350 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3351 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3352 PIPE_CONFIG(ADDR_SURF_P2) | 3353 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3354 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3355 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3356 PIPE_CONFIG(ADDR_SURF_P2) | 3357 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3358 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3359 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3360 PIPE_CONFIG(ADDR_SURF_P2) | 3361 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3362 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3363 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3364 PIPE_CONFIG(ADDR_SURF_P2) | 3365 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3366 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3367 3368 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3371 NUM_BANKS(ADDR_SURF_8_BANK)); 3372 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3373 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3374 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3375 NUM_BANKS(ADDR_SURF_8_BANK)); 3376 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3379 NUM_BANKS(ADDR_SURF_8_BANK)); 3380 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3383 NUM_BANKS(ADDR_SURF_8_BANK)); 3384 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3385 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3386 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3387 NUM_BANKS(ADDR_SURF_8_BANK)); 3388 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3389 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3390 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3391 NUM_BANKS(ADDR_SURF_8_BANK)); 3392 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3395 NUM_BANKS(ADDR_SURF_8_BANK)); 3396 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3399 NUM_BANKS(ADDR_SURF_16_BANK)); 3400 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3401 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3402 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3403 NUM_BANKS(ADDR_SURF_16_BANK)); 3404 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3407 NUM_BANKS(ADDR_SURF_16_BANK)); 3408 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3409 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3410 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3411 NUM_BANKS(ADDR_SURF_16_BANK)); 3412 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3413 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3414 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3415 NUM_BANKS(ADDR_SURF_16_BANK)); 3416 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3419 NUM_BANKS(ADDR_SURF_16_BANK)); 3420 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3421 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3422 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3423 NUM_BANKS(ADDR_SURF_8_BANK)); 3424 3425 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3426 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3427 reg_offset != 23) 3428 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3429 3430 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3431 if (reg_offset != 7) 3432 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3433 3434 break; 3435 } 3436 } 3437 3438 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, 3439 u32 se_num, u32 sh_num, u32 instance) 3440 { 3441 u32 data; 3442 3443 if (instance == 0xffffffff) 3444 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 3445 else 3446 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 3447 3448 if (se_num == 0xffffffff) 3449 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 3450 else 3451 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 3452 3453 if (sh_num == 0xffffffff) 3454 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 3455 else 3456 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 3457 3458 WREG32(mmGRBM_GFX_INDEX, data); 3459 } 3460 3461 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev, 3462 u32 me, u32 pipe, u32 q, u32 vm) 3463 { 3464 vi_srbm_select(adev, me, pipe, q, vm); 3465 } 3466 3467 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev) 3468 { 3469 u32 data, mask; 3470 3471 data = RREG32(mmCC_RB_BACKEND_DISABLE) | 3472 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3473 3474 data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE); 3475 3476 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 3477 adev->gfx.config.max_sh_per_se); 3478 3479 return (~data) & mask; 3480 } 3481 3482 static void 3483 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1) 3484 { 3485 switch (adev->asic_type) { 3486 case CHIP_FIJI: 3487 case CHIP_VEGAM: 3488 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) | 3489 RB_XSEL2(1) | PKR_MAP(2) | 3490 PKR_XSEL(1) | PKR_YSEL(1) | 3491 SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3); 3492 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) | 3493 SE_PAIR_YSEL(2); 3494 break; 3495 case CHIP_TONGA: 3496 case CHIP_POLARIS10: 3497 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3498 SE_XSEL(1) | SE_YSEL(1); 3499 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) | 3500 SE_PAIR_YSEL(2); 3501 break; 3502 case CHIP_TOPAZ: 3503 case CHIP_CARRIZO: 3504 *rconf |= RB_MAP_PKR0(2); 3505 *rconf1 |= 0x0; 3506 break; 3507 case CHIP_POLARIS11: 3508 case CHIP_POLARIS12: 3509 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3510 SE_XSEL(1) | SE_YSEL(1); 3511 *rconf1 |= 0x0; 3512 break; 3513 case CHIP_STONEY: 3514 *rconf |= 0x0; 3515 *rconf1 |= 0x0; 3516 break; 3517 default: 3518 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type); 3519 break; 3520 } 3521 } 3522 3523 static void 3524 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev, 3525 u32 raster_config, u32 raster_config_1, 3526 unsigned rb_mask, unsigned num_rb) 3527 { 3528 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1); 3529 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1); 3530 unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2); 3531 unsigned rb_per_se = num_rb / num_se; 3532 unsigned se_mask[4]; 3533 unsigned se; 3534 3535 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask; 3536 se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask; 3537 se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask; 3538 se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask; 3539 3540 WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4)); 3541 WARN_ON(!(sh_per_se == 1 || sh_per_se == 2)); 3542 WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2)); 3543 3544 if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) || 3545 (!se_mask[2] && !se_mask[3]))) { 3546 raster_config_1 &= ~SE_PAIR_MAP_MASK; 3547 3548 if (!se_mask[0] && !se_mask[1]) { 3549 raster_config_1 |= 3550 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3); 3551 } else { 3552 raster_config_1 |= 3553 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0); 3554 } 3555 } 3556 3557 for (se = 0; se < num_se; se++) { 3558 unsigned raster_config_se = raster_config; 3559 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se); 3560 unsigned pkr1_mask = pkr0_mask << rb_per_pkr; 3561 int idx = (se / 2) * 2; 3562 3563 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) { 3564 raster_config_se &= ~SE_MAP_MASK; 3565 3566 if (!se_mask[idx]) { 3567 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3); 3568 } else { 3569 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0); 3570 } 3571 } 3572 3573 pkr0_mask &= rb_mask; 3574 pkr1_mask &= rb_mask; 3575 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) { 3576 raster_config_se &= ~PKR_MAP_MASK; 3577 3578 if (!pkr0_mask) { 3579 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3); 3580 } else { 3581 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0); 3582 } 3583 } 3584 3585 if (rb_per_se >= 2) { 3586 unsigned rb0_mask = 1 << (se * rb_per_se); 3587 unsigned rb1_mask = rb0_mask << 1; 3588 3589 rb0_mask &= rb_mask; 3590 rb1_mask &= rb_mask; 3591 if (!rb0_mask || !rb1_mask) { 3592 raster_config_se &= ~RB_MAP_PKR0_MASK; 3593 3594 if (!rb0_mask) { 3595 raster_config_se |= 3596 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3); 3597 } else { 3598 raster_config_se |= 3599 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0); 3600 } 3601 } 3602 3603 if (rb_per_se > 2) { 3604 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr); 3605 rb1_mask = rb0_mask << 1; 3606 rb0_mask &= rb_mask; 3607 rb1_mask &= rb_mask; 3608 if (!rb0_mask || !rb1_mask) { 3609 raster_config_se &= ~RB_MAP_PKR1_MASK; 3610 3611 if (!rb0_mask) { 3612 raster_config_se |= 3613 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3); 3614 } else { 3615 raster_config_se |= 3616 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0); 3617 } 3618 } 3619 } 3620 } 3621 3622 /* GRBM_GFX_INDEX has a different offset on VI */ 3623 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff); 3624 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se); 3625 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3626 } 3627 3628 /* GRBM_GFX_INDEX has a different offset on VI */ 3629 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 3630 } 3631 3632 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev) 3633 { 3634 int i, j; 3635 u32 data; 3636 u32 raster_config = 0, raster_config_1 = 0; 3637 u32 active_rbs = 0; 3638 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 3639 adev->gfx.config.max_sh_per_se; 3640 unsigned num_rb_pipes; 3641 3642 mutex_lock(&adev->grbm_idx_mutex); 3643 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3644 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3645 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff); 3646 data = gfx_v8_0_get_rb_active_bitmap(adev); 3647 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 3648 rb_bitmap_width_per_sh); 3649 } 3650 } 3651 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 3652 3653 adev->gfx.config.backend_enable_mask = active_rbs; 3654 adev->gfx.config.num_rbs = hweight32(active_rbs); 3655 3656 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se * 3657 adev->gfx.config.max_shader_engines, 16); 3658 3659 gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1); 3660 3661 if (!adev->gfx.config.backend_enable_mask || 3662 adev->gfx.config.num_rbs >= num_rb_pipes) { 3663 WREG32(mmPA_SC_RASTER_CONFIG, raster_config); 3664 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3665 } else { 3666 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1, 3667 adev->gfx.config.backend_enable_mask, 3668 num_rb_pipes); 3669 } 3670 3671 /* cache the values for userspace */ 3672 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3673 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3674 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff); 3675 adev->gfx.config.rb_config[i][j].rb_backend_disable = 3676 RREG32(mmCC_RB_BACKEND_DISABLE); 3677 adev->gfx.config.rb_config[i][j].user_rb_backend_disable = 3678 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3679 adev->gfx.config.rb_config[i][j].raster_config = 3680 RREG32(mmPA_SC_RASTER_CONFIG); 3681 adev->gfx.config.rb_config[i][j].raster_config_1 = 3682 RREG32(mmPA_SC_RASTER_CONFIG_1); 3683 } 3684 } 3685 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 3686 mutex_unlock(&adev->grbm_idx_mutex); 3687 } 3688 3689 /** 3690 * gfx_v8_0_init_compute_vmid - gart enable 3691 * 3692 * @adev: amdgpu_device pointer 3693 * 3694 * Initialize compute vmid sh_mem registers 3695 * 3696 */ 3697 #define DEFAULT_SH_MEM_BASES (0x6000) 3698 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev) 3699 { 3700 int i; 3701 uint32_t sh_mem_config; 3702 uint32_t sh_mem_bases; 3703 3704 /* 3705 * Configure apertures: 3706 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 3707 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 3708 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 3709 */ 3710 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 3711 3712 sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 << 3713 SH_MEM_CONFIG__ADDRESS_MODE__SHIFT | 3714 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 3715 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT | 3716 MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | 3717 SH_MEM_CONFIG__PRIVATE_ATC_MASK; 3718 3719 mutex_lock(&adev->srbm_mutex); 3720 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3721 vi_srbm_select(adev, 0, 0, 0, i); 3722 /* CP and shaders */ 3723 WREG32(mmSH_MEM_CONFIG, sh_mem_config); 3724 WREG32(mmSH_MEM_APE1_BASE, 1); 3725 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3726 WREG32(mmSH_MEM_BASES, sh_mem_bases); 3727 } 3728 vi_srbm_select(adev, 0, 0, 0, 0); 3729 mutex_unlock(&adev->srbm_mutex); 3730 3731 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 3732 acccess. These should be enabled by FW for target VMIDs. */ 3733 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3734 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0); 3735 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0); 3736 WREG32(amdgpu_gds_reg_offset[i].gws, 0); 3737 WREG32(amdgpu_gds_reg_offset[i].oa, 0); 3738 } 3739 } 3740 3741 static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev) 3742 { 3743 int vmid; 3744 3745 /* 3746 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 3747 * access. Compute VMIDs should be enabled by FW for target VMIDs, 3748 * the driver can enable them for graphics. VMID0 should maintain 3749 * access so that HWS firmware can save/restore entries. 3750 */ 3751 for (vmid = 1; vmid < 16; vmid++) { 3752 WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0); 3753 WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0); 3754 WREG32(amdgpu_gds_reg_offset[vmid].gws, 0); 3755 WREG32(amdgpu_gds_reg_offset[vmid].oa, 0); 3756 } 3757 } 3758 3759 static void gfx_v8_0_config_init(struct amdgpu_device *adev) 3760 { 3761 switch (adev->asic_type) { 3762 default: 3763 adev->gfx.config.double_offchip_lds_buf = 1; 3764 break; 3765 case CHIP_CARRIZO: 3766 case CHIP_STONEY: 3767 adev->gfx.config.double_offchip_lds_buf = 0; 3768 break; 3769 } 3770 } 3771 3772 static void gfx_v8_0_constants_init(struct amdgpu_device *adev) 3773 { 3774 u32 tmp, sh_static_mem_cfg; 3775 int i; 3776 3777 WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF); 3778 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3779 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3780 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config); 3781 3782 gfx_v8_0_tiling_mode_table_init(adev); 3783 gfx_v8_0_setup_rb(adev); 3784 gfx_v8_0_get_cu_info(adev); 3785 gfx_v8_0_config_init(adev); 3786 3787 /* XXX SH_MEM regs */ 3788 /* where to put LDS, scratch, GPUVM in FSA64 space */ 3789 sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG, 3790 SWIZZLE_ENABLE, 1); 3791 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3792 ELEMENT_SIZE, 1); 3793 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3794 INDEX_STRIDE, 3); 3795 WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); 3796 3797 mutex_lock(&adev->srbm_mutex); 3798 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { 3799 vi_srbm_select(adev, 0, 0, 0, i); 3800 /* CP and shaders */ 3801 if (i == 0) { 3802 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC); 3803 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3804 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3805 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3806 WREG32(mmSH_MEM_CONFIG, tmp); 3807 WREG32(mmSH_MEM_BASES, 0); 3808 } else { 3809 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC); 3810 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3811 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3812 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3813 WREG32(mmSH_MEM_CONFIG, tmp); 3814 tmp = adev->gmc.shared_aperture_start >> 48; 3815 WREG32(mmSH_MEM_BASES, tmp); 3816 } 3817 3818 WREG32(mmSH_MEM_APE1_BASE, 1); 3819 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3820 } 3821 vi_srbm_select(adev, 0, 0, 0, 0); 3822 mutex_unlock(&adev->srbm_mutex); 3823 3824 gfx_v8_0_init_compute_vmid(adev); 3825 gfx_v8_0_init_gds_vmid(adev); 3826 3827 mutex_lock(&adev->grbm_idx_mutex); 3828 /* 3829 * making sure that the following register writes will be broadcasted 3830 * to all the shaders 3831 */ 3832 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 3833 3834 WREG32(mmPA_SC_FIFO_SIZE, 3835 (adev->gfx.config.sc_prim_fifo_size_frontend << 3836 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) | 3837 (adev->gfx.config.sc_prim_fifo_size_backend << 3838 PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) | 3839 (adev->gfx.config.sc_hiz_tile_fifo_size << 3840 PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) | 3841 (adev->gfx.config.sc_earlyz_tile_fifo_size << 3842 PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)); 3843 3844 tmp = RREG32(mmSPI_ARB_PRIORITY); 3845 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2); 3846 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2); 3847 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2); 3848 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2); 3849 WREG32(mmSPI_ARB_PRIORITY, tmp); 3850 3851 mutex_unlock(&adev->grbm_idx_mutex); 3852 3853 } 3854 3855 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 3856 { 3857 u32 i, j, k; 3858 u32 mask; 3859 3860 mutex_lock(&adev->grbm_idx_mutex); 3861 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3862 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3863 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff); 3864 for (k = 0; k < adev->usec_timeout; k++) { 3865 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0) 3866 break; 3867 udelay(1); 3868 } 3869 if (k == adev->usec_timeout) { 3870 gfx_v8_0_select_se_sh(adev, 0xffffffff, 3871 0xffffffff, 0xffffffff); 3872 mutex_unlock(&adev->grbm_idx_mutex); 3873 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 3874 i, j); 3875 return; 3876 } 3877 } 3878 } 3879 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 3880 mutex_unlock(&adev->grbm_idx_mutex); 3881 3882 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 3883 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 3884 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 3885 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 3886 for (k = 0; k < adev->usec_timeout; k++) { 3887 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 3888 break; 3889 udelay(1); 3890 } 3891 } 3892 3893 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 3894 bool enable) 3895 { 3896 u32 tmp = RREG32(mmCP_INT_CNTL_RING0); 3897 3898 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 3899 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 3900 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 3901 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 3902 3903 WREG32(mmCP_INT_CNTL_RING0, tmp); 3904 } 3905 3906 static void gfx_v8_0_init_csb(struct amdgpu_device *adev) 3907 { 3908 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 3909 /* csib */ 3910 WREG32(mmRLC_CSIB_ADDR_HI, 3911 adev->gfx.rlc.clear_state_gpu_addr >> 32); 3912 WREG32(mmRLC_CSIB_ADDR_LO, 3913 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 3914 WREG32(mmRLC_CSIB_LENGTH, 3915 adev->gfx.rlc.clear_state_size); 3916 } 3917 3918 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format, 3919 int ind_offset, 3920 int list_size, 3921 int *unique_indices, 3922 int *indices_count, 3923 int max_indices, 3924 int *ind_start_offsets, 3925 int *offset_count, 3926 int max_offset) 3927 { 3928 int indices; 3929 bool new_entry = true; 3930 3931 for (; ind_offset < list_size; ind_offset++) { 3932 3933 if (new_entry) { 3934 new_entry = false; 3935 ind_start_offsets[*offset_count] = ind_offset; 3936 *offset_count = *offset_count + 1; 3937 BUG_ON(*offset_count >= max_offset); 3938 } 3939 3940 if (register_list_format[ind_offset] == 0xFFFFFFFF) { 3941 new_entry = true; 3942 continue; 3943 } 3944 3945 ind_offset += 2; 3946 3947 /* look for the matching indice */ 3948 for (indices = 0; 3949 indices < *indices_count; 3950 indices++) { 3951 if (unique_indices[indices] == 3952 register_list_format[ind_offset]) 3953 break; 3954 } 3955 3956 if (indices >= *indices_count) { 3957 unique_indices[*indices_count] = 3958 register_list_format[ind_offset]; 3959 indices = *indices_count; 3960 *indices_count = *indices_count + 1; 3961 BUG_ON(*indices_count >= max_indices); 3962 } 3963 3964 register_list_format[ind_offset] = indices; 3965 } 3966 } 3967 3968 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev) 3969 { 3970 int i, temp, data; 3971 int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0}; 3972 int indices_count = 0; 3973 int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 3974 int offset_count = 0; 3975 3976 int list_size; 3977 unsigned int *register_list_format = 3978 kmemdup(adev->gfx.rlc.register_list_format, 3979 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 3980 if (!register_list_format) 3981 return -ENOMEM; 3982 3983 gfx_v8_0_parse_ind_reg_list(register_list_format, 3984 RLC_FormatDirectRegListLength, 3985 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 3986 unique_indices, 3987 &indices_count, 3988 ARRAY_SIZE(unique_indices), 3989 indirect_start_offsets, 3990 &offset_count, 3991 ARRAY_SIZE(indirect_start_offsets)); 3992 3993 /* save and restore list */ 3994 WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1); 3995 3996 WREG32(mmRLC_SRM_ARAM_ADDR, 0); 3997 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 3998 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]); 3999 4000 /* indirect list */ 4001 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start); 4002 for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++) 4003 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]); 4004 4005 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 4006 list_size = list_size >> 1; 4007 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size); 4008 WREG32(mmRLC_GPM_SCRATCH_DATA, list_size); 4009 4010 /* starting offsets starts */ 4011 WREG32(mmRLC_GPM_SCRATCH_ADDR, 4012 adev->gfx.rlc.starting_offsets_start); 4013 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 4014 WREG32(mmRLC_GPM_SCRATCH_DATA, 4015 indirect_start_offsets[i]); 4016 4017 /* unique indices */ 4018 temp = mmRLC_SRM_INDEX_CNTL_ADDR_0; 4019 data = mmRLC_SRM_INDEX_CNTL_DATA_0; 4020 for (i = 0; i < ARRAY_SIZE(unique_indices); i++) { 4021 if (unique_indices[i] != 0) { 4022 WREG32(temp + i, unique_indices[i] & 0x3FFFF); 4023 WREG32(data + i, unique_indices[i] >> 20); 4024 } 4025 } 4026 kfree(register_list_format); 4027 4028 return 0; 4029 } 4030 4031 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev) 4032 { 4033 WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1); 4034 } 4035 4036 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev) 4037 { 4038 uint32_t data; 4039 4040 WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60); 4041 4042 data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10); 4043 data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10); 4044 data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10); 4045 data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10); 4046 WREG32(mmRLC_PG_DELAY, data); 4047 4048 WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3); 4049 WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0); 4050 4051 } 4052 4053 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 4054 bool enable) 4055 { 4056 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0); 4057 } 4058 4059 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 4060 bool enable) 4061 { 4062 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0); 4063 } 4064 4065 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable) 4066 { 4067 WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1); 4068 } 4069 4070 static void gfx_v8_0_init_pg(struct amdgpu_device *adev) 4071 { 4072 if ((adev->asic_type == CHIP_CARRIZO) || 4073 (adev->asic_type == CHIP_STONEY)) { 4074 gfx_v8_0_init_csb(adev); 4075 gfx_v8_0_init_save_restore_list(adev); 4076 gfx_v8_0_enable_save_restore_machine(adev); 4077 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8); 4078 gfx_v8_0_init_power_gating(adev); 4079 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask); 4080 } else if ((adev->asic_type == CHIP_POLARIS11) || 4081 (adev->asic_type == CHIP_POLARIS12) || 4082 (adev->asic_type == CHIP_VEGAM)) { 4083 gfx_v8_0_init_csb(adev); 4084 gfx_v8_0_init_save_restore_list(adev); 4085 gfx_v8_0_enable_save_restore_machine(adev); 4086 gfx_v8_0_init_power_gating(adev); 4087 } 4088 4089 } 4090 4091 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev) 4092 { 4093 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0); 4094 4095 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4096 gfx_v8_0_wait_for_rlc_serdes(adev); 4097 } 4098 4099 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev) 4100 { 4101 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4102 udelay(50); 4103 4104 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 4105 udelay(50); 4106 } 4107 4108 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev) 4109 { 4110 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1); 4111 4112 /* carrizo do enable cp interrupt after cp inited */ 4113 if (!(adev->flags & AMD_IS_APU)) 4114 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4115 4116 udelay(50); 4117 } 4118 4119 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev) 4120 { 4121 if (amdgpu_sriov_vf(adev)) { 4122 gfx_v8_0_init_csb(adev); 4123 return 0; 4124 } 4125 4126 adev->gfx.rlc.funcs->stop(adev); 4127 adev->gfx.rlc.funcs->reset(adev); 4128 gfx_v8_0_init_pg(adev); 4129 adev->gfx.rlc.funcs->start(adev); 4130 4131 return 0; 4132 } 4133 4134 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 4135 { 4136 u32 tmp = RREG32(mmCP_ME_CNTL); 4137 4138 if (enable) { 4139 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0); 4140 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0); 4141 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0); 4142 } else { 4143 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1); 4144 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1); 4145 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1); 4146 } 4147 WREG32(mmCP_ME_CNTL, tmp); 4148 udelay(50); 4149 } 4150 4151 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev) 4152 { 4153 u32 count = 0; 4154 const struct cs_section_def *sect = NULL; 4155 const struct cs_extent_def *ext = NULL; 4156 4157 /* begin clear state */ 4158 count += 2; 4159 /* context control state */ 4160 count += 3; 4161 4162 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4163 for (ext = sect->section; ext->extent != NULL; ++ext) { 4164 if (sect->id == SECT_CONTEXT) 4165 count += 2 + ext->reg_count; 4166 else 4167 return 0; 4168 } 4169 } 4170 /* pa_sc_raster_config/pa_sc_raster_config1 */ 4171 count += 4; 4172 /* end clear state */ 4173 count += 2; 4174 /* clear state */ 4175 count += 2; 4176 4177 return count; 4178 } 4179 4180 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev) 4181 { 4182 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 4183 const struct cs_section_def *sect = NULL; 4184 const struct cs_extent_def *ext = NULL; 4185 int r, i; 4186 4187 /* init the CP */ 4188 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 4189 WREG32(mmCP_ENDIAN_SWAP, 0); 4190 WREG32(mmCP_DEVICE_ID, 1); 4191 4192 gfx_v8_0_cp_gfx_enable(adev, true); 4193 4194 r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4); 4195 if (r) { 4196 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 4197 return r; 4198 } 4199 4200 /* clear state buffer */ 4201 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4202 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4203 4204 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4205 amdgpu_ring_write(ring, 0x80000000); 4206 amdgpu_ring_write(ring, 0x80000000); 4207 4208 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4209 for (ext = sect->section; ext->extent != NULL; ++ext) { 4210 if (sect->id == SECT_CONTEXT) { 4211 amdgpu_ring_write(ring, 4212 PACKET3(PACKET3_SET_CONTEXT_REG, 4213 ext->reg_count)); 4214 amdgpu_ring_write(ring, 4215 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 4216 for (i = 0; i < ext->reg_count; i++) 4217 amdgpu_ring_write(ring, ext->extent[i]); 4218 } 4219 } 4220 } 4221 4222 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4223 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 4224 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config); 4225 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1); 4226 4227 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4228 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4229 4230 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4231 amdgpu_ring_write(ring, 0); 4232 4233 /* init the CE partitions */ 4234 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4235 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4236 amdgpu_ring_write(ring, 0x8000); 4237 amdgpu_ring_write(ring, 0x8000); 4238 4239 amdgpu_ring_commit(ring); 4240 4241 return 0; 4242 } 4243 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring) 4244 { 4245 u32 tmp; 4246 /* no gfx doorbells on iceland */ 4247 if (adev->asic_type == CHIP_TOPAZ) 4248 return; 4249 4250 tmp = RREG32(mmCP_RB_DOORBELL_CONTROL); 4251 4252 if (ring->use_doorbell) { 4253 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4254 DOORBELL_OFFSET, ring->doorbell_index); 4255 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4256 DOORBELL_HIT, 0); 4257 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4258 DOORBELL_EN, 1); 4259 } else { 4260 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 4261 } 4262 4263 WREG32(mmCP_RB_DOORBELL_CONTROL, tmp); 4264 4265 if (adev->flags & AMD_IS_APU) 4266 return; 4267 4268 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 4269 DOORBELL_RANGE_LOWER, 4270 adev->doorbell_index.gfx_ring0); 4271 WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 4272 4273 WREG32(mmCP_RB_DOORBELL_RANGE_UPPER, 4274 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 4275 } 4276 4277 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev) 4278 { 4279 struct amdgpu_ring *ring; 4280 u32 tmp; 4281 u32 rb_bufsz; 4282 u64 rb_addr, rptr_addr, wptr_gpu_addr; 4283 4284 /* Set the write pointer delay */ 4285 WREG32(mmCP_RB_WPTR_DELAY, 0); 4286 4287 /* set the RB to use vmid 0 */ 4288 WREG32(mmCP_RB_VMID, 0); 4289 4290 /* Set ring buffer size */ 4291 ring = &adev->gfx.gfx_ring[0]; 4292 rb_bufsz = order_base_2(ring->ring_size / 8); 4293 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 4294 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 4295 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3); 4296 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1); 4297 #ifdef __BIG_ENDIAN 4298 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 4299 #endif 4300 WREG32(mmCP_RB0_CNTL, tmp); 4301 4302 /* Initialize the ring buffer's read and write pointers */ 4303 WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK); 4304 ring->wptr = 0; 4305 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 4306 4307 /* set the wb address wether it's enabled or not */ 4308 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 4309 WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 4310 WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF); 4311 4312 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 4313 WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 4314 WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 4315 mdelay(1); 4316 WREG32(mmCP_RB0_CNTL, tmp); 4317 4318 rb_addr = ring->gpu_addr >> 8; 4319 WREG32(mmCP_RB0_BASE, rb_addr); 4320 WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4321 4322 gfx_v8_0_set_cpg_door_bell(adev, ring); 4323 /* start the ring */ 4324 amdgpu_ring_clear_ring(ring); 4325 gfx_v8_0_cp_gfx_start(adev); 4326 ring->sched.ready = true; 4327 4328 return 0; 4329 } 4330 4331 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 4332 { 4333 if (enable) { 4334 WREG32(mmCP_MEC_CNTL, 0); 4335 } else { 4336 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 4337 adev->gfx.kiq.ring.sched.ready = false; 4338 } 4339 udelay(50); 4340 } 4341 4342 /* KIQ functions */ 4343 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring) 4344 { 4345 uint32_t tmp; 4346 struct amdgpu_device *adev = ring->adev; 4347 4348 /* tell RLC which is KIQ queue */ 4349 tmp = RREG32(mmRLC_CP_SCHEDULERS); 4350 tmp &= 0xffffff00; 4351 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4352 WREG32(mmRLC_CP_SCHEDULERS, tmp); 4353 tmp |= 0x80; 4354 WREG32(mmRLC_CP_SCHEDULERS, tmp); 4355 } 4356 4357 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev) 4358 { 4359 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; 4360 uint64_t queue_mask = 0; 4361 int r, i; 4362 4363 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 4364 if (!test_bit(i, adev->gfx.mec.queue_bitmap)) 4365 continue; 4366 4367 /* This situation may be hit in the future if a new HW 4368 * generation exposes more than 64 queues. If so, the 4369 * definition of queue_mask needs updating */ 4370 if (WARN_ON(i >= (sizeof(queue_mask)*8))) { 4371 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 4372 break; 4373 } 4374 4375 queue_mask |= (1ull << i); 4376 } 4377 4378 r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8); 4379 if (r) { 4380 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4381 return r; 4382 } 4383 /* set resources */ 4384 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 4385 amdgpu_ring_write(kiq_ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */ 4386 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 4387 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 4388 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 4389 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 4390 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 4391 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 4392 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4393 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4394 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 4395 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 4396 4397 /* map queues */ 4398 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 4399 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 4400 amdgpu_ring_write(kiq_ring, 4401 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 4402 amdgpu_ring_write(kiq_ring, 4403 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) | 4404 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 4405 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 4406 PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */ 4407 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 4408 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 4409 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 4410 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 4411 } 4412 4413 amdgpu_ring_commit(kiq_ring); 4414 4415 return 0; 4416 } 4417 4418 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req) 4419 { 4420 int i, r = 0; 4421 4422 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) { 4423 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req); 4424 for (i = 0; i < adev->usec_timeout; i++) { 4425 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK)) 4426 break; 4427 udelay(1); 4428 } 4429 if (i == adev->usec_timeout) 4430 r = -ETIMEDOUT; 4431 } 4432 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0); 4433 WREG32(mmCP_HQD_PQ_RPTR, 0); 4434 WREG32(mmCP_HQD_PQ_WPTR, 0); 4435 4436 return r; 4437 } 4438 4439 static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd) 4440 { 4441 struct amdgpu_device *adev = ring->adev; 4442 4443 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 4444 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue)) { 4445 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 4446 mqd->cp_hqd_queue_priority = 4447 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 4448 } 4449 } 4450 } 4451 4452 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring) 4453 { 4454 struct amdgpu_device *adev = ring->adev; 4455 struct vi_mqd *mqd = ring->mqd_ptr; 4456 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4457 uint32_t tmp; 4458 4459 mqd->header = 0xC0310800; 4460 mqd->compute_pipelinestat_enable = 0x00000001; 4461 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4462 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4463 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4464 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4465 mqd->compute_misc_reserved = 0x00000003; 4466 mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr 4467 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4468 mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr 4469 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4470 eop_base_addr = ring->eop_gpu_addr >> 8; 4471 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4472 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4473 4474 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4475 tmp = RREG32(mmCP_HQD_EOP_CONTROL); 4476 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4477 (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1)); 4478 4479 mqd->cp_hqd_eop_control = tmp; 4480 4481 /* enable doorbell? */ 4482 tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL), 4483 CP_HQD_PQ_DOORBELL_CONTROL, 4484 DOORBELL_EN, 4485 ring->use_doorbell ? 1 : 0); 4486 4487 mqd->cp_hqd_pq_doorbell_control = tmp; 4488 4489 /* set the pointer to the MQD */ 4490 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 4491 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 4492 4493 /* set MQD vmid to 0 */ 4494 tmp = RREG32(mmCP_MQD_CONTROL); 4495 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4496 mqd->cp_mqd_control = tmp; 4497 4498 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4499 hqd_gpu_addr = ring->gpu_addr >> 8; 4500 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4501 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4502 4503 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4504 tmp = RREG32(mmCP_HQD_PQ_CONTROL); 4505 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4506 (order_base_2(ring->ring_size / 4) - 1)); 4507 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4508 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 4509 #ifdef __BIG_ENDIAN 4510 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 4511 #endif 4512 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 4513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 4514 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4515 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4516 mqd->cp_hqd_pq_control = tmp; 4517 4518 /* set the wb address whether it's enabled or not */ 4519 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 4520 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4521 mqd->cp_hqd_pq_rptr_report_addr_hi = 4522 upper_32_bits(wb_gpu_addr) & 0xffff; 4523 4524 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4525 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 4526 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4527 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4528 4529 tmp = 0; 4530 /* enable the doorbell if requested */ 4531 if (ring->use_doorbell) { 4532 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL); 4533 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4534 DOORBELL_OFFSET, ring->doorbell_index); 4535 4536 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4537 DOORBELL_EN, 1); 4538 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4539 DOORBELL_SOURCE, 0); 4540 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4541 DOORBELL_HIT, 0); 4542 } 4543 4544 mqd->cp_hqd_pq_doorbell_control = tmp; 4545 4546 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4547 ring->wptr = 0; 4548 mqd->cp_hqd_pq_wptr = ring->wptr; 4549 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); 4550 4551 /* set the vmid for the queue */ 4552 mqd->cp_hqd_vmid = 0; 4553 4554 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE); 4555 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 4556 mqd->cp_hqd_persistent_state = tmp; 4557 4558 /* set MTYPE */ 4559 tmp = RREG32(mmCP_HQD_IB_CONTROL); 4560 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4561 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3); 4562 mqd->cp_hqd_ib_control = tmp; 4563 4564 tmp = RREG32(mmCP_HQD_IQ_TIMER); 4565 tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3); 4566 mqd->cp_hqd_iq_timer = tmp; 4567 4568 tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL); 4569 tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3); 4570 mqd->cp_hqd_ctx_save_control = tmp; 4571 4572 /* defaults */ 4573 mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR); 4574 mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR); 4575 mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO); 4576 mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI); 4577 mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET); 4578 mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE); 4579 mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET); 4580 mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE); 4581 mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS); 4582 mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR); 4583 mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM); 4584 mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES); 4585 4586 /* set static priority for a queue/ring */ 4587 gfx_v8_0_mqd_set_priority(ring, mqd); 4588 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 4589 4590 /* map_queues packet doesn't need activate the queue, 4591 * so only kiq need set this field. 4592 */ 4593 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 4594 mqd->cp_hqd_active = 1; 4595 4596 return 0; 4597 } 4598 4599 static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev, 4600 struct vi_mqd *mqd) 4601 { 4602 uint32_t mqd_reg; 4603 uint32_t *mqd_data; 4604 4605 /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */ 4606 mqd_data = &mqd->cp_mqd_base_addr_lo; 4607 4608 /* disable wptr polling */ 4609 WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0); 4610 4611 /* program all HQD registers */ 4612 for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++) 4613 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4614 4615 /* Tonga errata: EOP RPTR/WPTR should be left unmodified. 4616 * This is safe since EOP RPTR==WPTR for any inactive HQD 4617 * on ASICs that do not support context-save. 4618 * EOP writes/reads can start anywhere in the ring. 4619 */ 4620 if (adev->asic_type != CHIP_TONGA) { 4621 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr); 4622 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr); 4623 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem); 4624 } 4625 4626 for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++) 4627 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4628 4629 /* activate the HQD */ 4630 for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++) 4631 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4632 4633 return 0; 4634 } 4635 4636 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) 4637 { 4638 struct amdgpu_device *adev = ring->adev; 4639 struct vi_mqd *mqd = ring->mqd_ptr; 4640 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 4641 4642 gfx_v8_0_kiq_setting(ring); 4643 4644 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4645 /* reset MQD to a clean status */ 4646 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4647 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); 4648 4649 /* reset ring buffer */ 4650 ring->wptr = 0; 4651 amdgpu_ring_clear_ring(ring); 4652 mutex_lock(&adev->srbm_mutex); 4653 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4654 gfx_v8_0_mqd_commit(adev, mqd); 4655 vi_srbm_select(adev, 0, 0, 0, 0); 4656 mutex_unlock(&adev->srbm_mutex); 4657 } else { 4658 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4659 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4660 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4661 mutex_lock(&adev->srbm_mutex); 4662 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4663 gfx_v8_0_mqd_init(ring); 4664 gfx_v8_0_mqd_commit(adev, mqd); 4665 vi_srbm_select(adev, 0, 0, 0, 0); 4666 mutex_unlock(&adev->srbm_mutex); 4667 4668 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4669 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); 4670 } 4671 4672 return 0; 4673 } 4674 4675 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) 4676 { 4677 struct amdgpu_device *adev = ring->adev; 4678 struct vi_mqd *mqd = ring->mqd_ptr; 4679 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4680 4681 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4682 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4683 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4684 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4685 mutex_lock(&adev->srbm_mutex); 4686 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4687 gfx_v8_0_mqd_init(ring); 4688 vi_srbm_select(adev, 0, 0, 0, 0); 4689 mutex_unlock(&adev->srbm_mutex); 4690 4691 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4692 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); 4693 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4694 /* reset MQD to a clean status */ 4695 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4696 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); 4697 /* reset ring buffer */ 4698 ring->wptr = 0; 4699 amdgpu_ring_clear_ring(ring); 4700 } else { 4701 amdgpu_ring_clear_ring(ring); 4702 } 4703 return 0; 4704 } 4705 4706 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev) 4707 { 4708 if (adev->asic_type > CHIP_TONGA) { 4709 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2); 4710 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2); 4711 } 4712 /* enable doorbells */ 4713 WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4714 } 4715 4716 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev) 4717 { 4718 struct amdgpu_ring *ring; 4719 int r; 4720 4721 ring = &adev->gfx.kiq.ring; 4722 4723 r = amdgpu_bo_reserve(ring->mqd_obj, false); 4724 if (unlikely(r != 0)) 4725 return r; 4726 4727 r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr); 4728 if (unlikely(r != 0)) 4729 return r; 4730 4731 gfx_v8_0_kiq_init_queue(ring); 4732 amdgpu_bo_kunmap(ring->mqd_obj); 4733 ring->mqd_ptr = NULL; 4734 amdgpu_bo_unreserve(ring->mqd_obj); 4735 ring->sched.ready = true; 4736 return 0; 4737 } 4738 4739 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev) 4740 { 4741 struct amdgpu_ring *ring = NULL; 4742 int r = 0, i; 4743 4744 gfx_v8_0_cp_compute_enable(adev, true); 4745 4746 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4747 ring = &adev->gfx.compute_ring[i]; 4748 4749 r = amdgpu_bo_reserve(ring->mqd_obj, false); 4750 if (unlikely(r != 0)) 4751 goto done; 4752 r = amdgpu_bo_kmap(ring->mqd_obj, &ring->mqd_ptr); 4753 if (!r) { 4754 r = gfx_v8_0_kcq_init_queue(ring); 4755 amdgpu_bo_kunmap(ring->mqd_obj); 4756 ring->mqd_ptr = NULL; 4757 } 4758 amdgpu_bo_unreserve(ring->mqd_obj); 4759 if (r) 4760 goto done; 4761 } 4762 4763 gfx_v8_0_set_mec_doorbell_range(adev); 4764 4765 r = gfx_v8_0_kiq_kcq_enable(adev); 4766 if (r) 4767 goto done; 4768 4769 done: 4770 return r; 4771 } 4772 4773 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev) 4774 { 4775 int r, i; 4776 struct amdgpu_ring *ring; 4777 4778 /* collect all the ring_tests here, gfx, kiq, compute */ 4779 ring = &adev->gfx.gfx_ring[0]; 4780 r = amdgpu_ring_test_helper(ring); 4781 if (r) 4782 return r; 4783 4784 ring = &adev->gfx.kiq.ring; 4785 r = amdgpu_ring_test_helper(ring); 4786 if (r) 4787 return r; 4788 4789 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4790 ring = &adev->gfx.compute_ring[i]; 4791 amdgpu_ring_test_helper(ring); 4792 } 4793 4794 return 0; 4795 } 4796 4797 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev) 4798 { 4799 int r; 4800 4801 if (!(adev->flags & AMD_IS_APU)) 4802 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4803 4804 r = gfx_v8_0_kiq_resume(adev); 4805 if (r) 4806 return r; 4807 4808 r = gfx_v8_0_cp_gfx_resume(adev); 4809 if (r) 4810 return r; 4811 4812 r = gfx_v8_0_kcq_resume(adev); 4813 if (r) 4814 return r; 4815 4816 r = gfx_v8_0_cp_test_all_rings(adev); 4817 if (r) 4818 return r; 4819 4820 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4821 4822 return 0; 4823 } 4824 4825 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable) 4826 { 4827 gfx_v8_0_cp_gfx_enable(adev, enable); 4828 gfx_v8_0_cp_compute_enable(adev, enable); 4829 } 4830 4831 static int gfx_v8_0_hw_init(void *handle) 4832 { 4833 int r; 4834 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4835 4836 gfx_v8_0_init_golden_registers(adev); 4837 gfx_v8_0_constants_init(adev); 4838 4839 r = adev->gfx.rlc.funcs->resume(adev); 4840 if (r) 4841 return r; 4842 4843 r = gfx_v8_0_cp_resume(adev); 4844 4845 return r; 4846 } 4847 4848 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev) 4849 { 4850 int r, i; 4851 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq.ring; 4852 4853 r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings); 4854 if (r) 4855 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4856 4857 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4858 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4859 4860 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 4861 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 4862 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */ 4863 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 4864 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) | 4865 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 4866 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 4867 amdgpu_ring_write(kiq_ring, 0); 4868 amdgpu_ring_write(kiq_ring, 0); 4869 amdgpu_ring_write(kiq_ring, 0); 4870 } 4871 r = amdgpu_ring_test_helper(kiq_ring); 4872 if (r) 4873 DRM_ERROR("KCQ disable failed\n"); 4874 4875 return r; 4876 } 4877 4878 static bool gfx_v8_0_is_idle(void *handle) 4879 { 4880 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4881 4882 if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE) 4883 || RREG32(mmGRBM_STATUS2) != 0x8) 4884 return false; 4885 else 4886 return true; 4887 } 4888 4889 static bool gfx_v8_0_rlc_is_idle(void *handle) 4890 { 4891 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4892 4893 if (RREG32(mmGRBM_STATUS2) != 0x8) 4894 return false; 4895 else 4896 return true; 4897 } 4898 4899 static int gfx_v8_0_wait_for_rlc_idle(void *handle) 4900 { 4901 unsigned int i; 4902 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4903 4904 for (i = 0; i < adev->usec_timeout; i++) { 4905 if (gfx_v8_0_rlc_is_idle(handle)) 4906 return 0; 4907 4908 udelay(1); 4909 } 4910 return -ETIMEDOUT; 4911 } 4912 4913 static int gfx_v8_0_wait_for_idle(void *handle) 4914 { 4915 unsigned int i; 4916 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4917 4918 for (i = 0; i < adev->usec_timeout; i++) { 4919 if (gfx_v8_0_is_idle(handle)) 4920 return 0; 4921 4922 udelay(1); 4923 } 4924 return -ETIMEDOUT; 4925 } 4926 4927 static int gfx_v8_0_hw_fini(void *handle) 4928 { 4929 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4930 4931 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4932 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4933 4934 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 4935 4936 amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0); 4937 4938 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4939 gfx_v8_0_kcq_disable(adev); 4940 4941 if (amdgpu_sriov_vf(adev)) { 4942 pr_debug("For SRIOV client, shouldn't do anything.\n"); 4943 return 0; 4944 } 4945 amdgpu_gfx_rlc_enter_safe_mode(adev); 4946 if (!gfx_v8_0_wait_for_idle(adev)) 4947 gfx_v8_0_cp_enable(adev, false); 4948 else 4949 pr_err("cp is busy, skip halt cp\n"); 4950 if (!gfx_v8_0_wait_for_rlc_idle(adev)) 4951 adev->gfx.rlc.funcs->stop(adev); 4952 else 4953 pr_err("rlc is busy, skip halt rlc\n"); 4954 amdgpu_gfx_rlc_exit_safe_mode(adev); 4955 4956 return 0; 4957 } 4958 4959 static int gfx_v8_0_suspend(void *handle) 4960 { 4961 return gfx_v8_0_hw_fini(handle); 4962 } 4963 4964 static int gfx_v8_0_resume(void *handle) 4965 { 4966 return gfx_v8_0_hw_init(handle); 4967 } 4968 4969 static bool gfx_v8_0_check_soft_reset(void *handle) 4970 { 4971 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4972 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 4973 u32 tmp; 4974 4975 /* GRBM_STATUS */ 4976 tmp = RREG32(mmGRBM_STATUS); 4977 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4978 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4979 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4980 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4981 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4982 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK | 4983 GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4984 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4985 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4986 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4987 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4988 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, 4989 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1); 4990 } 4991 4992 /* GRBM_STATUS2 */ 4993 tmp = RREG32(mmGRBM_STATUS2); 4994 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4995 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4996 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4997 4998 if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) || 4999 REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) || 5000 REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) { 5001 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5002 SOFT_RESET_CPF, 1); 5003 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5004 SOFT_RESET_CPC, 1); 5005 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5006 SOFT_RESET_CPG, 1); 5007 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, 5008 SOFT_RESET_GRBM, 1); 5009 } 5010 5011 /* SRBM_STATUS */ 5012 tmp = RREG32(mmSRBM_STATUS); 5013 if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING)) 5014 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, 5015 SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1); 5016 if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY)) 5017 srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, 5018 SRBM_SOFT_RESET, SOFT_RESET_SEM, 1); 5019 5020 if (grbm_soft_reset || srbm_soft_reset) { 5021 adev->gfx.grbm_soft_reset = grbm_soft_reset; 5022 adev->gfx.srbm_soft_reset = srbm_soft_reset; 5023 return true; 5024 } else { 5025 adev->gfx.grbm_soft_reset = 0; 5026 adev->gfx.srbm_soft_reset = 0; 5027 return false; 5028 } 5029 } 5030 5031 static int gfx_v8_0_pre_soft_reset(void *handle) 5032 { 5033 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5034 u32 grbm_soft_reset = 0; 5035 5036 if ((!adev->gfx.grbm_soft_reset) && 5037 (!adev->gfx.srbm_soft_reset)) 5038 return 0; 5039 5040 grbm_soft_reset = adev->gfx.grbm_soft_reset; 5041 5042 /* stop the rlc */ 5043 adev->gfx.rlc.funcs->stop(adev); 5044 5045 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || 5046 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX)) 5047 /* Disable GFX parsing/prefetching */ 5048 gfx_v8_0_cp_gfx_enable(adev, false); 5049 5050 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || 5051 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) || 5052 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) || 5053 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) { 5054 int i; 5055 5056 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5057 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 5058 5059 mutex_lock(&adev->srbm_mutex); 5060 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 5061 gfx_v8_0_deactivate_hqd(adev, 2); 5062 vi_srbm_select(adev, 0, 0, 0, 0); 5063 mutex_unlock(&adev->srbm_mutex); 5064 } 5065 /* Disable MEC parsing/prefetching */ 5066 gfx_v8_0_cp_compute_enable(adev, false); 5067 } 5068 5069 return 0; 5070 } 5071 5072 static int gfx_v8_0_soft_reset(void *handle) 5073 { 5074 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5075 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 5076 u32 tmp; 5077 5078 if ((!adev->gfx.grbm_soft_reset) && 5079 (!adev->gfx.srbm_soft_reset)) 5080 return 0; 5081 5082 grbm_soft_reset = adev->gfx.grbm_soft_reset; 5083 srbm_soft_reset = adev->gfx.srbm_soft_reset; 5084 5085 if (grbm_soft_reset || srbm_soft_reset) { 5086 tmp = RREG32(mmGMCON_DEBUG); 5087 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1); 5088 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1); 5089 WREG32(mmGMCON_DEBUG, tmp); 5090 udelay(50); 5091 } 5092 5093 if (grbm_soft_reset) { 5094 tmp = RREG32(mmGRBM_SOFT_RESET); 5095 tmp |= grbm_soft_reset; 5096 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 5097 WREG32(mmGRBM_SOFT_RESET, tmp); 5098 tmp = RREG32(mmGRBM_SOFT_RESET); 5099 5100 udelay(50); 5101 5102 tmp &= ~grbm_soft_reset; 5103 WREG32(mmGRBM_SOFT_RESET, tmp); 5104 tmp = RREG32(mmGRBM_SOFT_RESET); 5105 } 5106 5107 if (srbm_soft_reset) { 5108 tmp = RREG32(mmSRBM_SOFT_RESET); 5109 tmp |= srbm_soft_reset; 5110 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 5111 WREG32(mmSRBM_SOFT_RESET, tmp); 5112 tmp = RREG32(mmSRBM_SOFT_RESET); 5113 5114 udelay(50); 5115 5116 tmp &= ~srbm_soft_reset; 5117 WREG32(mmSRBM_SOFT_RESET, tmp); 5118 tmp = RREG32(mmSRBM_SOFT_RESET); 5119 } 5120 5121 if (grbm_soft_reset || srbm_soft_reset) { 5122 tmp = RREG32(mmGMCON_DEBUG); 5123 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0); 5124 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0); 5125 WREG32(mmGMCON_DEBUG, tmp); 5126 } 5127 5128 /* Wait a little for things to settle down */ 5129 udelay(50); 5130 5131 return 0; 5132 } 5133 5134 static int gfx_v8_0_post_soft_reset(void *handle) 5135 { 5136 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5137 u32 grbm_soft_reset = 0; 5138 5139 if ((!adev->gfx.grbm_soft_reset) && 5140 (!adev->gfx.srbm_soft_reset)) 5141 return 0; 5142 5143 grbm_soft_reset = adev->gfx.grbm_soft_reset; 5144 5145 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || 5146 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) || 5147 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) || 5148 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) { 5149 int i; 5150 5151 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5152 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 5153 5154 mutex_lock(&adev->srbm_mutex); 5155 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 5156 gfx_v8_0_deactivate_hqd(adev, 2); 5157 vi_srbm_select(adev, 0, 0, 0, 0); 5158 mutex_unlock(&adev->srbm_mutex); 5159 } 5160 gfx_v8_0_kiq_resume(adev); 5161 gfx_v8_0_kcq_resume(adev); 5162 } 5163 5164 if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || 5165 REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX)) 5166 gfx_v8_0_cp_gfx_resume(adev); 5167 5168 gfx_v8_0_cp_test_all_rings(adev); 5169 5170 adev->gfx.rlc.funcs->start(adev); 5171 5172 return 0; 5173 } 5174 5175 /** 5176 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot 5177 * 5178 * @adev: amdgpu_device pointer 5179 * 5180 * Fetches a GPU clock counter snapshot. 5181 * Returns the 64 bit clock counter snapshot. 5182 */ 5183 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5184 { 5185 uint64_t clock; 5186 5187 mutex_lock(&adev->gfx.gpu_clock_mutex); 5188 WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 5189 clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) | 5190 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 5191 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5192 return clock; 5193 } 5194 5195 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5196 uint32_t vmid, 5197 uint32_t gds_base, uint32_t gds_size, 5198 uint32_t gws_base, uint32_t gws_size, 5199 uint32_t oa_base, uint32_t oa_size) 5200 { 5201 /* GDS Base */ 5202 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5203 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5204 WRITE_DATA_DST_SEL(0))); 5205 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base); 5206 amdgpu_ring_write(ring, 0); 5207 amdgpu_ring_write(ring, gds_base); 5208 5209 /* GDS Size */ 5210 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5211 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5212 WRITE_DATA_DST_SEL(0))); 5213 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size); 5214 amdgpu_ring_write(ring, 0); 5215 amdgpu_ring_write(ring, gds_size); 5216 5217 /* GWS */ 5218 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5219 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5220 WRITE_DATA_DST_SEL(0))); 5221 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws); 5222 amdgpu_ring_write(ring, 0); 5223 amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5224 5225 /* OA */ 5226 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5227 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5228 WRITE_DATA_DST_SEL(0))); 5229 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa); 5230 amdgpu_ring_write(ring, 0); 5231 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); 5232 } 5233 5234 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 5235 { 5236 WREG32(mmSQ_IND_INDEX, 5237 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5238 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5239 (address << SQ_IND_INDEX__INDEX__SHIFT) | 5240 (SQ_IND_INDEX__FORCE_READ_MASK)); 5241 return RREG32(mmSQ_IND_DATA); 5242 } 5243 5244 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 5245 uint32_t wave, uint32_t thread, 5246 uint32_t regno, uint32_t num, uint32_t *out) 5247 { 5248 WREG32(mmSQ_IND_INDEX, 5249 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5250 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5251 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 5252 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 5253 (SQ_IND_INDEX__FORCE_READ_MASK) | 5254 (SQ_IND_INDEX__AUTO_INCR_MASK)); 5255 while (num--) 5256 *(out++) = RREG32(mmSQ_IND_DATA); 5257 } 5258 5259 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 5260 { 5261 /* type 0 wave data */ 5262 dst[(*no_fields)++] = 0; 5263 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 5264 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 5265 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 5266 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 5267 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 5268 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 5269 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 5270 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 5271 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 5272 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 5273 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 5274 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 5275 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO); 5276 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI); 5277 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO); 5278 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI); 5279 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 5280 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 5281 } 5282 5283 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 5284 uint32_t wave, uint32_t start, 5285 uint32_t size, uint32_t *dst) 5286 { 5287 wave_read_regs( 5288 adev, simd, wave, 0, 5289 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 5290 } 5291 5292 5293 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = { 5294 .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter, 5295 .select_se_sh = &gfx_v8_0_select_se_sh, 5296 .read_wave_data = &gfx_v8_0_read_wave_data, 5297 .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs, 5298 .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q 5299 }; 5300 5301 static int gfx_v8_0_early_init(void *handle) 5302 { 5303 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5304 5305 adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; 5306 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5307 AMDGPU_MAX_COMPUTE_RINGS); 5308 adev->gfx.funcs = &gfx_v8_0_gfx_funcs; 5309 gfx_v8_0_set_ring_funcs(adev); 5310 gfx_v8_0_set_irq_funcs(adev); 5311 gfx_v8_0_set_gds_init(adev); 5312 gfx_v8_0_set_rlc_funcs(adev); 5313 5314 return 0; 5315 } 5316 5317 static int gfx_v8_0_late_init(void *handle) 5318 { 5319 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5320 int r; 5321 5322 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5323 if (r) 5324 return r; 5325 5326 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5327 if (r) 5328 return r; 5329 5330 /* requires IBs so do in late init after IB pool is initialized */ 5331 r = gfx_v8_0_do_edc_gpr_workarounds(adev); 5332 if (r) 5333 return r; 5334 5335 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 5336 if (r) { 5337 DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r); 5338 return r; 5339 } 5340 5341 r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0); 5342 if (r) { 5343 DRM_ERROR( 5344 "amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n", 5345 r); 5346 return r; 5347 } 5348 5349 return 0; 5350 } 5351 5352 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 5353 bool enable) 5354 { 5355 if ((adev->asic_type == CHIP_POLARIS11) || 5356 (adev->asic_type == CHIP_POLARIS12) || 5357 (adev->asic_type == CHIP_VEGAM)) 5358 /* Send msg to SMU via Powerplay */ 5359 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable); 5360 5361 WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0); 5362 } 5363 5364 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 5365 bool enable) 5366 { 5367 WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0); 5368 } 5369 5370 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev, 5371 bool enable) 5372 { 5373 WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0); 5374 } 5375 5376 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 5377 bool enable) 5378 { 5379 WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0); 5380 } 5381 5382 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev, 5383 bool enable) 5384 { 5385 WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0); 5386 5387 /* Read any GFX register to wake up GFX. */ 5388 if (!enable) 5389 RREG32(mmDB_RENDER_CONTROL); 5390 } 5391 5392 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev, 5393 bool enable) 5394 { 5395 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 5396 cz_enable_gfx_cg_power_gating(adev, true); 5397 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 5398 cz_enable_gfx_pipeline_power_gating(adev, true); 5399 } else { 5400 cz_enable_gfx_cg_power_gating(adev, false); 5401 cz_enable_gfx_pipeline_power_gating(adev, false); 5402 } 5403 } 5404 5405 static int gfx_v8_0_set_powergating_state(void *handle, 5406 enum amd_powergating_state state) 5407 { 5408 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5409 bool enable = (state == AMD_PG_STATE_GATE); 5410 5411 if (amdgpu_sriov_vf(adev)) 5412 return 0; 5413 5414 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5415 AMD_PG_SUPPORT_RLC_SMU_HS | 5416 AMD_PG_SUPPORT_CP | 5417 AMD_PG_SUPPORT_GFX_DMG)) 5418 amdgpu_gfx_rlc_enter_safe_mode(adev); 5419 switch (adev->asic_type) { 5420 case CHIP_CARRIZO: 5421 case CHIP_STONEY: 5422 5423 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5424 cz_enable_sck_slow_down_on_power_up(adev, true); 5425 cz_enable_sck_slow_down_on_power_down(adev, true); 5426 } else { 5427 cz_enable_sck_slow_down_on_power_up(adev, false); 5428 cz_enable_sck_slow_down_on_power_down(adev, false); 5429 } 5430 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5431 cz_enable_cp_power_gating(adev, true); 5432 else 5433 cz_enable_cp_power_gating(adev, false); 5434 5435 cz_update_gfx_cg_power_gating(adev, enable); 5436 5437 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5438 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5439 else 5440 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5441 5442 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5443 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5444 else 5445 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5446 break; 5447 case CHIP_POLARIS11: 5448 case CHIP_POLARIS12: 5449 case CHIP_VEGAM: 5450 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5451 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5452 else 5453 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5454 5455 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5456 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5457 else 5458 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5459 5460 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable) 5461 polaris11_enable_gfx_quick_mg_power_gating(adev, true); 5462 else 5463 polaris11_enable_gfx_quick_mg_power_gating(adev, false); 5464 break; 5465 default: 5466 break; 5467 } 5468 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5469 AMD_PG_SUPPORT_RLC_SMU_HS | 5470 AMD_PG_SUPPORT_CP | 5471 AMD_PG_SUPPORT_GFX_DMG)) 5472 amdgpu_gfx_rlc_exit_safe_mode(adev); 5473 return 0; 5474 } 5475 5476 static void gfx_v8_0_get_clockgating_state(void *handle, u32 *flags) 5477 { 5478 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5479 int data; 5480 5481 if (amdgpu_sriov_vf(adev)) 5482 *flags = 0; 5483 5484 /* AMD_CG_SUPPORT_GFX_MGCG */ 5485 data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5486 if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK)) 5487 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5488 5489 /* AMD_CG_SUPPORT_GFX_CGLG */ 5490 data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5491 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5492 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5493 5494 /* AMD_CG_SUPPORT_GFX_CGLS */ 5495 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5496 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5497 5498 /* AMD_CG_SUPPORT_GFX_CGTS */ 5499 data = RREG32(mmCGTS_SM_CTRL_REG); 5500 if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK)) 5501 *flags |= AMD_CG_SUPPORT_GFX_CGTS; 5502 5503 /* AMD_CG_SUPPORT_GFX_CGTS_LS */ 5504 if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK)) 5505 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS; 5506 5507 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5508 data = RREG32(mmRLC_MEM_SLP_CNTL); 5509 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5510 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5511 5512 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5513 data = RREG32(mmCP_MEM_SLP_CNTL); 5514 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5515 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5516 } 5517 5518 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev, 5519 uint32_t reg_addr, uint32_t cmd) 5520 { 5521 uint32_t data; 5522 5523 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 5524 5525 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 5526 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 5527 5528 data = RREG32(mmRLC_SERDES_WR_CTRL); 5529 if (adev->asic_type == CHIP_STONEY) 5530 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5531 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5532 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5533 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5534 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5535 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5536 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5537 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5538 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5539 else 5540 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5541 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5542 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5543 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5544 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5545 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5546 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5547 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5548 RLC_SERDES_WR_CTRL__BPM_DATA_MASK | 5549 RLC_SERDES_WR_CTRL__REG_ADDR_MASK | 5550 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5551 data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK | 5552 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) | 5553 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) | 5554 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT)); 5555 5556 WREG32(mmRLC_SERDES_WR_CTRL, data); 5557 } 5558 5559 #define MSG_ENTER_RLC_SAFE_MODE 1 5560 #define MSG_EXIT_RLC_SAFE_MODE 0 5561 #define RLC_GPR_REG2__REQ_MASK 0x00000001 5562 #define RLC_GPR_REG2__REQ__SHIFT 0 5563 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001 5564 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e 5565 5566 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev) 5567 { 5568 uint32_t rlc_setting; 5569 5570 rlc_setting = RREG32(mmRLC_CNTL); 5571 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 5572 return false; 5573 5574 return true; 5575 } 5576 5577 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev) 5578 { 5579 uint32_t data; 5580 unsigned i; 5581 data = RREG32(mmRLC_CNTL); 5582 data |= RLC_SAFE_MODE__CMD_MASK; 5583 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5584 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5585 WREG32(mmRLC_SAFE_MODE, data); 5586 5587 /* wait for RLC_SAFE_MODE */ 5588 for (i = 0; i < adev->usec_timeout; i++) { 5589 if ((RREG32(mmRLC_GPM_STAT) & 5590 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5591 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) == 5592 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5593 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) 5594 break; 5595 udelay(1); 5596 } 5597 for (i = 0; i < adev->usec_timeout; i++) { 5598 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5599 break; 5600 udelay(1); 5601 } 5602 } 5603 5604 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev) 5605 { 5606 uint32_t data; 5607 unsigned i; 5608 5609 data = RREG32(mmRLC_CNTL); 5610 data |= RLC_SAFE_MODE__CMD_MASK; 5611 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5612 WREG32(mmRLC_SAFE_MODE, data); 5613 5614 for (i = 0; i < adev->usec_timeout; i++) { 5615 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5616 break; 5617 udelay(1); 5618 } 5619 } 5620 5621 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5622 { 5623 u32 data; 5624 5625 if (amdgpu_sriov_is_pp_one_vf(adev)) 5626 data = RREG32_NO_KIQ(mmRLC_SPM_VMID); 5627 else 5628 data = RREG32(mmRLC_SPM_VMID); 5629 5630 data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK; 5631 data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT; 5632 5633 if (amdgpu_sriov_is_pp_one_vf(adev)) 5634 WREG32_NO_KIQ(mmRLC_SPM_VMID, data); 5635 else 5636 WREG32(mmRLC_SPM_VMID, data); 5637 } 5638 5639 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = { 5640 .is_rlc_enabled = gfx_v8_0_is_rlc_enabled, 5641 .set_safe_mode = gfx_v8_0_set_safe_mode, 5642 .unset_safe_mode = gfx_v8_0_unset_safe_mode, 5643 .init = gfx_v8_0_rlc_init, 5644 .get_csb_size = gfx_v8_0_get_csb_size, 5645 .get_csb_buffer = gfx_v8_0_get_csb_buffer, 5646 .get_cp_table_num = gfx_v8_0_cp_jump_table_num, 5647 .resume = gfx_v8_0_rlc_resume, 5648 .stop = gfx_v8_0_rlc_stop, 5649 .reset = gfx_v8_0_rlc_reset, 5650 .start = gfx_v8_0_rlc_start, 5651 .update_spm_vmid = gfx_v8_0_update_spm_vmid 5652 }; 5653 5654 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5655 bool enable) 5656 { 5657 uint32_t temp, data; 5658 5659 amdgpu_gfx_rlc_enter_safe_mode(adev); 5660 5661 /* It is disabled by HW by default */ 5662 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 5663 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5664 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) 5665 /* 1 - RLC memory Light sleep */ 5666 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1); 5667 5668 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) 5669 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1); 5670 } 5671 5672 /* 3 - RLC_CGTT_MGCG_OVERRIDE */ 5673 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5674 if (adev->flags & AMD_IS_APU) 5675 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5676 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5677 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK); 5678 else 5679 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5680 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5681 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5682 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5683 5684 if (temp != data) 5685 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5686 5687 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5688 gfx_v8_0_wait_for_rlc_serdes(adev); 5689 5690 /* 5 - clear mgcg override */ 5691 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5692 5693 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) { 5694 /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */ 5695 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5696 data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK); 5697 data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT); 5698 data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK; 5699 data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK; 5700 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) && 5701 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS)) 5702 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK; 5703 data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK; 5704 data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT); 5705 if (temp != data) 5706 WREG32(mmCGTS_SM_CTRL_REG, data); 5707 } 5708 udelay(50); 5709 5710 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5711 gfx_v8_0_wait_for_rlc_serdes(adev); 5712 } else { 5713 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */ 5714 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5715 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5716 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5717 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5718 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5719 if (temp != data) 5720 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5721 5722 /* 2 - disable MGLS in RLC */ 5723 data = RREG32(mmRLC_MEM_SLP_CNTL); 5724 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 5725 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 5726 WREG32(mmRLC_MEM_SLP_CNTL, data); 5727 } 5728 5729 /* 3 - disable MGLS in CP */ 5730 data = RREG32(mmCP_MEM_SLP_CNTL); 5731 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 5732 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 5733 WREG32(mmCP_MEM_SLP_CNTL, data); 5734 } 5735 5736 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */ 5737 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5738 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK | 5739 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK); 5740 if (temp != data) 5741 WREG32(mmCGTS_SM_CTRL_REG, data); 5742 5743 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5744 gfx_v8_0_wait_for_rlc_serdes(adev); 5745 5746 /* 6 - set mgcg override */ 5747 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5748 5749 udelay(50); 5750 5751 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5752 gfx_v8_0_wait_for_rlc_serdes(adev); 5753 } 5754 5755 amdgpu_gfx_rlc_exit_safe_mode(adev); 5756 } 5757 5758 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5759 bool enable) 5760 { 5761 uint32_t temp, temp1, data, data1; 5762 5763 temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5764 5765 amdgpu_gfx_rlc_enter_safe_mode(adev); 5766 5767 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5768 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5769 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK; 5770 if (temp1 != data1) 5771 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5772 5773 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5774 gfx_v8_0_wait_for_rlc_serdes(adev); 5775 5776 /* 2 - clear cgcg override */ 5777 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5778 5779 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5780 gfx_v8_0_wait_for_rlc_serdes(adev); 5781 5782 /* 3 - write cmd to set CGLS */ 5783 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD); 5784 5785 /* 4 - enable cgcg */ 5786 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5787 5788 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5789 /* enable cgls*/ 5790 data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5791 5792 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5793 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK; 5794 5795 if (temp1 != data1) 5796 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5797 } else { 5798 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5799 } 5800 5801 if (temp != data) 5802 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5803 5804 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/ 5805 * Cmp_busy/GFX_Idle interrupts 5806 */ 5807 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5808 } else { 5809 /* disable cntx_empty_int_enable & GFX Idle interrupt */ 5810 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 5811 5812 /* TEST CGCG */ 5813 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5814 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK | 5815 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK); 5816 if (temp1 != data1) 5817 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5818 5819 /* read gfx register to wake up cgcg */ 5820 RREG32(mmCB_CGTT_SCLK_CTRL); 5821 RREG32(mmCB_CGTT_SCLK_CTRL); 5822 RREG32(mmCB_CGTT_SCLK_CTRL); 5823 RREG32(mmCB_CGTT_SCLK_CTRL); 5824 5825 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5826 gfx_v8_0_wait_for_rlc_serdes(adev); 5827 5828 /* write cmd to Set CGCG Overrride */ 5829 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5830 5831 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5832 gfx_v8_0_wait_for_rlc_serdes(adev); 5833 5834 /* write cmd to Clear CGLS */ 5835 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD); 5836 5837 /* disable cgcg, cgls should be disabled too. */ 5838 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | 5839 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5840 if (temp != data) 5841 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5842 /* enable interrupts again for PG */ 5843 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5844 } 5845 5846 gfx_v8_0_wait_for_rlc_serdes(adev); 5847 5848 amdgpu_gfx_rlc_exit_safe_mode(adev); 5849 } 5850 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5851 bool enable) 5852 { 5853 if (enable) { 5854 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS) 5855 * === MGCG + MGLS + TS(CG/LS) === 5856 */ 5857 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5858 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5859 } else { 5860 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS) 5861 * === CGCG + CGLS === 5862 */ 5863 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5864 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5865 } 5866 return 0; 5867 } 5868 5869 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev, 5870 enum amd_clockgating_state state) 5871 { 5872 uint32_t msg_id, pp_state = 0; 5873 uint32_t pp_support_state = 0; 5874 5875 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5876 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5877 pp_support_state = PP_STATE_SUPPORT_LS; 5878 pp_state = PP_STATE_LS; 5879 } 5880 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5881 pp_support_state |= PP_STATE_SUPPORT_CG; 5882 pp_state |= PP_STATE_CG; 5883 } 5884 if (state == AMD_CG_STATE_UNGATE) 5885 pp_state = 0; 5886 5887 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5888 PP_BLOCK_GFX_CG, 5889 pp_support_state, 5890 pp_state); 5891 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5892 } 5893 5894 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5895 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5896 pp_support_state = PP_STATE_SUPPORT_LS; 5897 pp_state = PP_STATE_LS; 5898 } 5899 5900 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5901 pp_support_state |= PP_STATE_SUPPORT_CG; 5902 pp_state |= PP_STATE_CG; 5903 } 5904 5905 if (state == AMD_CG_STATE_UNGATE) 5906 pp_state = 0; 5907 5908 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5909 PP_BLOCK_GFX_MG, 5910 pp_support_state, 5911 pp_state); 5912 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5913 } 5914 5915 return 0; 5916 } 5917 5918 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, 5919 enum amd_clockgating_state state) 5920 { 5921 5922 uint32_t msg_id, pp_state = 0; 5923 uint32_t pp_support_state = 0; 5924 5925 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5926 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5927 pp_support_state = PP_STATE_SUPPORT_LS; 5928 pp_state = PP_STATE_LS; 5929 } 5930 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5931 pp_support_state |= PP_STATE_SUPPORT_CG; 5932 pp_state |= PP_STATE_CG; 5933 } 5934 if (state == AMD_CG_STATE_UNGATE) 5935 pp_state = 0; 5936 5937 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5938 PP_BLOCK_GFX_CG, 5939 pp_support_state, 5940 pp_state); 5941 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5942 } 5943 5944 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) { 5945 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5946 pp_support_state = PP_STATE_SUPPORT_LS; 5947 pp_state = PP_STATE_LS; 5948 } 5949 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5950 pp_support_state |= PP_STATE_SUPPORT_CG; 5951 pp_state |= PP_STATE_CG; 5952 } 5953 if (state == AMD_CG_STATE_UNGATE) 5954 pp_state = 0; 5955 5956 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5957 PP_BLOCK_GFX_3D, 5958 pp_support_state, 5959 pp_state); 5960 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5961 } 5962 5963 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5964 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5965 pp_support_state = PP_STATE_SUPPORT_LS; 5966 pp_state = PP_STATE_LS; 5967 } 5968 5969 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5970 pp_support_state |= PP_STATE_SUPPORT_CG; 5971 pp_state |= PP_STATE_CG; 5972 } 5973 5974 if (state == AMD_CG_STATE_UNGATE) 5975 pp_state = 0; 5976 5977 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5978 PP_BLOCK_GFX_MG, 5979 pp_support_state, 5980 pp_state); 5981 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5982 } 5983 5984 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 5985 pp_support_state = PP_STATE_SUPPORT_LS; 5986 5987 if (state == AMD_CG_STATE_UNGATE) 5988 pp_state = 0; 5989 else 5990 pp_state = PP_STATE_LS; 5991 5992 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5993 PP_BLOCK_GFX_RLC, 5994 pp_support_state, 5995 pp_state); 5996 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5997 } 5998 5999 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 6000 pp_support_state = PP_STATE_SUPPORT_LS; 6001 6002 if (state == AMD_CG_STATE_UNGATE) 6003 pp_state = 0; 6004 else 6005 pp_state = PP_STATE_LS; 6006 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 6007 PP_BLOCK_GFX_CP, 6008 pp_support_state, 6009 pp_state); 6010 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 6011 } 6012 6013 return 0; 6014 } 6015 6016 static int gfx_v8_0_set_clockgating_state(void *handle, 6017 enum amd_clockgating_state state) 6018 { 6019 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 6020 6021 if (amdgpu_sriov_vf(adev)) 6022 return 0; 6023 6024 switch (adev->asic_type) { 6025 case CHIP_FIJI: 6026 case CHIP_CARRIZO: 6027 case CHIP_STONEY: 6028 gfx_v8_0_update_gfx_clock_gating(adev, 6029 state == AMD_CG_STATE_GATE); 6030 break; 6031 case CHIP_TONGA: 6032 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state); 6033 break; 6034 case CHIP_POLARIS10: 6035 case CHIP_POLARIS11: 6036 case CHIP_POLARIS12: 6037 case CHIP_VEGAM: 6038 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state); 6039 break; 6040 default: 6041 break; 6042 } 6043 return 0; 6044 } 6045 6046 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring) 6047 { 6048 return ring->adev->wb.wb[ring->rptr_offs]; 6049 } 6050 6051 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 6052 { 6053 struct amdgpu_device *adev = ring->adev; 6054 6055 if (ring->use_doorbell) 6056 /* XXX check if swapping is necessary on BE */ 6057 return ring->adev->wb.wb[ring->wptr_offs]; 6058 else 6059 return RREG32(mmCP_RB0_WPTR); 6060 } 6061 6062 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 6063 { 6064 struct amdgpu_device *adev = ring->adev; 6065 6066 if (ring->use_doorbell) { 6067 /* XXX check if swapping is necessary on BE */ 6068 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 6069 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 6070 } else { 6071 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 6072 (void)RREG32(mmCP_RB0_WPTR); 6073 } 6074 } 6075 6076 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 6077 { 6078 u32 ref_and_mask, reg_mem_engine; 6079 6080 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) || 6081 (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) { 6082 switch (ring->me) { 6083 case 1: 6084 ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe; 6085 break; 6086 case 2: 6087 ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe; 6088 break; 6089 default: 6090 return; 6091 } 6092 reg_mem_engine = 0; 6093 } else { 6094 ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK; 6095 reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */ 6096 } 6097 6098 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6099 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 6100 WAIT_REG_MEM_FUNCTION(3) | /* == */ 6101 reg_mem_engine)); 6102 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ); 6103 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE); 6104 amdgpu_ring_write(ring, ref_and_mask); 6105 amdgpu_ring_write(ring, ref_and_mask); 6106 amdgpu_ring_write(ring, 0x20); /* poll interval */ 6107 } 6108 6109 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) 6110 { 6111 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 6112 amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) | 6113 EVENT_INDEX(4)); 6114 6115 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 6116 amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) | 6117 EVENT_INDEX(0)); 6118 } 6119 6120 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 6121 struct amdgpu_job *job, 6122 struct amdgpu_ib *ib, 6123 uint32_t flags) 6124 { 6125 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6126 u32 header, control = 0; 6127 6128 if (ib->flags & AMDGPU_IB_FLAG_CE) 6129 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 6130 else 6131 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 6132 6133 control |= ib->length_dw | (vmid << 24); 6134 6135 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 6136 control |= INDIRECT_BUFFER_PRE_ENB(1); 6137 6138 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 6139 gfx_v8_0_ring_emit_de_meta(ring); 6140 } 6141 6142 amdgpu_ring_write(ring, header); 6143 amdgpu_ring_write(ring, 6144 #ifdef __BIG_ENDIAN 6145 (2 << 0) | 6146 #endif 6147 (ib->gpu_addr & 0xFFFFFFFC)); 6148 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6149 amdgpu_ring_write(ring, control); 6150 } 6151 6152 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 6153 struct amdgpu_job *job, 6154 struct amdgpu_ib *ib, 6155 uint32_t flags) 6156 { 6157 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6158 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 6159 6160 /* Currently, there is a high possibility to get wave ID mismatch 6161 * between ME and GDS, leading to a hw deadlock, because ME generates 6162 * different wave IDs than the GDS expects. This situation happens 6163 * randomly when at least 5 compute pipes use GDS ordered append. 6164 * The wave IDs generated by ME are also wrong after suspend/resume. 6165 * Those are probably bugs somewhere else in the kernel driver. 6166 * 6167 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6168 * GDS to 0 for this ring (me/pipe). 6169 */ 6170 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6171 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6172 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); 6173 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6174 } 6175 6176 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6177 amdgpu_ring_write(ring, 6178 #ifdef __BIG_ENDIAN 6179 (2 << 0) | 6180 #endif 6181 (ib->gpu_addr & 0xFFFFFFFC)); 6182 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6183 amdgpu_ring_write(ring, control); 6184 } 6185 6186 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr, 6187 u64 seq, unsigned flags) 6188 { 6189 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6190 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6191 6192 /* Workaround for cache flush problems. First send a dummy EOP 6193 * event down the pipe with seq one below. 6194 */ 6195 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6196 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6197 EOP_TC_ACTION_EN | 6198 EOP_TC_WB_ACTION_EN | 6199 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6200 EVENT_INDEX(5))); 6201 amdgpu_ring_write(ring, addr & 0xfffffffc); 6202 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6203 DATA_SEL(1) | INT_SEL(0)); 6204 amdgpu_ring_write(ring, lower_32_bits(seq - 1)); 6205 amdgpu_ring_write(ring, upper_32_bits(seq - 1)); 6206 6207 /* Then send the real EOP event down the pipe: 6208 * EVENT_WRITE_EOP - flush caches, send int */ 6209 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6210 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6211 EOP_TC_ACTION_EN | 6212 EOP_TC_WB_ACTION_EN | 6213 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6214 EVENT_INDEX(5))); 6215 amdgpu_ring_write(ring, addr & 0xfffffffc); 6216 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6217 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6218 amdgpu_ring_write(ring, lower_32_bits(seq)); 6219 amdgpu_ring_write(ring, upper_32_bits(seq)); 6220 6221 } 6222 6223 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6224 { 6225 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6226 uint32_t seq = ring->fence_drv.sync_seq; 6227 uint64_t addr = ring->fence_drv.gpu_addr; 6228 6229 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6230 amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */ 6231 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 6232 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */ 6233 amdgpu_ring_write(ring, addr & 0xfffffffc); 6234 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); 6235 amdgpu_ring_write(ring, seq); 6236 amdgpu_ring_write(ring, 0xffffffff); 6237 amdgpu_ring_write(ring, 4); /* poll interval */ 6238 } 6239 6240 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6241 unsigned vmid, uint64_t pd_addr) 6242 { 6243 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6244 6245 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6246 6247 /* wait for the invalidate to complete */ 6248 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6249 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 6250 WAIT_REG_MEM_FUNCTION(0) | /* always */ 6251 WAIT_REG_MEM_ENGINE(0))); /* me */ 6252 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); 6253 amdgpu_ring_write(ring, 0); 6254 amdgpu_ring_write(ring, 0); /* ref */ 6255 amdgpu_ring_write(ring, 0); /* mask */ 6256 amdgpu_ring_write(ring, 0x20); /* poll interval */ 6257 6258 /* compute doesn't have PFP */ 6259 if (usepfp) { 6260 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6261 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6262 amdgpu_ring_write(ring, 0x0); 6263 } 6264 } 6265 6266 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 6267 { 6268 return ring->adev->wb.wb[ring->wptr_offs]; 6269 } 6270 6271 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 6272 { 6273 struct amdgpu_device *adev = ring->adev; 6274 6275 /* XXX check if swapping is necessary on BE */ 6276 adev->wb.wb[ring->wptr_offs] = lower_32_bits(ring->wptr); 6277 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 6278 } 6279 6280 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, 6281 u64 addr, u64 seq, 6282 unsigned flags) 6283 { 6284 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6285 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6286 6287 /* RELEASE_MEM - flush caches, send int */ 6288 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 6289 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6290 EOP_TC_ACTION_EN | 6291 EOP_TC_WB_ACTION_EN | 6292 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6293 EVENT_INDEX(5))); 6294 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6295 amdgpu_ring_write(ring, addr & 0xfffffffc); 6296 amdgpu_ring_write(ring, upper_32_bits(addr)); 6297 amdgpu_ring_write(ring, lower_32_bits(seq)); 6298 amdgpu_ring_write(ring, upper_32_bits(seq)); 6299 } 6300 6301 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6302 u64 seq, unsigned int flags) 6303 { 6304 /* we only allocate 32bit for each seq wb address */ 6305 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 6306 6307 /* write fence seq to the "addr" */ 6308 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6309 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6310 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6311 amdgpu_ring_write(ring, lower_32_bits(addr)); 6312 amdgpu_ring_write(ring, upper_32_bits(addr)); 6313 amdgpu_ring_write(ring, lower_32_bits(seq)); 6314 6315 if (flags & AMDGPU_FENCE_FLAG_INT) { 6316 /* set register to trigger INT */ 6317 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6318 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6319 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6320 amdgpu_ring_write(ring, mmCPC_INT_STATUS); 6321 amdgpu_ring_write(ring, 0); 6322 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6323 } 6324 } 6325 6326 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring) 6327 { 6328 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 6329 amdgpu_ring_write(ring, 0); 6330 } 6331 6332 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 6333 { 6334 uint32_t dw2 = 0; 6335 6336 if (amdgpu_sriov_vf(ring->adev)) 6337 gfx_v8_0_ring_emit_ce_meta(ring); 6338 6339 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6340 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6341 gfx_v8_0_ring_emit_vgt_flush(ring); 6342 /* set load_global_config & load_global_uconfig */ 6343 dw2 |= 0x8001; 6344 /* set load_cs_sh_regs */ 6345 dw2 |= 0x01000000; 6346 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6347 dw2 |= 0x10002; 6348 6349 /* set load_ce_ram if preamble presented */ 6350 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 6351 dw2 |= 0x10000000; 6352 } else { 6353 /* still load_ce_ram if this is the first time preamble presented 6354 * although there is no context switch happens. 6355 */ 6356 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 6357 dw2 |= 0x10000000; 6358 } 6359 6360 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6361 amdgpu_ring_write(ring, dw2); 6362 amdgpu_ring_write(ring, 0); 6363 } 6364 6365 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 6366 { 6367 unsigned ret; 6368 6369 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6370 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 6371 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 6372 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 6373 ret = ring->wptr & ring->buf_mask; 6374 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 6375 return ret; 6376 } 6377 6378 static void gfx_v8_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 6379 { 6380 unsigned cur; 6381 6382 BUG_ON(offset > ring->buf_mask); 6383 BUG_ON(ring->ring[offset] != 0x55aa55aa); 6384 6385 cur = (ring->wptr & ring->buf_mask) - 1; 6386 if (likely(cur > offset)) 6387 ring->ring[offset] = cur - offset; 6388 else 6389 ring->ring[offset] = (ring->ring_size >> 2) - offset + cur; 6390 } 6391 6392 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6393 uint32_t reg_val_offs) 6394 { 6395 struct amdgpu_device *adev = ring->adev; 6396 6397 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6398 amdgpu_ring_write(ring, 0 | /* src: register*/ 6399 (5 << 8) | /* dst: memory */ 6400 (1 << 20)); /* write confirm */ 6401 amdgpu_ring_write(ring, reg); 6402 amdgpu_ring_write(ring, 0); 6403 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6404 reg_val_offs * 4)); 6405 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6406 reg_val_offs * 4)); 6407 } 6408 6409 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6410 uint32_t val) 6411 { 6412 uint32_t cmd; 6413 6414 switch (ring->funcs->type) { 6415 case AMDGPU_RING_TYPE_GFX: 6416 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6417 break; 6418 case AMDGPU_RING_TYPE_KIQ: 6419 cmd = 1 << 16; /* no inc addr */ 6420 break; 6421 default: 6422 cmd = WR_CONFIRM; 6423 break; 6424 } 6425 6426 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6427 amdgpu_ring_write(ring, cmd); 6428 amdgpu_ring_write(ring, reg); 6429 amdgpu_ring_write(ring, 0); 6430 amdgpu_ring_write(ring, val); 6431 } 6432 6433 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 6434 { 6435 struct amdgpu_device *adev = ring->adev; 6436 uint32_t value = 0; 6437 6438 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 6439 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 6440 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 6441 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 6442 WREG32(mmSQ_CMD, value); 6443 } 6444 6445 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6446 enum amdgpu_interrupt_state state) 6447 { 6448 WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE, 6449 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6450 } 6451 6452 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6453 int me, int pipe, 6454 enum amdgpu_interrupt_state state) 6455 { 6456 u32 mec_int_cntl, mec_int_cntl_reg; 6457 6458 /* 6459 * amdgpu controls only the first MEC. That's why this function only 6460 * handles the setting of interrupts for this specific MEC. All other 6461 * pipes' interrupts are set by amdkfd. 6462 */ 6463 6464 if (me == 1) { 6465 switch (pipe) { 6466 case 0: 6467 mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL; 6468 break; 6469 case 1: 6470 mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL; 6471 break; 6472 case 2: 6473 mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL; 6474 break; 6475 case 3: 6476 mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL; 6477 break; 6478 default: 6479 DRM_DEBUG("invalid pipe %d\n", pipe); 6480 return; 6481 } 6482 } else { 6483 DRM_DEBUG("invalid me %d\n", me); 6484 return; 6485 } 6486 6487 switch (state) { 6488 case AMDGPU_IRQ_STATE_DISABLE: 6489 mec_int_cntl = RREG32(mec_int_cntl_reg); 6490 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6491 WREG32(mec_int_cntl_reg, mec_int_cntl); 6492 break; 6493 case AMDGPU_IRQ_STATE_ENABLE: 6494 mec_int_cntl = RREG32(mec_int_cntl_reg); 6495 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6496 WREG32(mec_int_cntl_reg, mec_int_cntl); 6497 break; 6498 default: 6499 break; 6500 } 6501 } 6502 6503 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6504 struct amdgpu_irq_src *source, 6505 unsigned type, 6506 enum amdgpu_interrupt_state state) 6507 { 6508 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE, 6509 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6510 6511 return 0; 6512 } 6513 6514 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6515 struct amdgpu_irq_src *source, 6516 unsigned type, 6517 enum amdgpu_interrupt_state state) 6518 { 6519 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE, 6520 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6521 6522 return 0; 6523 } 6524 6525 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6526 struct amdgpu_irq_src *src, 6527 unsigned type, 6528 enum amdgpu_interrupt_state state) 6529 { 6530 switch (type) { 6531 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6532 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state); 6533 break; 6534 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6535 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6536 break; 6537 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6538 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6539 break; 6540 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6541 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6542 break; 6543 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6544 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6545 break; 6546 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 6547 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 6548 break; 6549 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 6550 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 6551 break; 6552 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 6553 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 6554 break; 6555 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 6556 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 6557 break; 6558 default: 6559 break; 6560 } 6561 return 0; 6562 } 6563 6564 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev, 6565 struct amdgpu_irq_src *source, 6566 unsigned int type, 6567 enum amdgpu_interrupt_state state) 6568 { 6569 int enable_flag; 6570 6571 switch (state) { 6572 case AMDGPU_IRQ_STATE_DISABLE: 6573 enable_flag = 0; 6574 break; 6575 6576 case AMDGPU_IRQ_STATE_ENABLE: 6577 enable_flag = 1; 6578 break; 6579 6580 default: 6581 return -EINVAL; 6582 } 6583 6584 WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6585 WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6586 WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6587 WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6588 WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6589 WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6590 enable_flag); 6591 WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6592 enable_flag); 6593 WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6594 enable_flag); 6595 WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6596 enable_flag); 6597 WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6598 enable_flag); 6599 WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6600 enable_flag); 6601 WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6602 enable_flag); 6603 WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6604 enable_flag); 6605 6606 return 0; 6607 } 6608 6609 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev, 6610 struct amdgpu_irq_src *source, 6611 unsigned int type, 6612 enum amdgpu_interrupt_state state) 6613 { 6614 int enable_flag; 6615 6616 switch (state) { 6617 case AMDGPU_IRQ_STATE_DISABLE: 6618 enable_flag = 1; 6619 break; 6620 6621 case AMDGPU_IRQ_STATE_ENABLE: 6622 enable_flag = 0; 6623 break; 6624 6625 default: 6626 return -EINVAL; 6627 } 6628 6629 WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL, 6630 enable_flag); 6631 6632 return 0; 6633 } 6634 6635 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev, 6636 struct amdgpu_irq_src *source, 6637 struct amdgpu_iv_entry *entry) 6638 { 6639 int i; 6640 u8 me_id, pipe_id, queue_id; 6641 struct amdgpu_ring *ring; 6642 6643 DRM_DEBUG("IH: CP EOP\n"); 6644 me_id = (entry->ring_id & 0x0c) >> 2; 6645 pipe_id = (entry->ring_id & 0x03) >> 0; 6646 queue_id = (entry->ring_id & 0x70) >> 4; 6647 6648 switch (me_id) { 6649 case 0: 6650 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 6651 break; 6652 case 1: 6653 case 2: 6654 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6655 ring = &adev->gfx.compute_ring[i]; 6656 /* Per-queue interrupt is supported for MEC starting from VI. 6657 * The interrupt can only be enabled/disabled per pipe instead of per queue. 6658 */ 6659 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 6660 amdgpu_fence_process(ring); 6661 } 6662 break; 6663 } 6664 return 0; 6665 } 6666 6667 static void gfx_v8_0_fault(struct amdgpu_device *adev, 6668 struct amdgpu_iv_entry *entry) 6669 { 6670 u8 me_id, pipe_id, queue_id; 6671 struct amdgpu_ring *ring; 6672 int i; 6673 6674 me_id = (entry->ring_id & 0x0c) >> 2; 6675 pipe_id = (entry->ring_id & 0x03) >> 0; 6676 queue_id = (entry->ring_id & 0x70) >> 4; 6677 6678 switch (me_id) { 6679 case 0: 6680 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 6681 break; 6682 case 1: 6683 case 2: 6684 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6685 ring = &adev->gfx.compute_ring[i]; 6686 if (ring->me == me_id && ring->pipe == pipe_id && 6687 ring->queue == queue_id) 6688 drm_sched_fault(&ring->sched); 6689 } 6690 break; 6691 } 6692 } 6693 6694 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev, 6695 struct amdgpu_irq_src *source, 6696 struct amdgpu_iv_entry *entry) 6697 { 6698 DRM_ERROR("Illegal register access in command stream\n"); 6699 gfx_v8_0_fault(adev, entry); 6700 return 0; 6701 } 6702 6703 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev, 6704 struct amdgpu_irq_src *source, 6705 struct amdgpu_iv_entry *entry) 6706 { 6707 DRM_ERROR("Illegal instruction in command stream\n"); 6708 gfx_v8_0_fault(adev, entry); 6709 return 0; 6710 } 6711 6712 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev, 6713 struct amdgpu_irq_src *source, 6714 struct amdgpu_iv_entry *entry) 6715 { 6716 DRM_ERROR("CP EDC/ECC error detected."); 6717 return 0; 6718 } 6719 6720 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data) 6721 { 6722 u32 enc, se_id, sh_id, cu_id; 6723 char type[20]; 6724 int sq_edc_source = -1; 6725 6726 enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING); 6727 se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID); 6728 6729 switch (enc) { 6730 case 0: 6731 DRM_INFO("SQ general purpose intr detected:" 6732 "se_id %d, immed_overflow %d, host_reg_overflow %d," 6733 "host_cmd_overflow %d, cmd_timestamp %d," 6734 "reg_timestamp %d, thread_trace_buff_full %d," 6735 "wlt %d, thread_trace %d.\n", 6736 se_id, 6737 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW), 6738 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW), 6739 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW), 6740 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP), 6741 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP), 6742 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL), 6743 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT), 6744 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE) 6745 ); 6746 break; 6747 case 1: 6748 case 2: 6749 6750 cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID); 6751 sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID); 6752 6753 /* 6754 * This function can be called either directly from ISR 6755 * or from BH in which case we can access SQ_EDC_INFO 6756 * instance 6757 */ 6758 if (in_task()) { 6759 mutex_lock(&adev->grbm_idx_mutex); 6760 gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id); 6761 6762 sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE); 6763 6764 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6765 mutex_unlock(&adev->grbm_idx_mutex); 6766 } 6767 6768 if (enc == 1) 6769 sprintf(type, "instruction intr"); 6770 else 6771 sprintf(type, "EDC/ECC error"); 6772 6773 DRM_INFO( 6774 "SQ %s detected: " 6775 "se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d " 6776 "trap %s, sq_ed_info.source %s.\n", 6777 type, se_id, sh_id, cu_id, 6778 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID), 6779 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID), 6780 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID), 6781 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false", 6782 (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable" 6783 ); 6784 break; 6785 default: 6786 DRM_ERROR("SQ invalid encoding type\n."); 6787 } 6788 } 6789 6790 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work) 6791 { 6792 6793 struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work); 6794 struct sq_work *sq_work = container_of(work, struct sq_work, work); 6795 6796 gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data); 6797 } 6798 6799 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev, 6800 struct amdgpu_irq_src *source, 6801 struct amdgpu_iv_entry *entry) 6802 { 6803 unsigned ih_data = entry->src_data[0]; 6804 6805 /* 6806 * Try to submit work so SQ_EDC_INFO can be accessed from 6807 * BH. If previous work submission hasn't finished yet 6808 * just print whatever info is possible directly from the ISR. 6809 */ 6810 if (work_pending(&adev->gfx.sq_work.work)) { 6811 gfx_v8_0_parse_sq_irq(adev, ih_data); 6812 } else { 6813 adev->gfx.sq_work.ih_data = ih_data; 6814 schedule_work(&adev->gfx.sq_work.work); 6815 } 6816 6817 return 0; 6818 } 6819 6820 static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring) 6821 { 6822 amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 6823 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6824 PACKET3_TC_ACTION_ENA | 6825 PACKET3_SH_KCACHE_ACTION_ENA | 6826 PACKET3_SH_ICACHE_ACTION_ENA | 6827 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6828 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6829 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6830 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6831 } 6832 6833 static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring) 6834 { 6835 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6836 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6837 PACKET3_TC_ACTION_ENA | 6838 PACKET3_SH_KCACHE_ACTION_ENA | 6839 PACKET3_SH_ICACHE_ACTION_ENA | 6840 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6841 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6842 amdgpu_ring_write(ring, 0xff); /* CP_COHER_SIZE_HI */ 6843 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6844 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6845 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6846 } 6847 6848 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { 6849 .name = "gfx_v8_0", 6850 .early_init = gfx_v8_0_early_init, 6851 .late_init = gfx_v8_0_late_init, 6852 .sw_init = gfx_v8_0_sw_init, 6853 .sw_fini = gfx_v8_0_sw_fini, 6854 .hw_init = gfx_v8_0_hw_init, 6855 .hw_fini = gfx_v8_0_hw_fini, 6856 .suspend = gfx_v8_0_suspend, 6857 .resume = gfx_v8_0_resume, 6858 .is_idle = gfx_v8_0_is_idle, 6859 .wait_for_idle = gfx_v8_0_wait_for_idle, 6860 .check_soft_reset = gfx_v8_0_check_soft_reset, 6861 .pre_soft_reset = gfx_v8_0_pre_soft_reset, 6862 .soft_reset = gfx_v8_0_soft_reset, 6863 .post_soft_reset = gfx_v8_0_post_soft_reset, 6864 .set_clockgating_state = gfx_v8_0_set_clockgating_state, 6865 .set_powergating_state = gfx_v8_0_set_powergating_state, 6866 .get_clockgating_state = gfx_v8_0_get_clockgating_state, 6867 }; 6868 6869 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { 6870 .type = AMDGPU_RING_TYPE_GFX, 6871 .align_mask = 0xff, 6872 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6873 .support_64bit_ptrs = false, 6874 .get_rptr = gfx_v8_0_ring_get_rptr, 6875 .get_wptr = gfx_v8_0_ring_get_wptr_gfx, 6876 .set_wptr = gfx_v8_0_ring_set_wptr_gfx, 6877 .emit_frame_size = /* maximum 215dw if count 16 IBs in */ 6878 5 + /* COND_EXEC */ 6879 7 + /* PIPELINE_SYNC */ 6880 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */ 6881 12 + /* FENCE for VM_FLUSH */ 6882 20 + /* GDS switch */ 6883 4 + /* double SWITCH_BUFFER, 6884 the first COND_EXEC jump to the place just 6885 prior to this double SWITCH_BUFFER */ 6886 5 + /* COND_EXEC */ 6887 7 + /* HDP_flush */ 6888 4 + /* VGT_flush */ 6889 14 + /* CE_META */ 6890 31 + /* DE_META */ 6891 3 + /* CNTX_CTRL */ 6892 5 + /* HDP_INVL */ 6893 12 + 12 + /* FENCE x2 */ 6894 2 + /* SWITCH_BUFFER */ 6895 5, /* SURFACE_SYNC */ 6896 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */ 6897 .emit_ib = gfx_v8_0_ring_emit_ib_gfx, 6898 .emit_fence = gfx_v8_0_ring_emit_fence_gfx, 6899 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6900 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6901 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6902 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6903 .test_ring = gfx_v8_0_ring_test_ring, 6904 .test_ib = gfx_v8_0_ring_test_ib, 6905 .insert_nop = amdgpu_ring_insert_nop, 6906 .pad_ib = amdgpu_ring_generic_pad_ib, 6907 .emit_switch_buffer = gfx_v8_ring_emit_sb, 6908 .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl, 6909 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, 6910 .patch_cond_exec = gfx_v8_0_ring_emit_patch_cond_exec, 6911 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6912 .soft_recovery = gfx_v8_0_ring_soft_recovery, 6913 .emit_mem_sync = gfx_v8_0_emit_mem_sync, 6914 }; 6915 6916 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { 6917 .type = AMDGPU_RING_TYPE_COMPUTE, 6918 .align_mask = 0xff, 6919 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6920 .support_64bit_ptrs = false, 6921 .get_rptr = gfx_v8_0_ring_get_rptr, 6922 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6923 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6924 .emit_frame_size = 6925 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6926 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6927 5 + /* hdp_invalidate */ 6928 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6929 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ 6930 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ 6931 7, /* gfx_v8_0_emit_mem_sync_compute */ 6932 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6933 .emit_ib = gfx_v8_0_ring_emit_ib_compute, 6934 .emit_fence = gfx_v8_0_ring_emit_fence_compute, 6935 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6936 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6937 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6938 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6939 .test_ring = gfx_v8_0_ring_test_ring, 6940 .test_ib = gfx_v8_0_ring_test_ib, 6941 .insert_nop = amdgpu_ring_insert_nop, 6942 .pad_ib = amdgpu_ring_generic_pad_ib, 6943 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6944 .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute, 6945 }; 6946 6947 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { 6948 .type = AMDGPU_RING_TYPE_KIQ, 6949 .align_mask = 0xff, 6950 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6951 .support_64bit_ptrs = false, 6952 .get_rptr = gfx_v8_0_ring_get_rptr, 6953 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6954 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6955 .emit_frame_size = 6956 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6957 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6958 5 + /* hdp_invalidate */ 6959 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6960 17 + /* gfx_v8_0_ring_emit_vm_flush */ 6961 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6962 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6963 .emit_fence = gfx_v8_0_ring_emit_fence_kiq, 6964 .test_ring = gfx_v8_0_ring_test_ring, 6965 .insert_nop = amdgpu_ring_insert_nop, 6966 .pad_ib = amdgpu_ring_generic_pad_ib, 6967 .emit_rreg = gfx_v8_0_ring_emit_rreg, 6968 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6969 }; 6970 6971 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev) 6972 { 6973 int i; 6974 6975 adev->gfx.kiq.ring.funcs = &gfx_v8_0_ring_funcs_kiq; 6976 6977 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6978 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx; 6979 6980 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6981 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute; 6982 } 6983 6984 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = { 6985 .set = gfx_v8_0_set_eop_interrupt_state, 6986 .process = gfx_v8_0_eop_irq, 6987 }; 6988 6989 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = { 6990 .set = gfx_v8_0_set_priv_reg_fault_state, 6991 .process = gfx_v8_0_priv_reg_irq, 6992 }; 6993 6994 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = { 6995 .set = gfx_v8_0_set_priv_inst_fault_state, 6996 .process = gfx_v8_0_priv_inst_irq, 6997 }; 6998 6999 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = { 7000 .set = gfx_v8_0_set_cp_ecc_int_state, 7001 .process = gfx_v8_0_cp_ecc_error_irq, 7002 }; 7003 7004 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = { 7005 .set = gfx_v8_0_set_sq_int_state, 7006 .process = gfx_v8_0_sq_irq, 7007 }; 7008 7009 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev) 7010 { 7011 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7012 adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs; 7013 7014 adev->gfx.priv_reg_irq.num_types = 1; 7015 adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs; 7016 7017 adev->gfx.priv_inst_irq.num_types = 1; 7018 adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs; 7019 7020 adev->gfx.cp_ecc_error_irq.num_types = 1; 7021 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs; 7022 7023 adev->gfx.sq_irq.num_types = 1; 7024 adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs; 7025 } 7026 7027 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev) 7028 { 7029 adev->gfx.rlc.funcs = &iceland_rlc_funcs; 7030 } 7031 7032 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev) 7033 { 7034 /* init asci gds info */ 7035 adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE); 7036 adev->gds.gws_size = 64; 7037 adev->gds.oa_size = 16; 7038 adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); 7039 } 7040 7041 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7042 u32 bitmap) 7043 { 7044 u32 data; 7045 7046 if (!bitmap) 7047 return; 7048 7049 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7050 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7051 7052 WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data); 7053 } 7054 7055 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7056 { 7057 u32 data, mask; 7058 7059 data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) | 7060 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG); 7061 7062 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7063 7064 return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask; 7065 } 7066 7067 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev) 7068 { 7069 int i, j, k, counter, active_cu_number = 0; 7070 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7071 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 7072 unsigned disable_masks[4 * 2]; 7073 u32 ao_cu_num; 7074 7075 memset(cu_info, 0, sizeof(*cu_info)); 7076 7077 if (adev->flags & AMD_IS_APU) 7078 ao_cu_num = 2; 7079 else 7080 ao_cu_num = adev->gfx.config.max_cu_per_sh; 7081 7082 amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2); 7083 7084 mutex_lock(&adev->grbm_idx_mutex); 7085 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7086 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7087 mask = 1; 7088 ao_bitmap = 0; 7089 counter = 0; 7090 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff); 7091 if (i < 4 && j < 2) 7092 gfx_v8_0_set_user_cu_inactive_bitmap( 7093 adev, disable_masks[i * 2 + j]); 7094 bitmap = gfx_v8_0_get_cu_active_bitmap(adev); 7095 cu_info->bitmap[i][j] = bitmap; 7096 7097 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7098 if (bitmap & mask) { 7099 if (counter < ao_cu_num) 7100 ao_bitmap |= mask; 7101 counter ++; 7102 } 7103 mask <<= 1; 7104 } 7105 active_cu_number += counter; 7106 if (i < 2 && j < 2) 7107 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7108 cu_info->ao_cu_bitmap[i][j] = ao_bitmap; 7109 } 7110 } 7111 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7112 mutex_unlock(&adev->grbm_idx_mutex); 7113 7114 cu_info->number = active_cu_number; 7115 cu_info->ao_cu_mask = ao_cu_mask; 7116 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7117 cu_info->max_waves_per_simd = 10; 7118 cu_info->max_scratch_slots_per_cu = 32; 7119 cu_info->wave_front_size = 64; 7120 cu_info->lds_size = 64; 7121 } 7122 7123 const struct amdgpu_ip_block_version gfx_v8_0_ip_block = 7124 { 7125 .type = AMD_IP_BLOCK_TYPE_GFX, 7126 .major = 8, 7127 .minor = 0, 7128 .rev = 0, 7129 .funcs = &gfx_v8_0_ip_funcs, 7130 }; 7131 7132 const struct amdgpu_ip_block_version gfx_v8_1_ip_block = 7133 { 7134 .type = AMD_IP_BLOCK_TYPE_GFX, 7135 .major = 8, 7136 .minor = 1, 7137 .rev = 0, 7138 .funcs = &gfx_v8_0_ip_funcs, 7139 }; 7140 7141 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 7142 { 7143 uint64_t ce_payload_addr; 7144 int cnt_ce; 7145 union { 7146 struct vi_ce_ib_state regular; 7147 struct vi_ce_ib_state_chained_ib chained; 7148 } ce_payload = {}; 7149 7150 if (ring->adev->virt.chained_ib_support) { 7151 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7152 offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload); 7153 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2; 7154 } else { 7155 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7156 offsetof(struct vi_gfx_meta_data, ce_payload); 7157 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2; 7158 } 7159 7160 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce)); 7161 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 7162 WRITE_DATA_DST_SEL(8) | 7163 WR_CONFIRM) | 7164 WRITE_DATA_CACHE_POLICY(0)); 7165 amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr)); 7166 amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr)); 7167 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2); 7168 } 7169 7170 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring) 7171 { 7172 uint64_t de_payload_addr, gds_addr, csa_addr; 7173 int cnt_de; 7174 union { 7175 struct vi_de_ib_state regular; 7176 struct vi_de_ib_state_chained_ib chained; 7177 } de_payload = {}; 7178 7179 csa_addr = amdgpu_csa_vaddr(ring->adev); 7180 gds_addr = csa_addr + 4096; 7181 if (ring->adev->virt.chained_ib_support) { 7182 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr); 7183 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr); 7184 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload); 7185 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2; 7186 } else { 7187 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr); 7188 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr); 7189 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload); 7190 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2; 7191 } 7192 7193 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de)); 7194 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 7195 WRITE_DATA_DST_SEL(8) | 7196 WR_CONFIRM) | 7197 WRITE_DATA_CACHE_POLICY(0)); 7198 amdgpu_ring_write(ring, lower_32_bits(de_payload_addr)); 7199 amdgpu_ring_write(ring, upper_32_bits(de_payload_addr)); 7200 amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2); 7201 } 7202