1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "amdgpu_ring.h" 33 #include "vi.h" 34 #include "vi_structs.h" 35 #include "vid.h" 36 #include "amdgpu_ucode.h" 37 #include "amdgpu_atombios.h" 38 #include "atombios_i2c.h" 39 #include "clearstate_vi.h" 40 41 #include "gmc/gmc_8_2_d.h" 42 #include "gmc/gmc_8_2_sh_mask.h" 43 44 #include "oss/oss_3_0_d.h" 45 #include "oss/oss_3_0_sh_mask.h" 46 47 #include "bif/bif_5_0_d.h" 48 #include "bif/bif_5_0_sh_mask.h" 49 #include "gca/gfx_8_0_d.h" 50 #include "gca/gfx_8_0_enum.h" 51 #include "gca/gfx_8_0_sh_mask.h" 52 53 #include "dce/dce_10_0_d.h" 54 #include "dce/dce_10_0_sh_mask.h" 55 56 #include "smu/smu_7_1_3_d.h" 57 58 #include "ivsrcid/ivsrcid_vislands30.h" 59 60 #define GFX8_NUM_GFX_RINGS 1 61 #define GFX8_MEC_HPD_SIZE 4096 62 63 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001 64 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001 65 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002 66 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003 67 68 #define ARRAY_MODE(x) ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT) 69 #define PIPE_CONFIG(x) ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT) 70 #define TILE_SPLIT(x) ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT) 71 #define MICRO_TILE_MODE_NEW(x) ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT) 72 #define SAMPLE_SPLIT(x) ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT) 73 #define BANK_WIDTH(x) ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT) 74 #define BANK_HEIGHT(x) ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT) 75 #define MACRO_TILE_ASPECT(x) ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT) 76 #define NUM_BANKS(x) ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT) 77 78 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK 0x00000001L 79 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK 0x00000002L 80 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK 0x00000004L 81 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK 0x00000008L 82 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK 0x00000010L 83 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK 0x00000020L 84 85 /* BPM SERDES CMD */ 86 #define SET_BPM_SERDES_CMD 1 87 #define CLE_BPM_SERDES_CMD 0 88 89 /* BPM Register Address*/ 90 enum { 91 BPM_REG_CGLS_EN = 0, /* Enable/Disable CGLS */ 92 BPM_REG_CGLS_ON, /* ON/OFF CGLS: shall be controlled by RLC FW */ 93 BPM_REG_CGCG_OVERRIDE, /* Set/Clear CGCG Override */ 94 BPM_REG_MGCG_OVERRIDE, /* Set/Clear MGCG Override */ 95 BPM_REG_FGCG_OVERRIDE, /* Set/Clear FGCG Override */ 96 BPM_REG_FGCG_MAX 97 }; 98 99 #define RLC_FormatDirectRegListLength 14 100 101 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin"); 102 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin"); 103 MODULE_FIRMWARE("amdgpu/carrizo_me.bin"); 104 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin"); 105 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin"); 106 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin"); 107 108 MODULE_FIRMWARE("amdgpu/stoney_ce.bin"); 109 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin"); 110 MODULE_FIRMWARE("amdgpu/stoney_me.bin"); 111 MODULE_FIRMWARE("amdgpu/stoney_mec.bin"); 112 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin"); 113 114 MODULE_FIRMWARE("amdgpu/tonga_ce.bin"); 115 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin"); 116 MODULE_FIRMWARE("amdgpu/tonga_me.bin"); 117 MODULE_FIRMWARE("amdgpu/tonga_mec.bin"); 118 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin"); 119 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin"); 120 121 MODULE_FIRMWARE("amdgpu/topaz_ce.bin"); 122 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin"); 123 MODULE_FIRMWARE("amdgpu/topaz_me.bin"); 124 MODULE_FIRMWARE("amdgpu/topaz_mec.bin"); 125 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin"); 126 127 MODULE_FIRMWARE("amdgpu/fiji_ce.bin"); 128 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin"); 129 MODULE_FIRMWARE("amdgpu/fiji_me.bin"); 130 MODULE_FIRMWARE("amdgpu/fiji_mec.bin"); 131 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin"); 132 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin"); 133 134 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin"); 135 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin"); 136 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin"); 137 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin"); 138 MODULE_FIRMWARE("amdgpu/polaris10_me.bin"); 139 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin"); 140 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin"); 141 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin"); 142 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin"); 143 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin"); 144 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin"); 145 146 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin"); 147 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin"); 148 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin"); 149 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin"); 150 MODULE_FIRMWARE("amdgpu/polaris11_me.bin"); 151 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin"); 152 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin"); 153 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin"); 154 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin"); 155 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin"); 156 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin"); 157 158 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin"); 159 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin"); 160 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin"); 161 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin"); 162 MODULE_FIRMWARE("amdgpu/polaris12_me.bin"); 163 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin"); 164 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin"); 165 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin"); 166 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin"); 167 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin"); 168 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin"); 169 170 MODULE_FIRMWARE("amdgpu/vegam_ce.bin"); 171 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin"); 172 MODULE_FIRMWARE("amdgpu/vegam_me.bin"); 173 MODULE_FIRMWARE("amdgpu/vegam_mec.bin"); 174 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin"); 175 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin"); 176 177 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] = 178 { 179 {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0}, 180 {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1}, 181 {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2}, 182 {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3}, 183 {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4}, 184 {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5}, 185 {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6}, 186 {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7}, 187 {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8}, 188 {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9}, 189 {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10}, 190 {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11}, 191 {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12}, 192 {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13}, 193 {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14}, 194 {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15} 195 }; 196 197 static const u32 golden_settings_tonga_a11[] = 198 { 199 mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208, 200 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 201 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 202 mmGB_GPU_ID, 0x0000000f, 0x00000000, 203 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 204 mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc, 205 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 206 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 207 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 208 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 209 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 210 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 211 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb, 212 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b, 213 mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876, 214 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 215 }; 216 217 static const u32 tonga_golden_common_all[] = 218 { 219 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 220 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 221 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 222 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 223 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 224 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 225 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 226 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 227 }; 228 229 static const u32 tonga_mgcg_cgcg_init[] = 230 { 231 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 232 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 233 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 234 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 235 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 236 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 237 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 238 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 239 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 240 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 241 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 242 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 243 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 244 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 245 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 246 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 247 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 248 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 249 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 250 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 251 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 252 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 253 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 254 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 255 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 256 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 257 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 258 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 259 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 260 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 261 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 262 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 263 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 264 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 265 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 266 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 267 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 268 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 269 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 270 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 271 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 272 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 273 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 274 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 275 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 276 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 277 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 278 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 279 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 280 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 281 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 282 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 283 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 284 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 285 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 286 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 287 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 288 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 289 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 290 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 291 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 292 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 293 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 294 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 295 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 296 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 297 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 298 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 299 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 300 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 301 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 302 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 303 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 304 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 305 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 306 }; 307 308 static const u32 golden_settings_vegam_a11[] = 309 { 310 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 311 mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000, 312 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 313 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 314 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 315 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 316 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a, 317 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e, 318 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 319 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 320 mmSQ_CONFIG, 0x07f80000, 0x01180000, 321 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 322 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 323 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 324 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 325 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054, 326 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 327 }; 328 329 static const u32 vegam_golden_common_all[] = 330 { 331 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 332 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 333 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 334 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 335 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 336 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 337 }; 338 339 static const u32 golden_settings_polaris11_a11[] = 340 { 341 mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208, 342 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 343 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 344 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 345 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 346 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 347 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 348 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 349 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 350 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 351 mmSQ_CONFIG, 0x07f80000, 0x01180000, 352 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 353 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 354 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3, 355 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 356 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210, 357 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 358 }; 359 360 static const u32 polaris11_golden_common_all[] = 361 { 362 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 363 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002, 364 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 365 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 366 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 367 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 368 }; 369 370 static const u32 golden_settings_polaris10_a11[] = 371 { 372 mmATC_MISC_CG, 0x000c0fc0, 0x000c0200, 373 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 374 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 375 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 376 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 377 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 378 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 379 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 380 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a, 381 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 382 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 383 mmSQ_CONFIG, 0x07f80000, 0x07180000, 384 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 385 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 386 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 387 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 388 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 389 }; 390 391 static const u32 polaris10_golden_common_all[] = 392 { 393 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 394 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 395 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 396 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 397 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 398 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 399 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 400 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 401 }; 402 403 static const u32 fiji_golden_common_all[] = 404 { 405 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 406 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a, 407 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e, 408 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 409 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 410 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 411 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 412 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 413 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 414 mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, 415 }; 416 417 static const u32 golden_settings_fiji_a10[] = 418 { 419 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 420 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 421 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 422 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 423 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 424 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 425 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 426 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 427 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 428 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff, 429 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 430 }; 431 432 static const u32 fiji_mgcg_cgcg_init[] = 433 { 434 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 435 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 436 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 437 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 438 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 439 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 440 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 441 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 442 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 443 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 444 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 445 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 446 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 447 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 448 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 449 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 450 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 451 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 452 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 453 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 454 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 455 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 456 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 457 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 458 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 459 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 460 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 461 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 462 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 463 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 464 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 465 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 466 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 467 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 468 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 469 }; 470 471 static const u32 golden_settings_iceland_a11[] = 472 { 473 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 474 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 475 mmDB_DEBUG3, 0xc0000000, 0xc0000000, 476 mmGB_GPU_ID, 0x0000000f, 0x00000000, 477 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 478 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 479 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002, 480 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 481 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 482 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 483 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 484 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 485 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 486 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1, 487 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 488 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010, 489 }; 490 491 static const u32 iceland_golden_common_all[] = 492 { 493 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 494 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 495 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 496 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 497 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 498 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 499 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 500 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 501 }; 502 503 static const u32 iceland_mgcg_cgcg_init[] = 504 { 505 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 506 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 507 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 508 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 509 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100, 510 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100, 511 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100, 512 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 513 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 514 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 515 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 516 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 517 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 518 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 519 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 520 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 521 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 522 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 523 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 524 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 525 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 526 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 527 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100, 528 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 529 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 530 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 531 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 532 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 533 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 534 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 535 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 536 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 537 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 538 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 539 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 540 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 541 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 542 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 543 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 544 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 545 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 546 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 547 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 548 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 549 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 550 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 551 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 552 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 553 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 554 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 555 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 556 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 557 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 558 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 559 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 560 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 561 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 562 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 563 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 564 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 565 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 566 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 567 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 568 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 569 }; 570 571 static const u32 cz_golden_settings_a11[] = 572 { 573 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 574 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 575 mmGB_GPU_ID, 0x0000000f, 0x00000000, 576 mmPA_SC_ENHANCE, 0xffffffff, 0x00000001, 577 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 578 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 579 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 580 mmTA_CNTL_AUX, 0x000f000f, 0x00010000, 581 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 582 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 583 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3, 584 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302 585 }; 586 587 static const u32 cz_golden_common_all[] = 588 { 589 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 590 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 591 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 592 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 593 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 594 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 595 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 596 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 597 }; 598 599 static const u32 cz_mgcg_cgcg_init[] = 600 { 601 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 602 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 603 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 604 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 605 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 606 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 607 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100, 608 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 609 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 610 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 611 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 612 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 613 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 614 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 615 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 616 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 617 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 618 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 619 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 620 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 621 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 622 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 623 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 624 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 625 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 626 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 627 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 628 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 629 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 630 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 631 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 632 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 633 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 634 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 635 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 636 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 637 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 638 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 639 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 640 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 641 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 642 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 643 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 644 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 645 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 646 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 647 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 648 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 649 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 650 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 651 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 652 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 653 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 654 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 655 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 656 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 657 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 658 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 659 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 660 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 661 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 662 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 663 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 664 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 665 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 666 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 667 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 668 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 669 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 670 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 671 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 672 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 673 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 674 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 675 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 676 }; 677 678 static const u32 stoney_golden_settings_a11[] = 679 { 680 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 681 mmGB_GPU_ID, 0x0000000f, 0x00000000, 682 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 683 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 684 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 685 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 686 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 687 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 688 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1, 689 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010, 690 }; 691 692 static const u32 stoney_golden_common_all[] = 693 { 694 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 695 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000, 696 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 697 mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001, 698 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 699 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 700 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 701 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 702 }; 703 704 static const u32 stoney_mgcg_cgcg_init[] = 705 { 706 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 707 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 708 mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 709 mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 710 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200, 711 }; 712 713 714 static const char * const sq_edc_source_names[] = { 715 "SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred", 716 "SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch", 717 "SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return", 718 "SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR", 719 "SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS", 720 "SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS", 721 "SQ_EDC_INFO_SOURCE_TA: EDC source is TA", 722 }; 723 724 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev); 725 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev); 726 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev); 727 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev); 728 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev); 729 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev); 730 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring); 731 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring); 732 733 #define CG_ACLK_CNTL__ACLK_DIVIDER_MASK 0x0000007fL 734 #define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT 0x00000000L 735 736 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev) 737 { 738 uint32_t data; 739 740 switch (adev->asic_type) { 741 case CHIP_TOPAZ: 742 amdgpu_device_program_register_sequence(adev, 743 iceland_mgcg_cgcg_init, 744 ARRAY_SIZE(iceland_mgcg_cgcg_init)); 745 amdgpu_device_program_register_sequence(adev, 746 golden_settings_iceland_a11, 747 ARRAY_SIZE(golden_settings_iceland_a11)); 748 amdgpu_device_program_register_sequence(adev, 749 iceland_golden_common_all, 750 ARRAY_SIZE(iceland_golden_common_all)); 751 break; 752 case CHIP_FIJI: 753 amdgpu_device_program_register_sequence(adev, 754 fiji_mgcg_cgcg_init, 755 ARRAY_SIZE(fiji_mgcg_cgcg_init)); 756 amdgpu_device_program_register_sequence(adev, 757 golden_settings_fiji_a10, 758 ARRAY_SIZE(golden_settings_fiji_a10)); 759 amdgpu_device_program_register_sequence(adev, 760 fiji_golden_common_all, 761 ARRAY_SIZE(fiji_golden_common_all)); 762 break; 763 764 case CHIP_TONGA: 765 amdgpu_device_program_register_sequence(adev, 766 tonga_mgcg_cgcg_init, 767 ARRAY_SIZE(tonga_mgcg_cgcg_init)); 768 amdgpu_device_program_register_sequence(adev, 769 golden_settings_tonga_a11, 770 ARRAY_SIZE(golden_settings_tonga_a11)); 771 amdgpu_device_program_register_sequence(adev, 772 tonga_golden_common_all, 773 ARRAY_SIZE(tonga_golden_common_all)); 774 break; 775 case CHIP_VEGAM: 776 amdgpu_device_program_register_sequence(adev, 777 golden_settings_vegam_a11, 778 ARRAY_SIZE(golden_settings_vegam_a11)); 779 amdgpu_device_program_register_sequence(adev, 780 vegam_golden_common_all, 781 ARRAY_SIZE(vegam_golden_common_all)); 782 break; 783 case CHIP_POLARIS11: 784 case CHIP_POLARIS12: 785 amdgpu_device_program_register_sequence(adev, 786 golden_settings_polaris11_a11, 787 ARRAY_SIZE(golden_settings_polaris11_a11)); 788 amdgpu_device_program_register_sequence(adev, 789 polaris11_golden_common_all, 790 ARRAY_SIZE(polaris11_golden_common_all)); 791 break; 792 case CHIP_POLARIS10: 793 amdgpu_device_program_register_sequence(adev, 794 golden_settings_polaris10_a11, 795 ARRAY_SIZE(golden_settings_polaris10_a11)); 796 amdgpu_device_program_register_sequence(adev, 797 polaris10_golden_common_all, 798 ARRAY_SIZE(polaris10_golden_common_all)); 799 data = RREG32_SMC(ixCG_ACLK_CNTL); 800 data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK; 801 data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT; 802 WREG32_SMC(ixCG_ACLK_CNTL, data); 803 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) && 804 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) || 805 (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) || 806 (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) { 807 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD); 808 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0); 809 } 810 break; 811 case CHIP_CARRIZO: 812 amdgpu_device_program_register_sequence(adev, 813 cz_mgcg_cgcg_init, 814 ARRAY_SIZE(cz_mgcg_cgcg_init)); 815 amdgpu_device_program_register_sequence(adev, 816 cz_golden_settings_a11, 817 ARRAY_SIZE(cz_golden_settings_a11)); 818 amdgpu_device_program_register_sequence(adev, 819 cz_golden_common_all, 820 ARRAY_SIZE(cz_golden_common_all)); 821 break; 822 case CHIP_STONEY: 823 amdgpu_device_program_register_sequence(adev, 824 stoney_mgcg_cgcg_init, 825 ARRAY_SIZE(stoney_mgcg_cgcg_init)); 826 amdgpu_device_program_register_sequence(adev, 827 stoney_golden_settings_a11, 828 ARRAY_SIZE(stoney_golden_settings_a11)); 829 amdgpu_device_program_register_sequence(adev, 830 stoney_golden_common_all, 831 ARRAY_SIZE(stoney_golden_common_all)); 832 break; 833 default: 834 break; 835 } 836 } 837 838 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring) 839 { 840 struct amdgpu_device *adev = ring->adev; 841 uint32_t tmp = 0; 842 unsigned i; 843 int r; 844 845 WREG32(mmSCRATCH_REG0, 0xCAFEDEAD); 846 r = amdgpu_ring_alloc(ring, 3); 847 if (r) 848 return r; 849 850 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 851 amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START); 852 amdgpu_ring_write(ring, 0xDEADBEEF); 853 amdgpu_ring_commit(ring); 854 855 for (i = 0; i < adev->usec_timeout; i++) { 856 tmp = RREG32(mmSCRATCH_REG0); 857 if (tmp == 0xDEADBEEF) 858 break; 859 udelay(1); 860 } 861 862 if (i >= adev->usec_timeout) 863 r = -ETIMEDOUT; 864 865 return r; 866 } 867 868 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 869 { 870 struct amdgpu_device *adev = ring->adev; 871 struct amdgpu_ib ib; 872 struct dma_fence *f = NULL; 873 874 unsigned int index; 875 uint64_t gpu_addr; 876 uint32_t tmp; 877 long r; 878 879 r = amdgpu_wb_get(adev, &index); 880 if (r) 881 return r; 882 883 gpu_addr = adev->wb.gpu_addr + (index * 4); 884 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 885 memset(&ib, 0, sizeof(ib)); 886 887 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 888 if (r) 889 goto err1; 890 891 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 892 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 893 ib.ptr[2] = lower_32_bits(gpu_addr); 894 ib.ptr[3] = upper_32_bits(gpu_addr); 895 ib.ptr[4] = 0xDEADBEEF; 896 ib.length_dw = 5; 897 898 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 899 if (r) 900 goto err2; 901 902 r = dma_fence_wait_timeout(f, false, timeout); 903 if (r == 0) { 904 r = -ETIMEDOUT; 905 goto err2; 906 } else if (r < 0) { 907 goto err2; 908 } 909 910 tmp = adev->wb.wb[index]; 911 if (tmp == 0xDEADBEEF) 912 r = 0; 913 else 914 r = -EINVAL; 915 916 err2: 917 amdgpu_ib_free(&ib, NULL); 918 dma_fence_put(f); 919 err1: 920 amdgpu_wb_free(adev, index); 921 return r; 922 } 923 924 925 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev) 926 { 927 amdgpu_ucode_release(&adev->gfx.pfp_fw); 928 amdgpu_ucode_release(&adev->gfx.me_fw); 929 amdgpu_ucode_release(&adev->gfx.ce_fw); 930 amdgpu_ucode_release(&adev->gfx.rlc_fw); 931 amdgpu_ucode_release(&adev->gfx.mec_fw); 932 if ((adev->asic_type != CHIP_STONEY) && 933 (adev->asic_type != CHIP_TOPAZ)) 934 amdgpu_ucode_release(&adev->gfx.mec2_fw); 935 936 kfree(adev->gfx.rlc.register_list_format); 937 } 938 939 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev) 940 { 941 const char *chip_name; 942 int err; 943 struct amdgpu_firmware_info *info = NULL; 944 const struct common_firmware_header *header = NULL; 945 const struct gfx_firmware_header_v1_0 *cp_hdr; 946 const struct rlc_firmware_header_v2_0 *rlc_hdr; 947 unsigned int *tmp = NULL, i; 948 949 DRM_DEBUG("\n"); 950 951 switch (adev->asic_type) { 952 case CHIP_TOPAZ: 953 chip_name = "topaz"; 954 break; 955 case CHIP_TONGA: 956 chip_name = "tonga"; 957 break; 958 case CHIP_CARRIZO: 959 chip_name = "carrizo"; 960 break; 961 case CHIP_FIJI: 962 chip_name = "fiji"; 963 break; 964 case CHIP_STONEY: 965 chip_name = "stoney"; 966 break; 967 case CHIP_POLARIS10: 968 chip_name = "polaris10"; 969 break; 970 case CHIP_POLARIS11: 971 chip_name = "polaris11"; 972 break; 973 case CHIP_POLARIS12: 974 chip_name = "polaris12"; 975 break; 976 case CHIP_VEGAM: 977 chip_name = "vegam"; 978 break; 979 default: 980 dev_warn(adev->dev, "Unsupported asic_type 0x%08x\n", adev->asic_type); 981 return -EINVAL; 982 } 983 984 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 985 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 986 AMDGPU_UCODE_OPTIONAL, 987 "amdgpu/%s_pfp_2.bin", chip_name); 988 if (err == -ENODEV) { 989 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 990 AMDGPU_UCODE_REQUIRED, 991 "amdgpu/%s_pfp.bin", chip_name); 992 } 993 } else { 994 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 995 AMDGPU_UCODE_REQUIRED, 996 "amdgpu/%s_pfp.bin", chip_name); 997 } 998 if (err) 999 goto out; 1000 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1001 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1002 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1003 1004 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1005 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1006 AMDGPU_UCODE_OPTIONAL, 1007 "amdgpu/%s_me_2.bin", chip_name); 1008 if (err == -ENODEV) { 1009 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1010 AMDGPU_UCODE_REQUIRED, 1011 "amdgpu/%s_me.bin", chip_name); 1012 } 1013 } else { 1014 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1015 AMDGPU_UCODE_REQUIRED, 1016 "amdgpu/%s_me.bin", chip_name); 1017 } 1018 if (err) 1019 goto out; 1020 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1021 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1022 1023 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1024 1025 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1026 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1027 AMDGPU_UCODE_OPTIONAL, 1028 "amdgpu/%s_ce_2.bin", chip_name); 1029 if (err == -ENODEV) { 1030 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1031 AMDGPU_UCODE_REQUIRED, 1032 "amdgpu/%s_ce.bin", chip_name); 1033 } 1034 } else { 1035 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1036 AMDGPU_UCODE_REQUIRED, 1037 "amdgpu/%s_ce.bin", chip_name); 1038 } 1039 if (err) 1040 goto out; 1041 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1042 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1043 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1044 1045 /* 1046 * Support for MCBP/Virtualization in combination with chained IBs is 1047 * formal released on feature version #46 1048 */ 1049 if (adev->gfx.ce_feature_version >= 46 && 1050 adev->gfx.pfp_feature_version >= 46) { 1051 adev->virt.chained_ib_support = true; 1052 drm_info(adev_to_drm(adev), "Chained IB support enabled!\n"); 1053 } else 1054 adev->virt.chained_ib_support = false; 1055 1056 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1057 AMDGPU_UCODE_REQUIRED, 1058 "amdgpu/%s_rlc.bin", chip_name); 1059 if (err) 1060 goto out; 1061 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1062 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1063 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1064 1065 adev->gfx.rlc.save_and_restore_offset = 1066 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1067 adev->gfx.rlc.clear_state_descriptor_offset = 1068 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1069 adev->gfx.rlc.avail_scratch_ram_locations = 1070 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1071 adev->gfx.rlc.reg_restore_list_size = 1072 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1073 adev->gfx.rlc.reg_list_format_start = 1074 le32_to_cpu(rlc_hdr->reg_list_format_start); 1075 adev->gfx.rlc.reg_list_format_separate_start = 1076 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1077 adev->gfx.rlc.starting_offsets_start = 1078 le32_to_cpu(rlc_hdr->starting_offsets_start); 1079 adev->gfx.rlc.reg_list_format_size_bytes = 1080 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1081 adev->gfx.rlc.reg_list_size_bytes = 1082 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1083 1084 adev->gfx.rlc.register_list_format = 1085 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1086 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1087 1088 if (!adev->gfx.rlc.register_list_format) { 1089 err = -ENOMEM; 1090 goto out; 1091 } 1092 1093 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1094 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1095 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1096 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1097 1098 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1099 1100 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1101 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1102 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1103 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1104 1105 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1106 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1107 AMDGPU_UCODE_OPTIONAL, 1108 "amdgpu/%s_mec_2.bin", chip_name); 1109 if (err == -ENODEV) { 1110 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1111 AMDGPU_UCODE_REQUIRED, 1112 "amdgpu/%s_mec.bin", chip_name); 1113 } 1114 } else { 1115 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1116 AMDGPU_UCODE_REQUIRED, 1117 "amdgpu/%s_mec.bin", chip_name); 1118 } 1119 if (err) 1120 goto out; 1121 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1122 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1123 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1124 1125 if ((adev->asic_type != CHIP_STONEY) && 1126 (adev->asic_type != CHIP_TOPAZ)) { 1127 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1128 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1129 AMDGPU_UCODE_OPTIONAL, 1130 "amdgpu/%s_mec2_2.bin", chip_name); 1131 if (err == -ENODEV) { 1132 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1133 AMDGPU_UCODE_REQUIRED, 1134 "amdgpu/%s_mec2.bin", chip_name); 1135 } 1136 } else { 1137 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1138 AMDGPU_UCODE_REQUIRED, 1139 "amdgpu/%s_mec2.bin", chip_name); 1140 } 1141 if (!err) { 1142 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1143 adev->gfx.mec2_fw->data; 1144 adev->gfx.mec2_fw_version = 1145 le32_to_cpu(cp_hdr->header.ucode_version); 1146 adev->gfx.mec2_feature_version = 1147 le32_to_cpu(cp_hdr->ucode_feature_version); 1148 } else { 1149 err = 0; 1150 adev->gfx.mec2_fw = NULL; 1151 } 1152 } 1153 1154 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1155 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1156 info->fw = adev->gfx.pfp_fw; 1157 header = (const struct common_firmware_header *)info->fw->data; 1158 adev->firmware.fw_size += 1159 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1160 1161 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1162 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1163 info->fw = adev->gfx.me_fw; 1164 header = (const struct common_firmware_header *)info->fw->data; 1165 adev->firmware.fw_size += 1166 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1167 1168 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1169 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1170 info->fw = adev->gfx.ce_fw; 1171 header = (const struct common_firmware_header *)info->fw->data; 1172 adev->firmware.fw_size += 1173 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1174 1175 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1176 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1177 info->fw = adev->gfx.rlc_fw; 1178 header = (const struct common_firmware_header *)info->fw->data; 1179 adev->firmware.fw_size += 1180 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1181 1182 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1183 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1184 info->fw = adev->gfx.mec_fw; 1185 header = (const struct common_firmware_header *)info->fw->data; 1186 adev->firmware.fw_size += 1187 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1188 1189 /* we need account JT in */ 1190 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1191 adev->firmware.fw_size += 1192 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE); 1193 1194 if (amdgpu_sriov_vf(adev)) { 1195 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE]; 1196 info->ucode_id = AMDGPU_UCODE_ID_STORAGE; 1197 info->fw = adev->gfx.mec_fw; 1198 adev->firmware.fw_size += 1199 ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE); 1200 } 1201 1202 if (adev->gfx.mec2_fw) { 1203 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1204 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1205 info->fw = adev->gfx.mec2_fw; 1206 header = (const struct common_firmware_header *)info->fw->data; 1207 adev->firmware.fw_size += 1208 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1209 } 1210 1211 out: 1212 if (err) { 1213 dev_err(adev->dev, "gfx8: Failed to load firmware %s gfx firmware\n", chip_name); 1214 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1215 amdgpu_ucode_release(&adev->gfx.me_fw); 1216 amdgpu_ucode_release(&adev->gfx.ce_fw); 1217 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1218 amdgpu_ucode_release(&adev->gfx.mec_fw); 1219 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1220 } 1221 return err; 1222 } 1223 1224 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 1225 { 1226 u32 count = 0; 1227 1228 if (adev->gfx.rlc.cs_data == NULL) 1229 return; 1230 if (buffer == NULL) 1231 return; 1232 1233 count = amdgpu_gfx_csb_preamble_start(buffer); 1234 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 1235 1236 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 1237 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 1238 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config); 1239 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1); 1240 1241 amdgpu_gfx_csb_preamble_end(buffer, count); 1242 } 1243 1244 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev) 1245 { 1246 if (adev->asic_type == CHIP_CARRIZO) 1247 return 5; 1248 else 1249 return 4; 1250 } 1251 1252 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev) 1253 { 1254 const struct cs_section_def *cs_data; 1255 int r; 1256 1257 adev->gfx.rlc.cs_data = vi_cs_data; 1258 1259 cs_data = adev->gfx.rlc.cs_data; 1260 1261 if (cs_data) { 1262 /* init clear state block */ 1263 r = amdgpu_gfx_rlc_init_csb(adev); 1264 if (r) 1265 return r; 1266 } 1267 1268 if ((adev->asic_type == CHIP_CARRIZO) || 1269 (adev->asic_type == CHIP_STONEY)) { 1270 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1271 r = amdgpu_gfx_rlc_init_cpt(adev); 1272 if (r) 1273 return r; 1274 } 1275 1276 /* init spm vmid with 0xf */ 1277 if (adev->gfx.rlc.funcs->update_spm_vmid) 1278 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf); 1279 1280 return 0; 1281 } 1282 1283 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev) 1284 { 1285 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1286 } 1287 1288 static int gfx_v8_0_mec_init(struct amdgpu_device *adev) 1289 { 1290 int r; 1291 u32 *hpd; 1292 size_t mec_hpd_size; 1293 1294 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1295 1296 /* take ownership of the relevant compute queues */ 1297 amdgpu_gfx_compute_queue_acquire(adev); 1298 1299 mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE; 1300 if (mec_hpd_size) { 1301 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1302 AMDGPU_GEM_DOMAIN_VRAM | 1303 AMDGPU_GEM_DOMAIN_GTT, 1304 &adev->gfx.mec.hpd_eop_obj, 1305 &adev->gfx.mec.hpd_eop_gpu_addr, 1306 (void **)&hpd); 1307 if (r) { 1308 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1309 return r; 1310 } 1311 1312 memset(hpd, 0, mec_hpd_size); 1313 1314 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1315 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1316 } 1317 1318 return 0; 1319 } 1320 1321 static const u32 vgpr_init_compute_shader[] = 1322 { 1323 0x7e000209, 0x7e020208, 1324 0x7e040207, 0x7e060206, 1325 0x7e080205, 0x7e0a0204, 1326 0x7e0c0203, 0x7e0e0202, 1327 0x7e100201, 0x7e120200, 1328 0x7e140209, 0x7e160208, 1329 0x7e180207, 0x7e1a0206, 1330 0x7e1c0205, 0x7e1e0204, 1331 0x7e200203, 0x7e220202, 1332 0x7e240201, 0x7e260200, 1333 0x7e280209, 0x7e2a0208, 1334 0x7e2c0207, 0x7e2e0206, 1335 0x7e300205, 0x7e320204, 1336 0x7e340203, 0x7e360202, 1337 0x7e380201, 0x7e3a0200, 1338 0x7e3c0209, 0x7e3e0208, 1339 0x7e400207, 0x7e420206, 1340 0x7e440205, 0x7e460204, 1341 0x7e480203, 0x7e4a0202, 1342 0x7e4c0201, 0x7e4e0200, 1343 0x7e500209, 0x7e520208, 1344 0x7e540207, 0x7e560206, 1345 0x7e580205, 0x7e5a0204, 1346 0x7e5c0203, 0x7e5e0202, 1347 0x7e600201, 0x7e620200, 1348 0x7e640209, 0x7e660208, 1349 0x7e680207, 0x7e6a0206, 1350 0x7e6c0205, 0x7e6e0204, 1351 0x7e700203, 0x7e720202, 1352 0x7e740201, 0x7e760200, 1353 0x7e780209, 0x7e7a0208, 1354 0x7e7c0207, 0x7e7e0206, 1355 0xbf8a0000, 0xbf810000, 1356 }; 1357 1358 static const u32 sgpr_init_compute_shader[] = 1359 { 1360 0xbe8a0100, 0xbe8c0102, 1361 0xbe8e0104, 0xbe900106, 1362 0xbe920108, 0xbe940100, 1363 0xbe960102, 0xbe980104, 1364 0xbe9a0106, 0xbe9c0108, 1365 0xbe9e0100, 0xbea00102, 1366 0xbea20104, 0xbea40106, 1367 0xbea60108, 0xbea80100, 1368 0xbeaa0102, 0xbeac0104, 1369 0xbeae0106, 0xbeb00108, 1370 0xbeb20100, 0xbeb40102, 1371 0xbeb60104, 0xbeb80106, 1372 0xbeba0108, 0xbebc0100, 1373 0xbebe0102, 0xbec00104, 1374 0xbec20106, 0xbec40108, 1375 0xbec60100, 0xbec80102, 1376 0xbee60004, 0xbee70005, 1377 0xbeea0006, 0xbeeb0007, 1378 0xbee80008, 0xbee90009, 1379 0xbefc0000, 0xbf8a0000, 1380 0xbf810000, 0x00000000, 1381 }; 1382 1383 static const u32 vgpr_init_regs[] = 1384 { 1385 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff, 1386 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1387 mmCOMPUTE_NUM_THREAD_X, 256*4, 1388 mmCOMPUTE_NUM_THREAD_Y, 1, 1389 mmCOMPUTE_NUM_THREAD_Z, 1, 1390 mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */ 1391 mmCOMPUTE_PGM_RSRC2, 20, 1392 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1393 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1394 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1395 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1396 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1397 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1398 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1399 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1400 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1401 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1402 }; 1403 1404 static const u32 sgpr1_init_regs[] = 1405 { 1406 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f, 1407 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1408 mmCOMPUTE_NUM_THREAD_X, 256*5, 1409 mmCOMPUTE_NUM_THREAD_Y, 1, 1410 mmCOMPUTE_NUM_THREAD_Z, 1, 1411 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1412 mmCOMPUTE_PGM_RSRC2, 20, 1413 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1414 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1415 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1416 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1417 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1418 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1419 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1420 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1421 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1422 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1423 }; 1424 1425 static const u32 sgpr2_init_regs[] = 1426 { 1427 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0, 1428 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, 1429 mmCOMPUTE_NUM_THREAD_X, 256*5, 1430 mmCOMPUTE_NUM_THREAD_Y, 1, 1431 mmCOMPUTE_NUM_THREAD_Z, 1, 1432 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1433 mmCOMPUTE_PGM_RSRC2, 20, 1434 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1435 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1436 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1437 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1438 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1439 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1440 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1441 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1442 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1443 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1444 }; 1445 1446 static const u32 sec_ded_counter_registers[] = 1447 { 1448 mmCPC_EDC_ATC_CNT, 1449 mmCPC_EDC_SCRATCH_CNT, 1450 mmCPC_EDC_UCODE_CNT, 1451 mmCPF_EDC_ATC_CNT, 1452 mmCPF_EDC_ROQ_CNT, 1453 mmCPF_EDC_TAG_CNT, 1454 mmCPG_EDC_ATC_CNT, 1455 mmCPG_EDC_DMA_CNT, 1456 mmCPG_EDC_TAG_CNT, 1457 mmDC_EDC_CSINVOC_CNT, 1458 mmDC_EDC_RESTORE_CNT, 1459 mmDC_EDC_STATE_CNT, 1460 mmGDS_EDC_CNT, 1461 mmGDS_EDC_GRBM_CNT, 1462 mmGDS_EDC_OA_DED, 1463 mmSPI_EDC_CNT, 1464 mmSQC_ATC_EDC_GATCL1_CNT, 1465 mmSQC_EDC_CNT, 1466 mmSQ_EDC_DED_CNT, 1467 mmSQ_EDC_INFO, 1468 mmSQ_EDC_SEC_CNT, 1469 mmTCC_EDC_CNT, 1470 mmTCP_ATC_EDC_GATCL1_CNT, 1471 mmTCP_EDC_CNT, 1472 mmTD_EDC_CNT 1473 }; 1474 1475 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 1476 { 1477 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 1478 struct amdgpu_ib ib; 1479 struct dma_fence *f = NULL; 1480 int r, i; 1481 u32 tmp; 1482 unsigned total_size, vgpr_offset, sgpr_offset; 1483 u64 gpu_addr; 1484 1485 /* only supported on CZ */ 1486 if (adev->asic_type != CHIP_CARRIZO) 1487 return 0; 1488 1489 /* bail if the compute ring is not ready */ 1490 if (!ring->sched.ready) 1491 return -EBUSY; 1492 1493 if (amdgpu_in_reset(adev)) { 1494 /* Set preempt condition to execute IB */ 1495 amdgpu_ring_set_preempt_cond_exec(ring, true); 1496 /* Flush HDP cache so the GPU can see the updated COND_EXEC value */ 1497 amdgpu_device_flush_hdp(adev, NULL); 1498 } 1499 1500 tmp = RREG32(mmGB_EDC_MODE); 1501 WREG32(mmGB_EDC_MODE, 0); 1502 1503 total_size = 1504 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1505 total_size += 1506 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1507 total_size += 1508 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1509 total_size = ALIGN(total_size, 256); 1510 vgpr_offset = total_size; 1511 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256); 1512 sgpr_offset = total_size; 1513 total_size += sizeof(sgpr_init_compute_shader); 1514 1515 /* allocate an indirect buffer to put the commands in */ 1516 memset(&ib, 0, sizeof(ib)); 1517 r = amdgpu_ib_get(adev, NULL, total_size, 1518 AMDGPU_IB_POOL_DIRECT, &ib); 1519 if (r) { 1520 drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r); 1521 return r; 1522 } 1523 1524 /* load the compute shaders */ 1525 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++) 1526 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i]; 1527 1528 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 1529 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 1530 1531 /* init the ib length to 0 */ 1532 ib.length_dw = 0; 1533 1534 /* VGPR */ 1535 /* write the register state for the compute dispatch */ 1536 for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) { 1537 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1538 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START; 1539 ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1]; 1540 } 1541 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1542 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 1543 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1544 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1545 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1546 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1547 1548 /* write dispatch packet */ 1549 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1550 ib.ptr[ib.length_dw++] = 8; /* x */ 1551 ib.ptr[ib.length_dw++] = 1; /* y */ 1552 ib.ptr[ib.length_dw++] = 1; /* z */ 1553 ib.ptr[ib.length_dw++] = 1554 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1555 1556 /* write CS partial flush packet */ 1557 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1558 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1559 1560 /* SGPR1 */ 1561 /* write the register state for the compute dispatch */ 1562 for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) { 1563 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1564 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START; 1565 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1]; 1566 } 1567 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1568 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 1569 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1570 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1571 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1572 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1573 1574 /* write dispatch packet */ 1575 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1576 ib.ptr[ib.length_dw++] = 8; /* x */ 1577 ib.ptr[ib.length_dw++] = 1; /* y */ 1578 ib.ptr[ib.length_dw++] = 1; /* z */ 1579 ib.ptr[ib.length_dw++] = 1580 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1581 1582 /* write CS partial flush packet */ 1583 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1584 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1585 1586 /* SGPR2 */ 1587 /* write the register state for the compute dispatch */ 1588 for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) { 1589 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1590 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START; 1591 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1]; 1592 } 1593 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1594 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 1595 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1596 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1597 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1598 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1599 1600 /* write dispatch packet */ 1601 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1602 ib.ptr[ib.length_dw++] = 8; /* x */ 1603 ib.ptr[ib.length_dw++] = 1; /* y */ 1604 ib.ptr[ib.length_dw++] = 1; /* z */ 1605 ib.ptr[ib.length_dw++] = 1606 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1607 1608 /* write CS partial flush packet */ 1609 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1610 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1611 1612 /* shedule the ib on the ring */ 1613 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1614 if (r) { 1615 drm_err(adev_to_drm(adev), "ib submit failed (%d).\n", r); 1616 goto fail; 1617 } 1618 1619 /* wait for the GPU to finish processing the IB */ 1620 r = dma_fence_wait(f, false); 1621 if (r) { 1622 drm_err(adev_to_drm(adev), "fence wait failed (%d).\n", r); 1623 goto fail; 1624 } 1625 1626 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2); 1627 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1); 1628 WREG32(mmGB_EDC_MODE, tmp); 1629 1630 tmp = RREG32(mmCC_GC_EDC_CONFIG); 1631 tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1; 1632 WREG32(mmCC_GC_EDC_CONFIG, tmp); 1633 1634 1635 /* read back registers to clear the counters */ 1636 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) 1637 RREG32(sec_ded_counter_registers[i]); 1638 1639 fail: 1640 amdgpu_ib_free(&ib, NULL); 1641 dma_fence_put(f); 1642 1643 return r; 1644 } 1645 1646 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev) 1647 { 1648 u32 gb_addr_config; 1649 u32 mc_arb_ramcfg; 1650 u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map; 1651 u32 tmp; 1652 int ret; 1653 1654 switch (adev->asic_type) { 1655 case CHIP_TOPAZ: 1656 adev->gfx.config.max_shader_engines = 1; 1657 adev->gfx.config.max_tile_pipes = 2; 1658 adev->gfx.config.max_cu_per_sh = 6; 1659 adev->gfx.config.max_sh_per_se = 1; 1660 adev->gfx.config.max_backends_per_se = 2; 1661 adev->gfx.config.max_texture_channel_caches = 2; 1662 adev->gfx.config.max_gprs = 256; 1663 adev->gfx.config.max_gs_threads = 32; 1664 adev->gfx.config.max_hw_contexts = 8; 1665 1666 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1667 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1668 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1669 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1670 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN; 1671 break; 1672 case CHIP_FIJI: 1673 adev->gfx.config.max_shader_engines = 4; 1674 adev->gfx.config.max_tile_pipes = 16; 1675 adev->gfx.config.max_cu_per_sh = 16; 1676 adev->gfx.config.max_sh_per_se = 1; 1677 adev->gfx.config.max_backends_per_se = 4; 1678 adev->gfx.config.max_texture_channel_caches = 16; 1679 adev->gfx.config.max_gprs = 256; 1680 adev->gfx.config.max_gs_threads = 32; 1681 adev->gfx.config.max_hw_contexts = 8; 1682 1683 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1684 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1685 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1686 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1687 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1688 break; 1689 case CHIP_POLARIS11: 1690 case CHIP_POLARIS12: 1691 ret = amdgpu_atombios_get_gfx_info(adev); 1692 if (ret) 1693 return ret; 1694 adev->gfx.config.max_gprs = 256; 1695 adev->gfx.config.max_gs_threads = 32; 1696 adev->gfx.config.max_hw_contexts = 8; 1697 1698 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1699 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1700 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1701 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1702 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN; 1703 break; 1704 case CHIP_POLARIS10: 1705 case CHIP_VEGAM: 1706 ret = amdgpu_atombios_get_gfx_info(adev); 1707 if (ret) 1708 return ret; 1709 adev->gfx.config.max_gprs = 256; 1710 adev->gfx.config.max_gs_threads = 32; 1711 adev->gfx.config.max_hw_contexts = 8; 1712 1713 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1714 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1715 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1716 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1717 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1718 break; 1719 case CHIP_TONGA: 1720 adev->gfx.config.max_shader_engines = 4; 1721 adev->gfx.config.max_tile_pipes = 8; 1722 adev->gfx.config.max_cu_per_sh = 8; 1723 adev->gfx.config.max_sh_per_se = 1; 1724 adev->gfx.config.max_backends_per_se = 2; 1725 adev->gfx.config.max_texture_channel_caches = 8; 1726 adev->gfx.config.max_gprs = 256; 1727 adev->gfx.config.max_gs_threads = 32; 1728 adev->gfx.config.max_hw_contexts = 8; 1729 1730 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1731 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1732 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1733 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1734 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1735 break; 1736 case CHIP_CARRIZO: 1737 adev->gfx.config.max_shader_engines = 1; 1738 adev->gfx.config.max_tile_pipes = 2; 1739 adev->gfx.config.max_sh_per_se = 1; 1740 adev->gfx.config.max_backends_per_se = 2; 1741 adev->gfx.config.max_cu_per_sh = 8; 1742 adev->gfx.config.max_texture_channel_caches = 2; 1743 adev->gfx.config.max_gprs = 256; 1744 adev->gfx.config.max_gs_threads = 32; 1745 adev->gfx.config.max_hw_contexts = 8; 1746 1747 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1748 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1749 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1750 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1751 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1752 break; 1753 case CHIP_STONEY: 1754 adev->gfx.config.max_shader_engines = 1; 1755 adev->gfx.config.max_tile_pipes = 2; 1756 adev->gfx.config.max_sh_per_se = 1; 1757 adev->gfx.config.max_backends_per_se = 1; 1758 adev->gfx.config.max_cu_per_sh = 3; 1759 adev->gfx.config.max_texture_channel_caches = 2; 1760 adev->gfx.config.max_gprs = 256; 1761 adev->gfx.config.max_gs_threads = 16; 1762 adev->gfx.config.max_hw_contexts = 8; 1763 1764 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1765 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1766 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1767 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1768 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1769 break; 1770 default: 1771 adev->gfx.config.max_shader_engines = 2; 1772 adev->gfx.config.max_tile_pipes = 4; 1773 adev->gfx.config.max_cu_per_sh = 2; 1774 adev->gfx.config.max_sh_per_se = 1; 1775 adev->gfx.config.max_backends_per_se = 2; 1776 adev->gfx.config.max_texture_channel_caches = 4; 1777 adev->gfx.config.max_gprs = 256; 1778 adev->gfx.config.max_gs_threads = 32; 1779 adev->gfx.config.max_hw_contexts = 8; 1780 1781 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1782 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1783 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1784 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1785 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1786 break; 1787 } 1788 1789 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG); 1790 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg; 1791 1792 adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg, 1793 MC_ARB_RAMCFG, NOOFBANK); 1794 adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg, 1795 MC_ARB_RAMCFG, NOOFRANKS); 1796 1797 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; 1798 adev->gfx.config.mem_max_burst_length_bytes = 256; 1799 if (adev->flags & AMD_IS_APU) { 1800 /* Get memory bank mapping mode. */ 1801 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING); 1802 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1803 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1804 1805 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING); 1806 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1807 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1808 1809 /* Validate settings in case only one DIMM installed. */ 1810 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12)) 1811 dimm00_addr_map = 0; 1812 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12)) 1813 dimm01_addr_map = 0; 1814 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12)) 1815 dimm10_addr_map = 0; 1816 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12)) 1817 dimm11_addr_map = 0; 1818 1819 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */ 1820 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */ 1821 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11)) 1822 adev->gfx.config.mem_row_size_in_kb = 2; 1823 else 1824 adev->gfx.config.mem_row_size_in_kb = 1; 1825 } else { 1826 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS); 1827 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 1828 if (adev->gfx.config.mem_row_size_in_kb > 4) 1829 adev->gfx.config.mem_row_size_in_kb = 4; 1830 } 1831 1832 adev->gfx.config.shader_engine_tile_size = 32; 1833 adev->gfx.config.num_gpus = 1; 1834 adev->gfx.config.multi_gpu_tile_size = 64; 1835 1836 /* fix up row size */ 1837 switch (adev->gfx.config.mem_row_size_in_kb) { 1838 case 1: 1839 default: 1840 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0); 1841 break; 1842 case 2: 1843 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1); 1844 break; 1845 case 4: 1846 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2); 1847 break; 1848 } 1849 adev->gfx.config.gb_addr_config = gb_addr_config; 1850 1851 return 0; 1852 } 1853 1854 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1855 int mec, int pipe, int queue) 1856 { 1857 int r; 1858 unsigned irq_type; 1859 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 1860 unsigned int hw_prio; 1861 1862 ring = &adev->gfx.compute_ring[ring_id]; 1863 1864 /* mec0 is me1 */ 1865 ring->me = mec + 1; 1866 ring->pipe = pipe; 1867 ring->queue = queue; 1868 1869 ring->ring_obj = NULL; 1870 ring->use_doorbell = true; 1871 ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id; 1872 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1873 + (ring_id * GFX8_MEC_HPD_SIZE); 1874 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1875 1876 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1877 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1878 + ring->pipe; 1879 1880 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1881 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT; 1882 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1883 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1884 hw_prio, NULL); 1885 if (r) 1886 return r; 1887 1888 1889 return 0; 1890 } 1891 1892 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work); 1893 1894 static int gfx_v8_0_sw_init(struct amdgpu_ip_block *ip_block) 1895 { 1896 int i, j, k, r, ring_id; 1897 int xcc_id = 0; 1898 struct amdgpu_ring *ring; 1899 struct amdgpu_device *adev = ip_block->adev; 1900 1901 switch (adev->asic_type) { 1902 case CHIP_TONGA: 1903 case CHIP_CARRIZO: 1904 case CHIP_FIJI: 1905 case CHIP_POLARIS10: 1906 case CHIP_POLARIS11: 1907 case CHIP_POLARIS12: 1908 case CHIP_VEGAM: 1909 adev->gfx.mec.num_mec = 2; 1910 break; 1911 case CHIP_TOPAZ: 1912 case CHIP_STONEY: 1913 default: 1914 adev->gfx.mec.num_mec = 1; 1915 break; 1916 } 1917 1918 adev->gfx.mec.num_pipe_per_mec = 4; 1919 adev->gfx.mec.num_queue_per_pipe = 8; 1920 1921 /* EOP Event */ 1922 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq); 1923 if (r) 1924 return r; 1925 1926 /* Privileged reg */ 1927 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT, 1928 &adev->gfx.priv_reg_irq); 1929 if (r) 1930 return r; 1931 1932 /* Privileged inst */ 1933 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT, 1934 &adev->gfx.priv_inst_irq); 1935 if (r) 1936 return r; 1937 1938 /* Add CP EDC/ECC irq */ 1939 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR, 1940 &adev->gfx.cp_ecc_error_irq); 1941 if (r) 1942 return r; 1943 1944 /* SQ interrupts. */ 1945 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG, 1946 &adev->gfx.sq_irq); 1947 if (r) { 1948 DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r); 1949 return r; 1950 } 1951 1952 INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func); 1953 1954 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1955 1956 r = gfx_v8_0_init_microcode(adev); 1957 if (r) { 1958 DRM_ERROR("Failed to load gfx firmware!\n"); 1959 return r; 1960 } 1961 1962 r = adev->gfx.rlc.funcs->init(adev); 1963 if (r) { 1964 DRM_ERROR("Failed to init rlc BOs!\n"); 1965 return r; 1966 } 1967 1968 r = gfx_v8_0_mec_init(adev); 1969 if (r) { 1970 DRM_ERROR("Failed to init MEC BOs!\n"); 1971 return r; 1972 } 1973 1974 /* set up the gfx ring */ 1975 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 1976 ring = &adev->gfx.gfx_ring[i]; 1977 ring->ring_obj = NULL; 1978 sprintf(ring->name, "gfx"); 1979 /* no gfx doorbells on iceland */ 1980 if (adev->asic_type != CHIP_TOPAZ) { 1981 ring->use_doorbell = true; 1982 ring->doorbell_index = adev->doorbell_index.gfx_ring0; 1983 } 1984 1985 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 1986 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 1987 AMDGPU_RING_PRIO_DEFAULT, NULL); 1988 if (r) 1989 return r; 1990 } 1991 1992 1993 /* set up the compute queues - allocate horizontally across pipes */ 1994 ring_id = 0; 1995 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1996 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1997 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1998 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 1999 k, j)) 2000 continue; 2001 2002 r = gfx_v8_0_compute_ring_init(adev, 2003 ring_id, 2004 i, k, j); 2005 if (r) 2006 return r; 2007 2008 ring_id++; 2009 } 2010 } 2011 } 2012 2013 r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE, 0); 2014 if (r) { 2015 DRM_ERROR("Failed to init KIQ BOs!\n"); 2016 return r; 2017 } 2018 2019 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 2020 if (r) 2021 return r; 2022 2023 /* create MQD for all compute queues as well as KIQ for SRIOV case */ 2024 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation), 0); 2025 if (r) 2026 return r; 2027 2028 adev->gfx.ce_ram_size = 0x8000; 2029 2030 r = gfx_v8_0_gpu_early_init(adev); 2031 if (r) 2032 return r; 2033 2034 adev->gfx.gfx_supported_reset = 2035 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 2036 adev->gfx.compute_supported_reset = 2037 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 2038 2039 if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_ip_block_soft_reset) { 2040 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; 2041 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; 2042 } 2043 2044 return 0; 2045 } 2046 2047 static int gfx_v8_0_sw_fini(struct amdgpu_ip_block *ip_block) 2048 { 2049 struct amdgpu_device *adev = ip_block->adev; 2050 int i; 2051 2052 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2053 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2054 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2055 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2056 2057 amdgpu_gfx_mqd_sw_fini(adev, 0); 2058 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 2059 amdgpu_gfx_kiq_fini(adev, 0); 2060 2061 gfx_v8_0_mec_fini(adev); 2062 amdgpu_gfx_rlc_fini(adev); 2063 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2064 &adev->gfx.rlc.clear_state_gpu_addr, 2065 (void **)&adev->gfx.rlc.cs_ptr); 2066 if ((adev->asic_type == CHIP_CARRIZO) || 2067 (adev->asic_type == CHIP_STONEY)) { 2068 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2069 &adev->gfx.rlc.cp_table_gpu_addr, 2070 (void **)&adev->gfx.rlc.cp_table_ptr); 2071 } 2072 gfx_v8_0_free_microcode(adev); 2073 2074 return 0; 2075 } 2076 2077 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev) 2078 { 2079 uint32_t *modearray, *mod2array; 2080 const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array); 2081 const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 2082 u32 reg_offset; 2083 2084 modearray = adev->gfx.config.tile_mode_array; 2085 mod2array = adev->gfx.config.macrotile_mode_array; 2086 2087 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2088 modearray[reg_offset] = 0; 2089 2090 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2091 mod2array[reg_offset] = 0; 2092 2093 switch (adev->asic_type) { 2094 case CHIP_TOPAZ: 2095 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2096 PIPE_CONFIG(ADDR_SURF_P2) | 2097 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2098 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2099 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2100 PIPE_CONFIG(ADDR_SURF_P2) | 2101 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2102 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2103 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2104 PIPE_CONFIG(ADDR_SURF_P2) | 2105 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2106 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2107 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2108 PIPE_CONFIG(ADDR_SURF_P2) | 2109 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2110 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2111 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2112 PIPE_CONFIG(ADDR_SURF_P2) | 2113 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2114 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2115 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2116 PIPE_CONFIG(ADDR_SURF_P2) | 2117 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2118 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2119 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2120 PIPE_CONFIG(ADDR_SURF_P2) | 2121 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2122 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2123 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2124 PIPE_CONFIG(ADDR_SURF_P2)); 2125 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2126 PIPE_CONFIG(ADDR_SURF_P2) | 2127 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2128 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2129 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2130 PIPE_CONFIG(ADDR_SURF_P2) | 2131 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2132 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2133 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2134 PIPE_CONFIG(ADDR_SURF_P2) | 2135 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2136 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2137 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2138 PIPE_CONFIG(ADDR_SURF_P2) | 2139 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2140 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2141 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2142 PIPE_CONFIG(ADDR_SURF_P2) | 2143 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2144 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2145 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2146 PIPE_CONFIG(ADDR_SURF_P2) | 2147 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2148 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2149 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2150 PIPE_CONFIG(ADDR_SURF_P2) | 2151 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2152 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2153 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2154 PIPE_CONFIG(ADDR_SURF_P2) | 2155 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2156 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2157 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2158 PIPE_CONFIG(ADDR_SURF_P2) | 2159 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2160 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2161 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2162 PIPE_CONFIG(ADDR_SURF_P2) | 2163 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2164 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2165 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2166 PIPE_CONFIG(ADDR_SURF_P2) | 2167 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2168 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2169 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2170 PIPE_CONFIG(ADDR_SURF_P2) | 2171 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2172 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2173 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2174 PIPE_CONFIG(ADDR_SURF_P2) | 2175 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2176 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2177 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2178 PIPE_CONFIG(ADDR_SURF_P2) | 2179 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2180 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2181 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2182 PIPE_CONFIG(ADDR_SURF_P2) | 2183 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2184 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2185 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2186 PIPE_CONFIG(ADDR_SURF_P2) | 2187 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2188 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2189 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2190 PIPE_CONFIG(ADDR_SURF_P2) | 2191 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2192 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2193 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2194 PIPE_CONFIG(ADDR_SURF_P2) | 2195 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2196 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2197 2198 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2199 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2200 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2201 NUM_BANKS(ADDR_SURF_8_BANK)); 2202 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2203 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2204 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2205 NUM_BANKS(ADDR_SURF_8_BANK)); 2206 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2207 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2208 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2209 NUM_BANKS(ADDR_SURF_8_BANK)); 2210 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2211 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2212 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2213 NUM_BANKS(ADDR_SURF_8_BANK)); 2214 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2215 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2216 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2217 NUM_BANKS(ADDR_SURF_8_BANK)); 2218 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2219 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2220 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2221 NUM_BANKS(ADDR_SURF_8_BANK)); 2222 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2223 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2224 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2225 NUM_BANKS(ADDR_SURF_8_BANK)); 2226 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2227 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2228 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2229 NUM_BANKS(ADDR_SURF_16_BANK)); 2230 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2231 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2232 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2233 NUM_BANKS(ADDR_SURF_16_BANK)); 2234 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2235 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2236 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2237 NUM_BANKS(ADDR_SURF_16_BANK)); 2238 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2239 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2240 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2241 NUM_BANKS(ADDR_SURF_16_BANK)); 2242 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2243 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2244 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2245 NUM_BANKS(ADDR_SURF_16_BANK)); 2246 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2247 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2248 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2249 NUM_BANKS(ADDR_SURF_16_BANK)); 2250 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2251 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2252 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2253 NUM_BANKS(ADDR_SURF_8_BANK)); 2254 2255 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2256 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 2257 reg_offset != 23) 2258 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2259 2260 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2261 if (reg_offset != 7) 2262 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2263 2264 break; 2265 case CHIP_FIJI: 2266 case CHIP_VEGAM: 2267 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2268 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2269 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2270 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2271 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2272 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2273 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2274 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2275 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2276 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2277 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2278 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2279 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2280 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2281 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2282 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2283 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2284 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2285 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2286 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2287 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2288 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2289 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2290 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2291 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2292 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2293 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2294 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2295 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2296 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2297 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2298 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2299 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2300 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2301 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2302 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2303 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2304 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2305 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2306 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2307 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2308 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2309 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2310 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2311 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2312 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2313 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2314 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2315 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2316 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2317 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2318 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2319 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2320 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2321 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2322 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2323 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2324 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2325 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2326 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2327 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2328 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2329 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2330 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2331 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2332 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2333 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2334 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2335 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2336 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2337 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2338 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2339 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2340 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2341 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2342 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2343 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2344 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2345 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2346 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2347 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2348 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2349 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2350 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2351 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2352 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2353 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2354 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2355 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2356 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2357 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2358 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2359 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2360 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2361 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2362 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2363 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2364 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2365 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2366 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2367 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2368 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2369 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2370 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2371 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2372 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2373 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2374 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2375 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2376 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2377 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2378 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2379 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2380 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2381 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2382 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2383 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2384 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2385 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2386 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2387 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2388 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2389 2390 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2391 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2392 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2393 NUM_BANKS(ADDR_SURF_8_BANK)); 2394 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2395 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2396 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2397 NUM_BANKS(ADDR_SURF_8_BANK)); 2398 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2399 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2400 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2401 NUM_BANKS(ADDR_SURF_8_BANK)); 2402 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2403 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2404 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2405 NUM_BANKS(ADDR_SURF_8_BANK)); 2406 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2407 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2408 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2409 NUM_BANKS(ADDR_SURF_8_BANK)); 2410 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2411 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2412 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2413 NUM_BANKS(ADDR_SURF_8_BANK)); 2414 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2415 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2416 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2417 NUM_BANKS(ADDR_SURF_8_BANK)); 2418 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2419 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2420 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2421 NUM_BANKS(ADDR_SURF_8_BANK)); 2422 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2423 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2424 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2425 NUM_BANKS(ADDR_SURF_8_BANK)); 2426 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2427 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2428 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2429 NUM_BANKS(ADDR_SURF_8_BANK)); 2430 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2431 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2432 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2433 NUM_BANKS(ADDR_SURF_8_BANK)); 2434 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2435 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2436 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2437 NUM_BANKS(ADDR_SURF_8_BANK)); 2438 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2439 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2440 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2441 NUM_BANKS(ADDR_SURF_8_BANK)); 2442 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2443 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2444 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2445 NUM_BANKS(ADDR_SURF_4_BANK)); 2446 2447 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2448 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2449 2450 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2451 if (reg_offset != 7) 2452 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2453 2454 break; 2455 case CHIP_TONGA: 2456 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2457 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2458 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2459 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2460 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2461 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2462 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2463 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2464 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2465 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2466 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2467 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2468 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2469 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2470 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2471 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2472 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2473 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2474 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2475 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2476 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2477 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2478 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2479 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2480 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2481 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2482 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2483 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2484 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2485 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2486 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2487 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2488 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2489 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2490 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2491 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2492 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2493 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2494 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2495 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2496 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2497 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2498 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2499 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2500 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2501 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2502 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2503 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2504 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2505 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2506 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2507 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2508 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2509 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2510 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2511 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2512 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2513 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2514 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2515 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2516 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2517 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2518 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2519 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2520 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2521 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2522 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2523 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2524 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2525 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2526 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2527 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2528 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2529 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2530 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2531 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2532 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2533 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2534 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2535 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2536 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2537 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2538 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2539 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2540 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2541 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2542 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2544 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2545 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2546 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2547 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2548 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2549 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2550 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2551 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2552 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2553 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2554 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2555 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2556 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2557 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2558 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2559 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2560 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2561 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2562 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2563 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2564 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2565 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2566 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2567 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2568 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2569 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2570 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2571 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2572 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2573 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2574 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2575 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2576 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2577 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2578 2579 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2580 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2581 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2582 NUM_BANKS(ADDR_SURF_16_BANK)); 2583 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2584 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2585 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2586 NUM_BANKS(ADDR_SURF_16_BANK)); 2587 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2588 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2589 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2590 NUM_BANKS(ADDR_SURF_16_BANK)); 2591 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2594 NUM_BANKS(ADDR_SURF_16_BANK)); 2595 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2596 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2597 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2598 NUM_BANKS(ADDR_SURF_16_BANK)); 2599 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2600 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2601 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2602 NUM_BANKS(ADDR_SURF_16_BANK)); 2603 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2604 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2605 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2606 NUM_BANKS(ADDR_SURF_16_BANK)); 2607 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2608 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2609 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2610 NUM_BANKS(ADDR_SURF_16_BANK)); 2611 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2612 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2613 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2614 NUM_BANKS(ADDR_SURF_16_BANK)); 2615 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2616 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2617 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2618 NUM_BANKS(ADDR_SURF_16_BANK)); 2619 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2620 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2621 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2622 NUM_BANKS(ADDR_SURF_16_BANK)); 2623 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2624 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2625 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2626 NUM_BANKS(ADDR_SURF_8_BANK)); 2627 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2630 NUM_BANKS(ADDR_SURF_4_BANK)); 2631 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2632 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2633 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2634 NUM_BANKS(ADDR_SURF_4_BANK)); 2635 2636 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2637 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2638 2639 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2640 if (reg_offset != 7) 2641 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2642 2643 break; 2644 case CHIP_POLARIS11: 2645 case CHIP_POLARIS12: 2646 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2647 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2648 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2649 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2650 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2651 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2652 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2653 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2654 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2655 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2656 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2657 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2658 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2659 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2660 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2661 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2662 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2663 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2664 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2665 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2666 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2667 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2668 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2669 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2670 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2671 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2672 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2673 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2674 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2675 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2676 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2677 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2678 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2679 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2680 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2681 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2682 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2683 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2684 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2685 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2686 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2687 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2688 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2689 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2690 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2691 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2692 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2693 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2694 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2695 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2696 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2697 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2698 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2699 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2700 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2701 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2702 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2703 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2704 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2705 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2706 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2707 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2708 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2709 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2710 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2711 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2712 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2713 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2714 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2715 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2716 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2717 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2718 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2719 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2720 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2721 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2722 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2723 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2724 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2725 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2726 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2727 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2728 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2729 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2730 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2731 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2732 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2733 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2734 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2735 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2736 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2737 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2738 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2739 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2740 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2741 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2742 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2743 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2744 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2745 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2746 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2747 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2748 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2749 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2750 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2751 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2752 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2753 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2754 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2755 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2756 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2757 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2758 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2759 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2760 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2761 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2762 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2763 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2764 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2765 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2766 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2767 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2768 2769 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2770 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2771 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2772 NUM_BANKS(ADDR_SURF_16_BANK)); 2773 2774 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2775 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2776 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2777 NUM_BANKS(ADDR_SURF_16_BANK)); 2778 2779 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2782 NUM_BANKS(ADDR_SURF_16_BANK)); 2783 2784 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2785 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2786 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2787 NUM_BANKS(ADDR_SURF_16_BANK)); 2788 2789 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2790 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2791 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2792 NUM_BANKS(ADDR_SURF_16_BANK)); 2793 2794 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2795 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2796 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2797 NUM_BANKS(ADDR_SURF_16_BANK)); 2798 2799 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2800 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2801 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2802 NUM_BANKS(ADDR_SURF_16_BANK)); 2803 2804 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2805 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2806 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2807 NUM_BANKS(ADDR_SURF_16_BANK)); 2808 2809 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2810 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2811 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2812 NUM_BANKS(ADDR_SURF_16_BANK)); 2813 2814 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2815 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2816 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2817 NUM_BANKS(ADDR_SURF_16_BANK)); 2818 2819 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2820 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2821 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2822 NUM_BANKS(ADDR_SURF_16_BANK)); 2823 2824 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2827 NUM_BANKS(ADDR_SURF_16_BANK)); 2828 2829 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2830 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2831 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2832 NUM_BANKS(ADDR_SURF_8_BANK)); 2833 2834 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2835 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2836 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2837 NUM_BANKS(ADDR_SURF_4_BANK)); 2838 2839 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2840 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2841 2842 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2843 if (reg_offset != 7) 2844 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2845 2846 break; 2847 case CHIP_POLARIS10: 2848 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2849 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2850 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2851 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2852 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2853 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2854 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2855 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2856 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2857 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2858 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2859 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2860 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2861 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2862 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2863 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2864 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2865 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2866 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2867 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2868 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2869 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2870 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2871 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2872 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2873 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2874 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2875 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2876 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2877 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2878 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2879 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2880 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2881 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2882 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2883 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2884 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2885 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2886 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2887 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2888 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2889 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2890 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2891 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2892 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2893 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2894 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2895 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2896 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2897 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2898 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2899 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2900 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2901 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2902 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2903 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2904 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2905 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2906 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2907 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2908 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2909 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2910 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2911 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2912 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2913 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2914 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2915 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2916 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2917 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2918 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2919 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2920 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2921 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2922 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2923 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2924 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2925 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2926 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2927 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2928 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2929 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2930 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2931 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2932 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2933 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2934 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2935 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2936 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2937 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2938 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2939 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2940 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2941 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2942 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2943 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2944 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2945 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2946 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2947 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2948 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2949 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2950 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2951 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2952 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2953 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2954 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2955 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2956 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2957 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2958 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2959 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2960 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2961 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2962 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2963 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2964 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2965 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2966 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2967 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2968 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2969 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2970 2971 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2972 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2973 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2974 NUM_BANKS(ADDR_SURF_16_BANK)); 2975 2976 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2977 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2978 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2979 NUM_BANKS(ADDR_SURF_16_BANK)); 2980 2981 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2982 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2983 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2984 NUM_BANKS(ADDR_SURF_16_BANK)); 2985 2986 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2987 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2988 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2989 NUM_BANKS(ADDR_SURF_16_BANK)); 2990 2991 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2992 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2993 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2994 NUM_BANKS(ADDR_SURF_16_BANK)); 2995 2996 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2997 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2998 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2999 NUM_BANKS(ADDR_SURF_16_BANK)); 3000 3001 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3002 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3003 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3004 NUM_BANKS(ADDR_SURF_16_BANK)); 3005 3006 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3007 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3008 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3009 NUM_BANKS(ADDR_SURF_16_BANK)); 3010 3011 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3012 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3013 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3014 NUM_BANKS(ADDR_SURF_16_BANK)); 3015 3016 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3017 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3018 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3019 NUM_BANKS(ADDR_SURF_16_BANK)); 3020 3021 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3022 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3023 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3024 NUM_BANKS(ADDR_SURF_16_BANK)); 3025 3026 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3027 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3028 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3029 NUM_BANKS(ADDR_SURF_8_BANK)); 3030 3031 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3032 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3033 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3034 NUM_BANKS(ADDR_SURF_4_BANK)); 3035 3036 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3037 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3038 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3039 NUM_BANKS(ADDR_SURF_4_BANK)); 3040 3041 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3042 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3043 3044 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3045 if (reg_offset != 7) 3046 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3047 3048 break; 3049 case CHIP_STONEY: 3050 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3051 PIPE_CONFIG(ADDR_SURF_P2) | 3052 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3053 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3054 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3055 PIPE_CONFIG(ADDR_SURF_P2) | 3056 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3057 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3058 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3059 PIPE_CONFIG(ADDR_SURF_P2) | 3060 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3061 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3062 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3063 PIPE_CONFIG(ADDR_SURF_P2) | 3064 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3065 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3066 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3067 PIPE_CONFIG(ADDR_SURF_P2) | 3068 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3069 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3070 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3071 PIPE_CONFIG(ADDR_SURF_P2) | 3072 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3073 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3074 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3075 PIPE_CONFIG(ADDR_SURF_P2) | 3076 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3077 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3078 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3079 PIPE_CONFIG(ADDR_SURF_P2)); 3080 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3081 PIPE_CONFIG(ADDR_SURF_P2) | 3082 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3083 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3084 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3085 PIPE_CONFIG(ADDR_SURF_P2) | 3086 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3087 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3088 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3089 PIPE_CONFIG(ADDR_SURF_P2) | 3090 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3091 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3092 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3093 PIPE_CONFIG(ADDR_SURF_P2) | 3094 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3095 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3096 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3097 PIPE_CONFIG(ADDR_SURF_P2) | 3098 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3099 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3100 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3101 PIPE_CONFIG(ADDR_SURF_P2) | 3102 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3103 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3104 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3105 PIPE_CONFIG(ADDR_SURF_P2) | 3106 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3107 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3108 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3109 PIPE_CONFIG(ADDR_SURF_P2) | 3110 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3111 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3112 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3113 PIPE_CONFIG(ADDR_SURF_P2) | 3114 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3115 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3116 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3117 PIPE_CONFIG(ADDR_SURF_P2) | 3118 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3119 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3120 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3121 PIPE_CONFIG(ADDR_SURF_P2) | 3122 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3123 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3124 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3125 PIPE_CONFIG(ADDR_SURF_P2) | 3126 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3127 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3128 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3129 PIPE_CONFIG(ADDR_SURF_P2) | 3130 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3131 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3132 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3133 PIPE_CONFIG(ADDR_SURF_P2) | 3134 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3135 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3136 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3137 PIPE_CONFIG(ADDR_SURF_P2) | 3138 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3139 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3140 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3141 PIPE_CONFIG(ADDR_SURF_P2) | 3142 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3143 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3144 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3145 PIPE_CONFIG(ADDR_SURF_P2) | 3146 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3147 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3148 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3149 PIPE_CONFIG(ADDR_SURF_P2) | 3150 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3151 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3152 3153 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3154 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3155 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3156 NUM_BANKS(ADDR_SURF_8_BANK)); 3157 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3158 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3159 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3160 NUM_BANKS(ADDR_SURF_8_BANK)); 3161 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3162 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3163 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3164 NUM_BANKS(ADDR_SURF_8_BANK)); 3165 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3166 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3167 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3168 NUM_BANKS(ADDR_SURF_8_BANK)); 3169 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3170 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3171 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3172 NUM_BANKS(ADDR_SURF_8_BANK)); 3173 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3174 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3175 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3176 NUM_BANKS(ADDR_SURF_8_BANK)); 3177 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3178 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3179 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3180 NUM_BANKS(ADDR_SURF_8_BANK)); 3181 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3182 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3183 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3184 NUM_BANKS(ADDR_SURF_16_BANK)); 3185 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3186 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3187 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3188 NUM_BANKS(ADDR_SURF_16_BANK)); 3189 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3190 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3191 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3192 NUM_BANKS(ADDR_SURF_16_BANK)); 3193 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3194 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3195 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3196 NUM_BANKS(ADDR_SURF_16_BANK)); 3197 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3200 NUM_BANKS(ADDR_SURF_16_BANK)); 3201 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3202 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3203 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3204 NUM_BANKS(ADDR_SURF_16_BANK)); 3205 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3206 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3207 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3208 NUM_BANKS(ADDR_SURF_8_BANK)); 3209 3210 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3211 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3212 reg_offset != 23) 3213 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3214 3215 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3216 if (reg_offset != 7) 3217 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3218 3219 break; 3220 default: 3221 dev_warn(adev->dev, 3222 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n", 3223 adev->asic_type); 3224 fallthrough; 3225 3226 case CHIP_CARRIZO: 3227 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3228 PIPE_CONFIG(ADDR_SURF_P2) | 3229 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3230 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3231 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3232 PIPE_CONFIG(ADDR_SURF_P2) | 3233 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3234 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3235 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3236 PIPE_CONFIG(ADDR_SURF_P2) | 3237 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3238 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3239 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3240 PIPE_CONFIG(ADDR_SURF_P2) | 3241 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3242 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3243 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3244 PIPE_CONFIG(ADDR_SURF_P2) | 3245 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3246 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3247 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3248 PIPE_CONFIG(ADDR_SURF_P2) | 3249 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3250 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3251 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3252 PIPE_CONFIG(ADDR_SURF_P2) | 3253 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3254 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3255 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3256 PIPE_CONFIG(ADDR_SURF_P2)); 3257 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3258 PIPE_CONFIG(ADDR_SURF_P2) | 3259 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3260 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3261 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3262 PIPE_CONFIG(ADDR_SURF_P2) | 3263 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3264 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3265 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3266 PIPE_CONFIG(ADDR_SURF_P2) | 3267 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3268 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3269 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3270 PIPE_CONFIG(ADDR_SURF_P2) | 3271 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3272 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3273 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3274 PIPE_CONFIG(ADDR_SURF_P2) | 3275 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3276 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3277 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3278 PIPE_CONFIG(ADDR_SURF_P2) | 3279 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3280 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3281 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3282 PIPE_CONFIG(ADDR_SURF_P2) | 3283 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3284 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3285 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3286 PIPE_CONFIG(ADDR_SURF_P2) | 3287 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3288 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3289 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3290 PIPE_CONFIG(ADDR_SURF_P2) | 3291 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3292 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3293 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3294 PIPE_CONFIG(ADDR_SURF_P2) | 3295 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3296 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3297 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3298 PIPE_CONFIG(ADDR_SURF_P2) | 3299 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3300 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3301 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3302 PIPE_CONFIG(ADDR_SURF_P2) | 3303 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3304 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3305 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3306 PIPE_CONFIG(ADDR_SURF_P2) | 3307 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3308 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3309 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3310 PIPE_CONFIG(ADDR_SURF_P2) | 3311 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3312 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3313 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3314 PIPE_CONFIG(ADDR_SURF_P2) | 3315 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3316 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3317 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3318 PIPE_CONFIG(ADDR_SURF_P2) | 3319 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3320 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3321 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3322 PIPE_CONFIG(ADDR_SURF_P2) | 3323 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3324 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3325 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3326 PIPE_CONFIG(ADDR_SURF_P2) | 3327 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3328 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3329 3330 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3331 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3332 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3333 NUM_BANKS(ADDR_SURF_8_BANK)); 3334 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3335 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3336 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3337 NUM_BANKS(ADDR_SURF_8_BANK)); 3338 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3339 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3340 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3341 NUM_BANKS(ADDR_SURF_8_BANK)); 3342 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3343 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3344 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3345 NUM_BANKS(ADDR_SURF_8_BANK)); 3346 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3347 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3348 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3349 NUM_BANKS(ADDR_SURF_8_BANK)); 3350 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3351 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3352 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3353 NUM_BANKS(ADDR_SURF_8_BANK)); 3354 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3355 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3356 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3357 NUM_BANKS(ADDR_SURF_8_BANK)); 3358 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3359 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3360 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3361 NUM_BANKS(ADDR_SURF_16_BANK)); 3362 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3363 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3364 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3365 NUM_BANKS(ADDR_SURF_16_BANK)); 3366 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3369 NUM_BANKS(ADDR_SURF_16_BANK)); 3370 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3371 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3372 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3373 NUM_BANKS(ADDR_SURF_16_BANK)); 3374 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3375 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3376 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3377 NUM_BANKS(ADDR_SURF_16_BANK)); 3378 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3379 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3380 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3381 NUM_BANKS(ADDR_SURF_16_BANK)); 3382 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3383 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3384 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3385 NUM_BANKS(ADDR_SURF_8_BANK)); 3386 3387 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3388 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3389 reg_offset != 23) 3390 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3391 3392 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3393 if (reg_offset != 7) 3394 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3395 3396 break; 3397 } 3398 } 3399 3400 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, 3401 u32 se_num, u32 sh_num, u32 instance, 3402 int xcc_id) 3403 { 3404 u32 data; 3405 3406 if (instance == 0xffffffff) 3407 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 3408 else 3409 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 3410 3411 if (se_num == 0xffffffff) 3412 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 3413 else 3414 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 3415 3416 if (sh_num == 0xffffffff) 3417 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 3418 else 3419 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 3420 3421 WREG32(mmGRBM_GFX_INDEX, data); 3422 } 3423 3424 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev, 3425 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 3426 { 3427 vi_srbm_select(adev, me, pipe, q, vm); 3428 } 3429 3430 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev) 3431 { 3432 u32 data, mask; 3433 3434 data = RREG32(mmCC_RB_BACKEND_DISABLE) | 3435 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3436 3437 data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE); 3438 3439 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 3440 adev->gfx.config.max_sh_per_se); 3441 3442 return (~data) & mask; 3443 } 3444 3445 static void 3446 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1) 3447 { 3448 switch (adev->asic_type) { 3449 case CHIP_FIJI: 3450 case CHIP_VEGAM: 3451 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) | 3452 RB_XSEL2(1) | PKR_MAP(2) | 3453 PKR_XSEL(1) | PKR_YSEL(1) | 3454 SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3); 3455 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) | 3456 SE_PAIR_YSEL(2); 3457 break; 3458 case CHIP_TONGA: 3459 case CHIP_POLARIS10: 3460 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3461 SE_XSEL(1) | SE_YSEL(1); 3462 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) | 3463 SE_PAIR_YSEL(2); 3464 break; 3465 case CHIP_TOPAZ: 3466 case CHIP_CARRIZO: 3467 *rconf |= RB_MAP_PKR0(2); 3468 *rconf1 |= 0x0; 3469 break; 3470 case CHIP_POLARIS11: 3471 case CHIP_POLARIS12: 3472 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3473 SE_XSEL(1) | SE_YSEL(1); 3474 *rconf1 |= 0x0; 3475 break; 3476 case CHIP_STONEY: 3477 *rconf |= 0x0; 3478 *rconf1 |= 0x0; 3479 break; 3480 default: 3481 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type); 3482 break; 3483 } 3484 } 3485 3486 static void 3487 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev, 3488 u32 raster_config, u32 raster_config_1, 3489 unsigned rb_mask, unsigned num_rb) 3490 { 3491 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1); 3492 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1); 3493 unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2); 3494 unsigned rb_per_se = num_rb / num_se; 3495 unsigned se_mask[4]; 3496 unsigned se; 3497 3498 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask; 3499 se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask; 3500 se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask; 3501 se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask; 3502 3503 WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4)); 3504 WARN_ON(!(sh_per_se == 1 || sh_per_se == 2)); 3505 WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2)); 3506 3507 if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) || 3508 (!se_mask[2] && !se_mask[3]))) { 3509 raster_config_1 &= ~SE_PAIR_MAP_MASK; 3510 3511 if (!se_mask[0] && !se_mask[1]) { 3512 raster_config_1 |= 3513 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3); 3514 } else { 3515 raster_config_1 |= 3516 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0); 3517 } 3518 } 3519 3520 for (se = 0; se < num_se; se++) { 3521 unsigned raster_config_se = raster_config; 3522 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se); 3523 unsigned pkr1_mask = pkr0_mask << rb_per_pkr; 3524 int idx = (se / 2) * 2; 3525 3526 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) { 3527 raster_config_se &= ~SE_MAP_MASK; 3528 3529 if (!se_mask[idx]) { 3530 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3); 3531 } else { 3532 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0); 3533 } 3534 } 3535 3536 pkr0_mask &= rb_mask; 3537 pkr1_mask &= rb_mask; 3538 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) { 3539 raster_config_se &= ~PKR_MAP_MASK; 3540 3541 if (!pkr0_mask) { 3542 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3); 3543 } else { 3544 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0); 3545 } 3546 } 3547 3548 if (rb_per_se >= 2) { 3549 unsigned rb0_mask = 1 << (se * rb_per_se); 3550 unsigned rb1_mask = rb0_mask << 1; 3551 3552 rb0_mask &= rb_mask; 3553 rb1_mask &= rb_mask; 3554 if (!rb0_mask || !rb1_mask) { 3555 raster_config_se &= ~RB_MAP_PKR0_MASK; 3556 3557 if (!rb0_mask) { 3558 raster_config_se |= 3559 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3); 3560 } else { 3561 raster_config_se |= 3562 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0); 3563 } 3564 } 3565 3566 if (rb_per_se > 2) { 3567 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr); 3568 rb1_mask = rb0_mask << 1; 3569 rb0_mask &= rb_mask; 3570 rb1_mask &= rb_mask; 3571 if (!rb0_mask || !rb1_mask) { 3572 raster_config_se &= ~RB_MAP_PKR1_MASK; 3573 3574 if (!rb0_mask) { 3575 raster_config_se |= 3576 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3); 3577 } else { 3578 raster_config_se |= 3579 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0); 3580 } 3581 } 3582 } 3583 } 3584 3585 /* GRBM_GFX_INDEX has a different offset on VI */ 3586 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff, 0); 3587 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se); 3588 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3589 } 3590 3591 /* GRBM_GFX_INDEX has a different offset on VI */ 3592 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3593 } 3594 3595 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev) 3596 { 3597 int i, j; 3598 u32 data; 3599 u32 raster_config = 0, raster_config_1 = 0; 3600 u32 active_rbs = 0; 3601 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 3602 adev->gfx.config.max_sh_per_se; 3603 unsigned num_rb_pipes; 3604 3605 mutex_lock(&adev->grbm_idx_mutex); 3606 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3607 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3608 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3609 data = gfx_v8_0_get_rb_active_bitmap(adev); 3610 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 3611 rb_bitmap_width_per_sh); 3612 } 3613 } 3614 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3615 3616 adev->gfx.config.backend_enable_mask = active_rbs; 3617 adev->gfx.config.num_rbs = hweight32(active_rbs); 3618 3619 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se * 3620 adev->gfx.config.max_shader_engines, 16); 3621 3622 gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1); 3623 3624 if (!adev->gfx.config.backend_enable_mask || 3625 adev->gfx.config.num_rbs >= num_rb_pipes) { 3626 WREG32(mmPA_SC_RASTER_CONFIG, raster_config); 3627 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3628 } else { 3629 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1, 3630 adev->gfx.config.backend_enable_mask, 3631 num_rb_pipes); 3632 } 3633 3634 /* cache the values for userspace */ 3635 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3636 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3637 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3638 adev->gfx.config.rb_config[i][j].rb_backend_disable = 3639 RREG32(mmCC_RB_BACKEND_DISABLE); 3640 adev->gfx.config.rb_config[i][j].user_rb_backend_disable = 3641 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3642 adev->gfx.config.rb_config[i][j].raster_config = 3643 RREG32(mmPA_SC_RASTER_CONFIG); 3644 adev->gfx.config.rb_config[i][j].raster_config_1 = 3645 RREG32(mmPA_SC_RASTER_CONFIG_1); 3646 } 3647 } 3648 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3649 mutex_unlock(&adev->grbm_idx_mutex); 3650 } 3651 3652 #define DEFAULT_SH_MEM_BASES (0x6000) 3653 /** 3654 * gfx_v8_0_init_compute_vmid - gart enable 3655 * 3656 * @adev: amdgpu_device pointer 3657 * 3658 * Initialize compute vmid sh_mem registers 3659 * 3660 */ 3661 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev) 3662 { 3663 int i; 3664 uint32_t sh_mem_config; 3665 uint32_t sh_mem_bases; 3666 3667 /* 3668 * Configure apertures: 3669 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 3670 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 3671 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 3672 */ 3673 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 3674 3675 sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 << 3676 SH_MEM_CONFIG__ADDRESS_MODE__SHIFT | 3677 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 3678 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT | 3679 MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | 3680 SH_MEM_CONFIG__PRIVATE_ATC_MASK; 3681 3682 mutex_lock(&adev->srbm_mutex); 3683 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3684 vi_srbm_select(adev, 0, 0, 0, i); 3685 /* CP and shaders */ 3686 WREG32(mmSH_MEM_CONFIG, sh_mem_config); 3687 WREG32(mmSH_MEM_APE1_BASE, 1); 3688 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3689 WREG32(mmSH_MEM_BASES, sh_mem_bases); 3690 } 3691 vi_srbm_select(adev, 0, 0, 0, 0); 3692 mutex_unlock(&adev->srbm_mutex); 3693 3694 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 3695 access. These should be enabled by FW for target VMIDs. */ 3696 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3697 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0); 3698 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0); 3699 WREG32(amdgpu_gds_reg_offset[i].gws, 0); 3700 WREG32(amdgpu_gds_reg_offset[i].oa, 0); 3701 } 3702 } 3703 3704 static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev) 3705 { 3706 int vmid; 3707 3708 /* 3709 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 3710 * access. Compute VMIDs should be enabled by FW for target VMIDs, 3711 * the driver can enable them for graphics. VMID0 should maintain 3712 * access so that HWS firmware can save/restore entries. 3713 */ 3714 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 3715 WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0); 3716 WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0); 3717 WREG32(amdgpu_gds_reg_offset[vmid].gws, 0); 3718 WREG32(amdgpu_gds_reg_offset[vmid].oa, 0); 3719 } 3720 } 3721 3722 static void gfx_v8_0_config_init(struct amdgpu_device *adev) 3723 { 3724 switch (adev->asic_type) { 3725 default: 3726 adev->gfx.config.double_offchip_lds_buf = 1; 3727 break; 3728 case CHIP_CARRIZO: 3729 case CHIP_STONEY: 3730 adev->gfx.config.double_offchip_lds_buf = 0; 3731 break; 3732 } 3733 } 3734 3735 static void gfx_v8_0_constants_init(struct amdgpu_device *adev) 3736 { 3737 u32 tmp, sh_static_mem_cfg; 3738 int i; 3739 3740 WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF); 3741 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3742 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3743 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config); 3744 3745 gfx_v8_0_tiling_mode_table_init(adev); 3746 gfx_v8_0_setup_rb(adev); 3747 gfx_v8_0_get_cu_info(adev); 3748 gfx_v8_0_config_init(adev); 3749 3750 /* XXX SH_MEM regs */ 3751 /* where to put LDS, scratch, GPUVM in FSA64 space */ 3752 sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG, 3753 SWIZZLE_ENABLE, 1); 3754 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3755 ELEMENT_SIZE, 1); 3756 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3757 INDEX_STRIDE, 3); 3758 WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); 3759 3760 mutex_lock(&adev->srbm_mutex); 3761 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { 3762 vi_srbm_select(adev, 0, 0, 0, i); 3763 /* CP and shaders */ 3764 if (i == 0) { 3765 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC); 3766 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3767 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3768 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3769 WREG32(mmSH_MEM_CONFIG, tmp); 3770 WREG32(mmSH_MEM_BASES, 0); 3771 } else { 3772 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC); 3773 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3774 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3775 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3776 WREG32(mmSH_MEM_CONFIG, tmp); 3777 tmp = adev->gmc.shared_aperture_start >> 48; 3778 WREG32(mmSH_MEM_BASES, tmp); 3779 } 3780 3781 WREG32(mmSH_MEM_APE1_BASE, 1); 3782 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3783 } 3784 vi_srbm_select(adev, 0, 0, 0, 0); 3785 mutex_unlock(&adev->srbm_mutex); 3786 3787 gfx_v8_0_init_compute_vmid(adev); 3788 gfx_v8_0_init_gds_vmid(adev); 3789 3790 mutex_lock(&adev->grbm_idx_mutex); 3791 /* 3792 * making sure that the following register writes will be broadcasted 3793 * to all the shaders 3794 */ 3795 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3796 3797 WREG32(mmPA_SC_FIFO_SIZE, 3798 (adev->gfx.config.sc_prim_fifo_size_frontend << 3799 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) | 3800 (adev->gfx.config.sc_prim_fifo_size_backend << 3801 PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) | 3802 (adev->gfx.config.sc_hiz_tile_fifo_size << 3803 PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) | 3804 (adev->gfx.config.sc_earlyz_tile_fifo_size << 3805 PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)); 3806 3807 tmp = RREG32(mmSPI_ARB_PRIORITY); 3808 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2); 3809 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2); 3810 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2); 3811 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2); 3812 WREG32(mmSPI_ARB_PRIORITY, tmp); 3813 3814 mutex_unlock(&adev->grbm_idx_mutex); 3815 3816 } 3817 3818 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 3819 { 3820 u32 i, j, k; 3821 u32 mask; 3822 3823 mutex_lock(&adev->grbm_idx_mutex); 3824 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3825 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3826 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3827 for (k = 0; k < adev->usec_timeout; k++) { 3828 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0) 3829 break; 3830 udelay(1); 3831 } 3832 if (k == adev->usec_timeout) { 3833 gfx_v8_0_select_se_sh(adev, 0xffffffff, 3834 0xffffffff, 0xffffffff, 0); 3835 mutex_unlock(&adev->grbm_idx_mutex); 3836 drm_info(adev_to_drm(adev), "Timeout wait for RLC serdes %u,%u\n", 3837 i, j); 3838 return; 3839 } 3840 } 3841 } 3842 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3843 mutex_unlock(&adev->grbm_idx_mutex); 3844 3845 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 3846 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 3847 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 3848 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 3849 for (k = 0; k < adev->usec_timeout; k++) { 3850 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 3851 break; 3852 udelay(1); 3853 } 3854 } 3855 3856 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 3857 bool enable) 3858 { 3859 u32 tmp = RREG32(mmCP_INT_CNTL_RING0); 3860 3861 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 3862 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 3863 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 3864 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 3865 3866 WREG32(mmCP_INT_CNTL_RING0, tmp); 3867 } 3868 3869 static void gfx_v8_0_init_csb(struct amdgpu_device *adev) 3870 { 3871 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 3872 /* csib */ 3873 WREG32(mmRLC_CSIB_ADDR_HI, 3874 adev->gfx.rlc.clear_state_gpu_addr >> 32); 3875 WREG32(mmRLC_CSIB_ADDR_LO, 3876 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 3877 WREG32(mmRLC_CSIB_LENGTH, 3878 adev->gfx.rlc.clear_state_size); 3879 } 3880 3881 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format, 3882 int ind_offset, 3883 int list_size, 3884 int *unique_indices, 3885 int *indices_count, 3886 int max_indices, 3887 int *ind_start_offsets, 3888 int *offset_count, 3889 int max_offset) 3890 { 3891 int indices; 3892 bool new_entry = true; 3893 3894 for (; ind_offset < list_size; ind_offset++) { 3895 3896 if (new_entry) { 3897 new_entry = false; 3898 ind_start_offsets[*offset_count] = ind_offset; 3899 *offset_count = *offset_count + 1; 3900 BUG_ON(*offset_count >= max_offset); 3901 } 3902 3903 if (register_list_format[ind_offset] == 0xFFFFFFFF) { 3904 new_entry = true; 3905 continue; 3906 } 3907 3908 ind_offset += 2; 3909 3910 /* look for the matching indice */ 3911 for (indices = 0; 3912 indices < *indices_count; 3913 indices++) { 3914 if (unique_indices[indices] == 3915 register_list_format[ind_offset]) 3916 break; 3917 } 3918 3919 if (indices >= *indices_count) { 3920 unique_indices[*indices_count] = 3921 register_list_format[ind_offset]; 3922 indices = *indices_count; 3923 *indices_count = *indices_count + 1; 3924 BUG_ON(*indices_count >= max_indices); 3925 } 3926 3927 register_list_format[ind_offset] = indices; 3928 } 3929 } 3930 3931 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev) 3932 { 3933 int i, temp, data; 3934 int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0}; 3935 int indices_count = 0; 3936 int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 3937 int offset_count = 0; 3938 3939 int list_size; 3940 unsigned int *register_list_format = 3941 kmemdup(adev->gfx.rlc.register_list_format, 3942 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 3943 if (!register_list_format) 3944 return -ENOMEM; 3945 3946 gfx_v8_0_parse_ind_reg_list(register_list_format, 3947 RLC_FormatDirectRegListLength, 3948 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 3949 unique_indices, 3950 &indices_count, 3951 ARRAY_SIZE(unique_indices), 3952 indirect_start_offsets, 3953 &offset_count, 3954 ARRAY_SIZE(indirect_start_offsets)); 3955 3956 /* save and restore list */ 3957 WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1); 3958 3959 WREG32(mmRLC_SRM_ARAM_ADDR, 0); 3960 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 3961 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]); 3962 3963 /* indirect list */ 3964 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start); 3965 for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++) 3966 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]); 3967 3968 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 3969 list_size = list_size >> 1; 3970 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size); 3971 WREG32(mmRLC_GPM_SCRATCH_DATA, list_size); 3972 3973 /* starting offsets starts */ 3974 WREG32(mmRLC_GPM_SCRATCH_ADDR, 3975 adev->gfx.rlc.starting_offsets_start); 3976 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 3977 WREG32(mmRLC_GPM_SCRATCH_DATA, 3978 indirect_start_offsets[i]); 3979 3980 /* unique indices */ 3981 temp = mmRLC_SRM_INDEX_CNTL_ADDR_0; 3982 data = mmRLC_SRM_INDEX_CNTL_DATA_0; 3983 for (i = 0; i < ARRAY_SIZE(unique_indices); i++) { 3984 if (unique_indices[i] != 0) { 3985 WREG32(temp + i, unique_indices[i] & 0x3FFFF); 3986 WREG32(data + i, unique_indices[i] >> 20); 3987 } 3988 } 3989 kfree(register_list_format); 3990 3991 return 0; 3992 } 3993 3994 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev) 3995 { 3996 WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1); 3997 } 3998 3999 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev) 4000 { 4001 uint32_t data; 4002 4003 WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60); 4004 4005 data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10); 4006 data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10); 4007 data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10); 4008 data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10); 4009 WREG32(mmRLC_PG_DELAY, data); 4010 4011 WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3); 4012 WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0); 4013 4014 } 4015 4016 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 4017 bool enable) 4018 { 4019 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0); 4020 } 4021 4022 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 4023 bool enable) 4024 { 4025 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0); 4026 } 4027 4028 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable) 4029 { 4030 WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1); 4031 } 4032 4033 static void gfx_v8_0_init_pg(struct amdgpu_device *adev) 4034 { 4035 if ((adev->asic_type == CHIP_CARRIZO) || 4036 (adev->asic_type == CHIP_STONEY)) { 4037 gfx_v8_0_init_csb(adev); 4038 gfx_v8_0_init_save_restore_list(adev); 4039 gfx_v8_0_enable_save_restore_machine(adev); 4040 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8); 4041 gfx_v8_0_init_power_gating(adev); 4042 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask); 4043 } else if ((adev->asic_type == CHIP_POLARIS11) || 4044 (adev->asic_type == CHIP_POLARIS12) || 4045 (adev->asic_type == CHIP_VEGAM)) { 4046 gfx_v8_0_init_csb(adev); 4047 gfx_v8_0_init_save_restore_list(adev); 4048 gfx_v8_0_enable_save_restore_machine(adev); 4049 gfx_v8_0_init_power_gating(adev); 4050 } 4051 4052 } 4053 4054 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev) 4055 { 4056 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0); 4057 4058 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4059 gfx_v8_0_wait_for_rlc_serdes(adev); 4060 } 4061 4062 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev) 4063 { 4064 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4065 udelay(50); 4066 4067 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 4068 udelay(50); 4069 } 4070 4071 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev) 4072 { 4073 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1); 4074 4075 /* carrizo do enable cp interrupt after cp inited */ 4076 if (!(adev->flags & AMD_IS_APU)) 4077 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4078 4079 udelay(50); 4080 } 4081 4082 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev) 4083 { 4084 if (amdgpu_sriov_vf(adev)) { 4085 gfx_v8_0_init_csb(adev); 4086 return 0; 4087 } 4088 4089 adev->gfx.rlc.funcs->stop(adev); 4090 adev->gfx.rlc.funcs->reset(adev); 4091 gfx_v8_0_init_pg(adev); 4092 adev->gfx.rlc.funcs->start(adev); 4093 4094 return 0; 4095 } 4096 4097 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 4098 { 4099 u32 tmp = RREG32(mmCP_ME_CNTL); 4100 4101 if (enable) { 4102 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0); 4103 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0); 4104 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0); 4105 } else { 4106 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1); 4107 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1); 4108 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1); 4109 } 4110 WREG32(mmCP_ME_CNTL, tmp); 4111 udelay(50); 4112 } 4113 4114 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev) 4115 { 4116 u32 count = 0; 4117 const struct cs_section_def *sect = NULL; 4118 const struct cs_extent_def *ext = NULL; 4119 4120 /* begin clear state */ 4121 count += 2; 4122 /* context control state */ 4123 count += 3; 4124 4125 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4126 for (ext = sect->section; ext->extent != NULL; ++ext) { 4127 if (sect->id == SECT_CONTEXT) 4128 count += 2 + ext->reg_count; 4129 else 4130 return 0; 4131 } 4132 } 4133 /* pa_sc_raster_config/pa_sc_raster_config1 */ 4134 count += 4; 4135 /* end clear state */ 4136 count += 2; 4137 /* clear state */ 4138 count += 2; 4139 4140 return count; 4141 } 4142 4143 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev) 4144 { 4145 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 4146 const struct cs_section_def *sect = NULL; 4147 const struct cs_extent_def *ext = NULL; 4148 int r, i; 4149 4150 /* init the CP */ 4151 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 4152 WREG32(mmCP_ENDIAN_SWAP, 0); 4153 WREG32(mmCP_DEVICE_ID, 1); 4154 4155 gfx_v8_0_cp_gfx_enable(adev, true); 4156 4157 r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4); 4158 if (r) { 4159 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 4160 return r; 4161 } 4162 4163 /* clear state buffer */ 4164 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4165 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4166 4167 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4168 amdgpu_ring_write(ring, 0x80000000); 4169 amdgpu_ring_write(ring, 0x80000000); 4170 4171 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4172 for (ext = sect->section; ext->extent != NULL; ++ext) { 4173 if (sect->id == SECT_CONTEXT) { 4174 amdgpu_ring_write(ring, 4175 PACKET3(PACKET3_SET_CONTEXT_REG, 4176 ext->reg_count)); 4177 amdgpu_ring_write(ring, 4178 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 4179 for (i = 0; i < ext->reg_count; i++) 4180 amdgpu_ring_write(ring, ext->extent[i]); 4181 } 4182 } 4183 } 4184 4185 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4186 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 4187 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config); 4188 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1); 4189 4190 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4191 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4192 4193 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4194 amdgpu_ring_write(ring, 0); 4195 4196 /* init the CE partitions */ 4197 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4198 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4199 amdgpu_ring_write(ring, 0x8000); 4200 amdgpu_ring_write(ring, 0x8000); 4201 4202 amdgpu_ring_commit(ring); 4203 4204 return 0; 4205 } 4206 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring) 4207 { 4208 u32 tmp; 4209 /* no gfx doorbells on iceland */ 4210 if (adev->asic_type == CHIP_TOPAZ) 4211 return; 4212 4213 tmp = RREG32(mmCP_RB_DOORBELL_CONTROL); 4214 4215 if (ring->use_doorbell) { 4216 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4217 DOORBELL_OFFSET, ring->doorbell_index); 4218 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4219 DOORBELL_HIT, 0); 4220 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4221 DOORBELL_EN, 1); 4222 } else { 4223 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 4224 } 4225 4226 WREG32(mmCP_RB_DOORBELL_CONTROL, tmp); 4227 4228 if (adev->flags & AMD_IS_APU) 4229 return; 4230 4231 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 4232 DOORBELL_RANGE_LOWER, 4233 adev->doorbell_index.gfx_ring0); 4234 WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 4235 4236 WREG32(mmCP_RB_DOORBELL_RANGE_UPPER, 4237 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 4238 } 4239 4240 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev) 4241 { 4242 struct amdgpu_ring *ring; 4243 u32 tmp; 4244 u32 rb_bufsz; 4245 u64 rb_addr, rptr_addr, wptr_gpu_addr; 4246 4247 /* Set the write pointer delay */ 4248 WREG32(mmCP_RB_WPTR_DELAY, 0); 4249 4250 /* set the RB to use vmid 0 */ 4251 WREG32(mmCP_RB_VMID, 0); 4252 4253 /* Set ring buffer size */ 4254 ring = &adev->gfx.gfx_ring[0]; 4255 rb_bufsz = order_base_2(ring->ring_size / 8); 4256 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 4257 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 4258 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3); 4259 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1); 4260 #ifdef __BIG_ENDIAN 4261 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 4262 #endif 4263 WREG32(mmCP_RB0_CNTL, tmp); 4264 4265 /* Initialize the ring buffer's read and write pointers */ 4266 WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK); 4267 ring->wptr = 0; 4268 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 4269 4270 /* set the wb address whether it's enabled or not */ 4271 rptr_addr = ring->rptr_gpu_addr; 4272 WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 4273 WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF); 4274 4275 wptr_gpu_addr = ring->wptr_gpu_addr; 4276 WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 4277 WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 4278 mdelay(1); 4279 WREG32(mmCP_RB0_CNTL, tmp); 4280 4281 rb_addr = ring->gpu_addr >> 8; 4282 WREG32(mmCP_RB0_BASE, rb_addr); 4283 WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4284 4285 gfx_v8_0_set_cpg_door_bell(adev, ring); 4286 /* start the ring */ 4287 amdgpu_ring_clear_ring(ring); 4288 gfx_v8_0_cp_gfx_start(adev); 4289 4290 return 0; 4291 } 4292 4293 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 4294 { 4295 if (enable) { 4296 WREG32(mmCP_MEC_CNTL, 0); 4297 } else { 4298 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 4299 adev->gfx.kiq[0].ring.sched.ready = false; 4300 } 4301 udelay(50); 4302 } 4303 4304 /* KIQ functions */ 4305 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring) 4306 { 4307 uint32_t tmp; 4308 struct amdgpu_device *adev = ring->adev; 4309 4310 /* tell RLC which is KIQ queue */ 4311 tmp = RREG32(mmRLC_CP_SCHEDULERS); 4312 tmp &= 0xffffff00; 4313 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4314 WREG32(mmRLC_CP_SCHEDULERS, tmp | 0x80); 4315 } 4316 4317 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev) 4318 { 4319 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 4320 uint64_t queue_mask = 0; 4321 int r, i; 4322 4323 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 4324 if (!test_bit(i, adev->gfx.mec_bitmap[0].queue_bitmap)) 4325 continue; 4326 4327 /* This situation may be hit in the future if a new HW 4328 * generation exposes more than 64 queues. If so, the 4329 * definition of queue_mask needs updating */ 4330 if (WARN_ON(i >= (sizeof(queue_mask)*8))) { 4331 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 4332 break; 4333 } 4334 4335 queue_mask |= (1ull << i); 4336 } 4337 4338 r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8); 4339 if (r) { 4340 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4341 return r; 4342 } 4343 /* set resources */ 4344 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 4345 amdgpu_ring_write(kiq_ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */ 4346 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 4347 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 4348 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 4349 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 4350 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 4351 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 4352 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4353 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4354 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 4355 uint64_t wptr_addr = ring->wptr_gpu_addr; 4356 4357 /* map queues */ 4358 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 4359 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 4360 amdgpu_ring_write(kiq_ring, 4361 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 4362 amdgpu_ring_write(kiq_ring, 4363 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) | 4364 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 4365 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 4366 PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */ 4367 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 4368 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 4369 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 4370 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 4371 } 4372 4373 amdgpu_ring_commit(kiq_ring); 4374 4375 return 0; 4376 } 4377 4378 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req) 4379 { 4380 int i, r = 0; 4381 4382 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) { 4383 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req); 4384 for (i = 0; i < adev->usec_timeout; i++) { 4385 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK)) 4386 break; 4387 udelay(1); 4388 } 4389 if (i == adev->usec_timeout) 4390 r = -ETIMEDOUT; 4391 } 4392 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0); 4393 WREG32(mmCP_HQD_PQ_RPTR, 0); 4394 WREG32(mmCP_HQD_PQ_WPTR, 0); 4395 4396 return r; 4397 } 4398 4399 static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd) 4400 { 4401 struct amdgpu_device *adev = ring->adev; 4402 4403 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 4404 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 4405 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 4406 mqd->cp_hqd_queue_priority = 4407 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 4408 } 4409 } 4410 } 4411 4412 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring) 4413 { 4414 struct amdgpu_device *adev = ring->adev; 4415 struct vi_mqd *mqd = ring->mqd_ptr; 4416 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4417 uint32_t tmp; 4418 4419 mqd->header = 0xC0310800; 4420 mqd->compute_pipelinestat_enable = 0x00000001; 4421 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4422 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4423 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4424 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4425 mqd->compute_misc_reserved = 0x00000003; 4426 mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr 4427 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4428 mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr 4429 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4430 eop_base_addr = ring->eop_gpu_addr >> 8; 4431 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4432 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4433 4434 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4435 tmp = RREG32(mmCP_HQD_EOP_CONTROL); 4436 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4437 (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1)); 4438 4439 mqd->cp_hqd_eop_control = tmp; 4440 4441 /* enable doorbell? */ 4442 tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL), 4443 CP_HQD_PQ_DOORBELL_CONTROL, 4444 DOORBELL_EN, 4445 ring->use_doorbell ? 1 : 0); 4446 4447 mqd->cp_hqd_pq_doorbell_control = tmp; 4448 4449 /* set the pointer to the MQD */ 4450 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 4451 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 4452 4453 /* set MQD vmid to 0 */ 4454 tmp = RREG32(mmCP_MQD_CONTROL); 4455 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4456 mqd->cp_mqd_control = tmp; 4457 4458 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4459 hqd_gpu_addr = ring->gpu_addr >> 8; 4460 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4461 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4462 4463 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4464 tmp = RREG32(mmCP_HQD_PQ_CONTROL); 4465 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4466 (order_base_2(ring->ring_size / 4) - 1)); 4467 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4468 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4469 #ifdef __BIG_ENDIAN 4470 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 4471 #endif 4472 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 4473 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 4474 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4475 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4476 mqd->cp_hqd_pq_control = tmp; 4477 4478 /* set the wb address whether it's enabled or not */ 4479 wb_gpu_addr = ring->rptr_gpu_addr; 4480 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4481 mqd->cp_hqd_pq_rptr_report_addr_hi = 4482 upper_32_bits(wb_gpu_addr) & 0xffff; 4483 4484 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4485 wb_gpu_addr = ring->wptr_gpu_addr; 4486 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4487 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4488 4489 tmp = 0; 4490 /* enable the doorbell if requested */ 4491 if (ring->use_doorbell) { 4492 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL); 4493 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4494 DOORBELL_OFFSET, ring->doorbell_index); 4495 4496 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4497 DOORBELL_EN, 1); 4498 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4499 DOORBELL_SOURCE, 0); 4500 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4501 DOORBELL_HIT, 0); 4502 } 4503 4504 mqd->cp_hqd_pq_doorbell_control = tmp; 4505 4506 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4507 ring->wptr = 0; 4508 mqd->cp_hqd_pq_wptr = ring->wptr; 4509 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); 4510 4511 /* set the vmid for the queue */ 4512 mqd->cp_hqd_vmid = 0; 4513 4514 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE); 4515 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 4516 mqd->cp_hqd_persistent_state = tmp; 4517 4518 /* set MTYPE */ 4519 tmp = RREG32(mmCP_HQD_IB_CONTROL); 4520 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4521 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3); 4522 mqd->cp_hqd_ib_control = tmp; 4523 4524 tmp = RREG32(mmCP_HQD_IQ_TIMER); 4525 tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3); 4526 mqd->cp_hqd_iq_timer = tmp; 4527 4528 tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL); 4529 tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3); 4530 mqd->cp_hqd_ctx_save_control = tmp; 4531 4532 /* defaults */ 4533 mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR); 4534 mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR); 4535 mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO); 4536 mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI); 4537 mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET); 4538 mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE); 4539 mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET); 4540 mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE); 4541 mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS); 4542 mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR); 4543 mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM); 4544 mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES); 4545 4546 /* set static priority for a queue/ring */ 4547 gfx_v8_0_mqd_set_priority(ring, mqd); 4548 tmp = RREG32(mmCP_HQD_QUANTUM); 4549 if (ring != &adev->gfx.kiq[0].ring) { 4550 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); 4551 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 4552 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10); 4553 } 4554 mqd->cp_hqd_quantum = tmp; 4555 4556 /* map_queues packet doesn't need activate the queue, 4557 * so only kiq need set this field. 4558 */ 4559 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 4560 mqd->cp_hqd_active = 1; 4561 4562 return 0; 4563 } 4564 4565 static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev, 4566 struct vi_mqd *mqd) 4567 { 4568 uint32_t mqd_reg; 4569 uint32_t *mqd_data; 4570 4571 /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */ 4572 mqd_data = &mqd->cp_mqd_base_addr_lo; 4573 4574 /* disable wptr polling */ 4575 WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0); 4576 4577 /* program all HQD registers */ 4578 for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++) 4579 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4580 4581 /* Tonga errata: EOP RPTR/WPTR should be left unmodified. 4582 * This is safe since EOP RPTR==WPTR for any inactive HQD 4583 * on ASICs that do not support context-save. 4584 * EOP writes/reads can start anywhere in the ring. 4585 */ 4586 if (adev->asic_type != CHIP_TONGA) { 4587 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr); 4588 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr); 4589 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem); 4590 } 4591 4592 for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++) 4593 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4594 4595 /* activate the HQD */ 4596 for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++) 4597 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4598 4599 return 0; 4600 } 4601 4602 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) 4603 { 4604 struct amdgpu_device *adev = ring->adev; 4605 struct vi_mqd *mqd = ring->mqd_ptr; 4606 4607 gfx_v8_0_kiq_setting(ring); 4608 4609 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4610 /* reset MQD to a clean status */ 4611 if (adev->gfx.kiq[0].mqd_backup) 4612 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation)); 4613 4614 /* reset ring buffer */ 4615 ring->wptr = 0; 4616 amdgpu_ring_clear_ring(ring); 4617 mutex_lock(&adev->srbm_mutex); 4618 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4619 gfx_v8_0_mqd_commit(adev, mqd); 4620 vi_srbm_select(adev, 0, 0, 0, 0); 4621 mutex_unlock(&adev->srbm_mutex); 4622 } else { 4623 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4624 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4625 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4626 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4627 amdgpu_ring_clear_ring(ring); 4628 mutex_lock(&adev->srbm_mutex); 4629 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4630 gfx_v8_0_mqd_init(ring); 4631 gfx_v8_0_mqd_commit(adev, mqd); 4632 vi_srbm_select(adev, 0, 0, 0, 0); 4633 mutex_unlock(&adev->srbm_mutex); 4634 4635 if (adev->gfx.kiq[0].mqd_backup) 4636 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation)); 4637 } 4638 4639 return 0; 4640 } 4641 4642 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) 4643 { 4644 struct amdgpu_device *adev = ring->adev; 4645 struct vi_mqd *mqd = ring->mqd_ptr; 4646 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4647 4648 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4649 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4650 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4651 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4652 mutex_lock(&adev->srbm_mutex); 4653 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4654 gfx_v8_0_mqd_init(ring); 4655 vi_srbm_select(adev, 0, 0, 0, 0); 4656 mutex_unlock(&adev->srbm_mutex); 4657 4658 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4659 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); 4660 } else { 4661 /* restore MQD to a clean status */ 4662 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4663 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); 4664 /* reset ring buffer */ 4665 ring->wptr = 0; 4666 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4667 amdgpu_ring_clear_ring(ring); 4668 } 4669 return 0; 4670 } 4671 4672 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev) 4673 { 4674 if (adev->asic_type > CHIP_TONGA) { 4675 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2); 4676 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2); 4677 } 4678 /* enable doorbells */ 4679 WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4680 } 4681 4682 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev) 4683 { 4684 gfx_v8_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 4685 return 0; 4686 } 4687 4688 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev) 4689 { 4690 int i, r; 4691 4692 gfx_v8_0_cp_compute_enable(adev, true); 4693 4694 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4695 r = gfx_v8_0_kcq_init_queue(&adev->gfx.compute_ring[i]); 4696 if (r) 4697 return r; 4698 } 4699 4700 gfx_v8_0_set_mec_doorbell_range(adev); 4701 4702 return gfx_v8_0_kiq_kcq_enable(adev); 4703 } 4704 4705 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev) 4706 { 4707 int r, i; 4708 struct amdgpu_ring *ring; 4709 4710 /* collect all the ring_tests here, gfx, kiq, compute */ 4711 ring = &adev->gfx.gfx_ring[0]; 4712 r = amdgpu_ring_test_helper(ring); 4713 if (r) 4714 return r; 4715 4716 ring = &adev->gfx.kiq[0].ring; 4717 r = amdgpu_ring_test_helper(ring); 4718 if (r) 4719 return r; 4720 4721 r = 0; 4722 4723 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4724 ring = &adev->gfx.compute_ring[i]; 4725 r |= amdgpu_ring_test_helper(ring); 4726 } 4727 4728 return r; 4729 } 4730 4731 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev) 4732 { 4733 int r; 4734 4735 if (!(adev->flags & AMD_IS_APU)) 4736 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4737 4738 r = gfx_v8_0_kiq_resume(adev); 4739 if (r) 4740 return r; 4741 4742 r = gfx_v8_0_cp_gfx_resume(adev); 4743 if (r) 4744 return r; 4745 4746 r = gfx_v8_0_kcq_resume(adev); 4747 if (r) 4748 return r; 4749 4750 r = gfx_v8_0_cp_test_all_rings(adev); 4751 if (r) 4752 return r; 4753 4754 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4755 4756 return 0; 4757 } 4758 4759 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable) 4760 { 4761 gfx_v8_0_cp_gfx_enable(adev, enable); 4762 gfx_v8_0_cp_compute_enable(adev, enable); 4763 } 4764 4765 static int gfx_v8_0_hw_init(struct amdgpu_ip_block *ip_block) 4766 { 4767 int r; 4768 struct amdgpu_device *adev = ip_block->adev; 4769 4770 gfx_v8_0_init_golden_registers(adev); 4771 gfx_v8_0_constants_init(adev); 4772 4773 r = adev->gfx.rlc.funcs->resume(adev); 4774 if (r) 4775 return r; 4776 4777 r = gfx_v8_0_cp_resume(adev); 4778 4779 return r; 4780 } 4781 4782 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev) 4783 { 4784 int r, i; 4785 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 4786 4787 r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings); 4788 if (r) 4789 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4790 4791 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4792 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4793 4794 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 4795 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 4796 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */ 4797 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 4798 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) | 4799 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 4800 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 4801 amdgpu_ring_write(kiq_ring, 0); 4802 amdgpu_ring_write(kiq_ring, 0); 4803 amdgpu_ring_write(kiq_ring, 0); 4804 } 4805 /* Submit unmap queue packet */ 4806 amdgpu_ring_commit(kiq_ring); 4807 /* 4808 * Ring test will do a basic scratch register change check. Just run 4809 * this to ensure that unmap queues that is submitted before got 4810 * processed successfully before returning. 4811 */ 4812 r = amdgpu_ring_test_helper(kiq_ring); 4813 if (r) 4814 DRM_ERROR("KCQ disable failed\n"); 4815 4816 return r; 4817 } 4818 4819 static bool gfx_v8_0_is_idle(struct amdgpu_ip_block *ip_block) 4820 { 4821 struct amdgpu_device *adev = ip_block->adev; 4822 4823 if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE) 4824 || RREG32(mmGRBM_STATUS2) != 0x8) 4825 return false; 4826 else 4827 return true; 4828 } 4829 4830 static bool gfx_v8_0_rlc_is_idle(void *handle) 4831 { 4832 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4833 4834 if (RREG32(mmGRBM_STATUS2) != 0x8) 4835 return false; 4836 else 4837 return true; 4838 } 4839 4840 static int gfx_v8_0_wait_for_rlc_idle(void *handle) 4841 { 4842 unsigned int i; 4843 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4844 4845 for (i = 0; i < adev->usec_timeout; i++) { 4846 if (gfx_v8_0_rlc_is_idle(handle)) 4847 return 0; 4848 4849 udelay(1); 4850 } 4851 return -ETIMEDOUT; 4852 } 4853 4854 static int gfx_v8_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 4855 { 4856 unsigned int i; 4857 struct amdgpu_device *adev = ip_block->adev; 4858 4859 for (i = 0; i < adev->usec_timeout; i++) { 4860 if (gfx_v8_0_is_idle(ip_block)) 4861 return 0; 4862 4863 udelay(1); 4864 } 4865 return -ETIMEDOUT; 4866 } 4867 4868 static int gfx_v8_0_hw_fini(struct amdgpu_ip_block *ip_block) 4869 { 4870 struct amdgpu_device *adev = ip_block->adev; 4871 4872 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4873 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4874 4875 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 4876 4877 amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0); 4878 4879 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4880 gfx_v8_0_kcq_disable(adev); 4881 4882 if (amdgpu_sriov_vf(adev)) { 4883 pr_debug("For SRIOV client, shouldn't do anything.\n"); 4884 return 0; 4885 } 4886 4887 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4888 if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_idle(ip_block)) 4889 pr_err("cp is busy, skip halt cp\n"); 4890 if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_rlc_idle(adev)) 4891 pr_err("rlc is busy\n"); 4892 gfx_v8_0_cp_enable(adev, false); 4893 adev->gfx.rlc.funcs->stop(adev); 4894 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4895 4896 return 0; 4897 } 4898 4899 static int gfx_v8_0_suspend(struct amdgpu_ip_block *ip_block) 4900 { 4901 return gfx_v8_0_hw_fini(ip_block); 4902 } 4903 4904 static int gfx_v8_0_resume(struct amdgpu_ip_block *ip_block) 4905 { 4906 return gfx_v8_0_hw_init(ip_block); 4907 } 4908 4909 static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) 4910 { 4911 struct amdgpu_device *adev = ip_block->adev; 4912 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 4913 u32 tmp; 4914 int i; 4915 int r; 4916 4917 grbm_soft_reset = 4918 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1) | 4919 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_GFX, 1) | 4920 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CP, 1) | 4921 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPF, 1) | 4922 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPC, 1) | 4923 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPG, 1); 4924 4925 srbm_soft_reset = 4926 REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1) | 4927 REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_SEM, 1); 4928 4929 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4930 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4931 4932 mutex_lock(&adev->srbm_mutex); 4933 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4934 gfx_v8_0_deactivate_hqd(adev, 2); 4935 vi_srbm_select(adev, 0, 0, 0, 0); 4936 mutex_unlock(&adev->srbm_mutex); 4937 4938 udelay(50); 4939 } 4940 4941 ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE); 4942 ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_UNGATE); 4943 ip_block->version->funcs->suspend(ip_block); 4944 4945 if (grbm_soft_reset || srbm_soft_reset) { 4946 tmp = RREG32(mmGMCON_DEBUG); 4947 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1); 4948 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1); 4949 WREG32(mmGMCON_DEBUG, tmp); 4950 4951 udelay(100); 4952 } 4953 4954 if (grbm_soft_reset) { 4955 tmp = RREG32(mmGRBM_SOFT_RESET); 4956 tmp |= grbm_soft_reset; 4957 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4958 WREG32(mmGRBM_SOFT_RESET, tmp); 4959 tmp = RREG32(mmGRBM_SOFT_RESET); 4960 4961 udelay(100); 4962 4963 tmp &= ~grbm_soft_reset; 4964 WREG32(mmGRBM_SOFT_RESET, tmp); 4965 tmp = RREG32(mmGRBM_SOFT_RESET); 4966 4967 udelay(100); 4968 } 4969 4970 if (srbm_soft_reset) { 4971 tmp = RREG32(mmSRBM_SOFT_RESET); 4972 tmp |= srbm_soft_reset; 4973 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 4974 WREG32(mmSRBM_SOFT_RESET, tmp); 4975 tmp = RREG32(mmSRBM_SOFT_RESET); 4976 4977 udelay(100); 4978 4979 tmp &= ~srbm_soft_reset; 4980 WREG32(mmSRBM_SOFT_RESET, tmp); 4981 tmp = RREG32(mmSRBM_SOFT_RESET); 4982 4983 udelay(100); 4984 } 4985 4986 if (grbm_soft_reset || srbm_soft_reset) { 4987 tmp = RREG32(mmGMCON_DEBUG); 4988 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0); 4989 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0); 4990 WREG32(mmGMCON_DEBUG, tmp); 4991 } 4992 4993 /* Wait a little for things to settle down */ 4994 udelay(100); 4995 4996 r = ip_block->version->funcs->resume(ip_block); 4997 r |= ip_block->version->funcs->late_init(ip_block); 4998 if (r) 4999 return r; 5000 5001 ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_GATE); 5002 ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_GATE); 5003 5004 return 0; 5005 } 5006 5007 /** 5008 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot 5009 * 5010 * @adev: amdgpu_device pointer 5011 * 5012 * Fetches a GPU clock counter snapshot. 5013 * Returns the 64 bit clock counter snapshot. 5014 */ 5015 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5016 { 5017 uint64_t clock; 5018 5019 mutex_lock(&adev->gfx.gpu_clock_mutex); 5020 WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 5021 clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) | 5022 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 5023 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5024 return clock; 5025 } 5026 5027 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5028 uint32_t vmid, 5029 uint32_t gds_base, uint32_t gds_size, 5030 uint32_t gws_base, uint32_t gws_size, 5031 uint32_t oa_base, uint32_t oa_size) 5032 { 5033 /* GDS Base */ 5034 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5035 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5036 WRITE_DATA_DST_SEL(0))); 5037 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base); 5038 amdgpu_ring_write(ring, 0); 5039 amdgpu_ring_write(ring, gds_base); 5040 5041 /* GDS Size */ 5042 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5043 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5044 WRITE_DATA_DST_SEL(0))); 5045 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size); 5046 amdgpu_ring_write(ring, 0); 5047 amdgpu_ring_write(ring, gds_size); 5048 5049 /* GWS */ 5050 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5051 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5052 WRITE_DATA_DST_SEL(0))); 5053 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws); 5054 amdgpu_ring_write(ring, 0); 5055 amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5056 5057 /* OA */ 5058 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5059 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5060 WRITE_DATA_DST_SEL(0))); 5061 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa); 5062 amdgpu_ring_write(ring, 0); 5063 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); 5064 } 5065 5066 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 5067 { 5068 WREG32(mmSQ_IND_INDEX, 5069 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5070 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5071 (address << SQ_IND_INDEX__INDEX__SHIFT) | 5072 (SQ_IND_INDEX__FORCE_READ_MASK)); 5073 return RREG32(mmSQ_IND_DATA); 5074 } 5075 5076 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 5077 uint32_t wave, uint32_t thread, 5078 uint32_t regno, uint32_t num, uint32_t *out) 5079 { 5080 WREG32(mmSQ_IND_INDEX, 5081 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5082 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5083 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 5084 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 5085 (SQ_IND_INDEX__FORCE_READ_MASK) | 5086 (SQ_IND_INDEX__AUTO_INCR_MASK)); 5087 while (num--) 5088 *(out++) = RREG32(mmSQ_IND_DATA); 5089 } 5090 5091 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 5092 { 5093 /* type 0 wave data */ 5094 dst[(*no_fields)++] = 0; 5095 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 5096 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 5097 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 5098 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 5099 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 5100 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 5101 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 5102 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 5103 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 5104 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 5105 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 5106 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 5107 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO); 5108 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI); 5109 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO); 5110 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI); 5111 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 5112 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 5113 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 5114 } 5115 5116 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 5117 uint32_t wave, uint32_t start, 5118 uint32_t size, uint32_t *dst) 5119 { 5120 wave_read_regs( 5121 adev, simd, wave, 0, 5122 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 5123 } 5124 5125 /** 5126 * gfx_v8_0_get_hdp_flush_mask - get the reference and mask for HDP flush 5127 * 5128 * @ring: amdgpu_ring structure holding ring information 5129 * @ref_and_mask: pointer to store the reference and mask 5130 * @reg_mem_engine: pointer to store the register memory engine 5131 * 5132 * Calculates the reference and mask for HDP flush based on the ring type and me. 5133 */ 5134 static void gfx_v8_0_get_hdp_flush_mask(struct amdgpu_ring *ring, 5135 uint32_t *ref_and_mask, uint32_t *reg_mem_engine) 5136 { 5137 if (!ring || !ref_and_mask || !reg_mem_engine) { 5138 DRM_INFO("%s:invalid params\n", __func__); 5139 return; 5140 } 5141 5142 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) || 5143 (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) { 5144 switch (ring->me) { 5145 case 1: 5146 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe; 5147 break; 5148 case 2: 5149 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe; 5150 break; 5151 default: 5152 return; 5153 } 5154 *reg_mem_engine = 0; 5155 } else { 5156 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK; 5157 *reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */ 5158 } 5159 } 5160 5161 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = { 5162 .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter, 5163 .select_se_sh = &gfx_v8_0_select_se_sh, 5164 .read_wave_data = &gfx_v8_0_read_wave_data, 5165 .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs, 5166 .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q, 5167 .get_hdp_flush_mask = &gfx_v8_0_get_hdp_flush_mask, 5168 }; 5169 5170 static int gfx_v8_0_early_init(struct amdgpu_ip_block *ip_block) 5171 { 5172 struct amdgpu_device *adev = ip_block->adev; 5173 5174 adev->gfx.xcc_mask = 1; 5175 adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; 5176 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5177 AMDGPU_MAX_COMPUTE_RINGS); 5178 adev->gfx.funcs = &gfx_v8_0_gfx_funcs; 5179 gfx_v8_0_set_ring_funcs(adev); 5180 gfx_v8_0_set_irq_funcs(adev); 5181 gfx_v8_0_set_gds_init(adev); 5182 gfx_v8_0_set_rlc_funcs(adev); 5183 5184 return 0; 5185 } 5186 5187 static int gfx_v8_0_late_init(struct amdgpu_ip_block *ip_block) 5188 { 5189 struct amdgpu_device *adev = ip_block->adev; 5190 int r; 5191 5192 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5193 if (r) 5194 return r; 5195 5196 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5197 if (r) 5198 return r; 5199 5200 /* requires IBs so do in late init after IB pool is initialized */ 5201 r = gfx_v8_0_do_edc_gpr_workarounds(adev); 5202 if (r) 5203 return r; 5204 5205 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 5206 if (r) { 5207 DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r); 5208 return r; 5209 } 5210 5211 r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0); 5212 if (r) { 5213 DRM_ERROR( 5214 "amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n", 5215 r); 5216 return r; 5217 } 5218 5219 return 0; 5220 } 5221 5222 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 5223 bool enable) 5224 { 5225 if ((adev->asic_type == CHIP_POLARIS11) || 5226 (adev->asic_type == CHIP_POLARIS12) || 5227 (adev->asic_type == CHIP_VEGAM)) 5228 /* Send msg to SMU via Powerplay */ 5229 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable, 0); 5230 5231 WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0); 5232 } 5233 5234 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 5235 bool enable) 5236 { 5237 WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0); 5238 } 5239 5240 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev, 5241 bool enable) 5242 { 5243 WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0); 5244 } 5245 5246 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 5247 bool enable) 5248 { 5249 WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0); 5250 } 5251 5252 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev, 5253 bool enable) 5254 { 5255 WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0); 5256 5257 /* Read any GFX register to wake up GFX. */ 5258 if (!enable) 5259 RREG32(mmDB_RENDER_CONTROL); 5260 } 5261 5262 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev, 5263 bool enable) 5264 { 5265 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 5266 cz_enable_gfx_cg_power_gating(adev, true); 5267 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 5268 cz_enable_gfx_pipeline_power_gating(adev, true); 5269 } else { 5270 cz_enable_gfx_cg_power_gating(adev, false); 5271 cz_enable_gfx_pipeline_power_gating(adev, false); 5272 } 5273 } 5274 5275 static int gfx_v8_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5276 enum amd_powergating_state state) 5277 { 5278 struct amdgpu_device *adev = ip_block->adev; 5279 bool enable = (state == AMD_PG_STATE_GATE); 5280 5281 if (amdgpu_sriov_vf(adev)) 5282 return 0; 5283 5284 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5285 AMD_PG_SUPPORT_RLC_SMU_HS | 5286 AMD_PG_SUPPORT_CP | 5287 AMD_PG_SUPPORT_GFX_DMG)) 5288 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5289 switch (adev->asic_type) { 5290 case CHIP_CARRIZO: 5291 case CHIP_STONEY: 5292 5293 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5294 cz_enable_sck_slow_down_on_power_up(adev, true); 5295 cz_enable_sck_slow_down_on_power_down(adev, true); 5296 } else { 5297 cz_enable_sck_slow_down_on_power_up(adev, false); 5298 cz_enable_sck_slow_down_on_power_down(adev, false); 5299 } 5300 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5301 cz_enable_cp_power_gating(adev, true); 5302 else 5303 cz_enable_cp_power_gating(adev, false); 5304 5305 cz_update_gfx_cg_power_gating(adev, enable); 5306 5307 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5308 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5309 else 5310 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5311 5312 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5313 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5314 else 5315 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5316 break; 5317 case CHIP_POLARIS11: 5318 case CHIP_POLARIS12: 5319 case CHIP_VEGAM: 5320 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5321 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5322 else 5323 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5324 5325 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5326 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5327 else 5328 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5329 5330 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable) 5331 polaris11_enable_gfx_quick_mg_power_gating(adev, true); 5332 else 5333 polaris11_enable_gfx_quick_mg_power_gating(adev, false); 5334 break; 5335 default: 5336 break; 5337 } 5338 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5339 AMD_PG_SUPPORT_RLC_SMU_HS | 5340 AMD_PG_SUPPORT_CP | 5341 AMD_PG_SUPPORT_GFX_DMG)) 5342 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5343 return 0; 5344 } 5345 5346 static void gfx_v8_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5347 { 5348 struct amdgpu_device *adev = ip_block->adev; 5349 int data; 5350 5351 if (amdgpu_sriov_vf(adev)) 5352 *flags = 0; 5353 5354 /* AMD_CG_SUPPORT_GFX_MGCG */ 5355 data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5356 if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK)) 5357 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5358 5359 /* AMD_CG_SUPPORT_GFX_CGLG */ 5360 data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5361 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5362 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5363 5364 /* AMD_CG_SUPPORT_GFX_CGLS */ 5365 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5366 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5367 5368 /* AMD_CG_SUPPORT_GFX_CGTS */ 5369 data = RREG32(mmCGTS_SM_CTRL_REG); 5370 if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK)) 5371 *flags |= AMD_CG_SUPPORT_GFX_CGTS; 5372 5373 /* AMD_CG_SUPPORT_GFX_CGTS_LS */ 5374 if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK)) 5375 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS; 5376 5377 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5378 data = RREG32(mmRLC_MEM_SLP_CNTL); 5379 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5380 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5381 5382 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5383 data = RREG32(mmCP_MEM_SLP_CNTL); 5384 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5385 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5386 } 5387 5388 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev, 5389 uint32_t reg_addr, uint32_t cmd) 5390 { 5391 uint32_t data; 5392 5393 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 5394 5395 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 5396 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 5397 5398 data = RREG32(mmRLC_SERDES_WR_CTRL); 5399 if (adev->asic_type == CHIP_STONEY) 5400 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5401 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5402 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5403 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5404 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5405 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5406 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5407 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5408 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5409 else 5410 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5411 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5412 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5413 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5414 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5415 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5416 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5417 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5418 RLC_SERDES_WR_CTRL__BPM_DATA_MASK | 5419 RLC_SERDES_WR_CTRL__REG_ADDR_MASK | 5420 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5421 data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK | 5422 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) | 5423 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) | 5424 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT)); 5425 5426 WREG32(mmRLC_SERDES_WR_CTRL, data); 5427 } 5428 5429 #define MSG_ENTER_RLC_SAFE_MODE 1 5430 #define MSG_EXIT_RLC_SAFE_MODE 0 5431 #define RLC_GPR_REG2__REQ_MASK 0x00000001 5432 #define RLC_GPR_REG2__REQ__SHIFT 0 5433 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001 5434 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e 5435 5436 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev) 5437 { 5438 uint32_t rlc_setting; 5439 5440 rlc_setting = RREG32(mmRLC_CNTL); 5441 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 5442 return false; 5443 5444 return true; 5445 } 5446 5447 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5448 { 5449 uint32_t data; 5450 unsigned i; 5451 data = RREG32(mmRLC_CNTL); 5452 data |= RLC_SAFE_MODE__CMD_MASK; 5453 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5454 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5455 WREG32(mmRLC_SAFE_MODE, data); 5456 5457 /* wait for RLC_SAFE_MODE */ 5458 for (i = 0; i < adev->usec_timeout; i++) { 5459 if ((RREG32(mmRLC_GPM_STAT) & 5460 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5461 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) == 5462 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5463 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) 5464 break; 5465 udelay(1); 5466 } 5467 for (i = 0; i < adev->usec_timeout; i++) { 5468 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5469 break; 5470 udelay(1); 5471 } 5472 } 5473 5474 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5475 { 5476 uint32_t data; 5477 unsigned i; 5478 5479 data = RREG32(mmRLC_CNTL); 5480 data |= RLC_SAFE_MODE__CMD_MASK; 5481 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5482 WREG32(mmRLC_SAFE_MODE, data); 5483 5484 for (i = 0; i < adev->usec_timeout; i++) { 5485 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5486 break; 5487 udelay(1); 5488 } 5489 } 5490 5491 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5492 struct amdgpu_ring *ring, unsigned vmid) 5493 { 5494 u32 data; 5495 5496 amdgpu_gfx_off_ctrl(adev, false); 5497 5498 if (amdgpu_sriov_is_pp_one_vf(adev)) 5499 data = RREG32_NO_KIQ(mmRLC_SPM_VMID); 5500 else 5501 data = RREG32(mmRLC_SPM_VMID); 5502 5503 data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK; 5504 data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT; 5505 5506 if (amdgpu_sriov_is_pp_one_vf(adev)) 5507 WREG32_NO_KIQ(mmRLC_SPM_VMID, data); 5508 else 5509 WREG32(mmRLC_SPM_VMID, data); 5510 5511 amdgpu_gfx_off_ctrl(adev, true); 5512 } 5513 5514 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = { 5515 .is_rlc_enabled = gfx_v8_0_is_rlc_enabled, 5516 .set_safe_mode = gfx_v8_0_set_safe_mode, 5517 .unset_safe_mode = gfx_v8_0_unset_safe_mode, 5518 .init = gfx_v8_0_rlc_init, 5519 .get_csb_size = gfx_v8_0_get_csb_size, 5520 .get_csb_buffer = gfx_v8_0_get_csb_buffer, 5521 .get_cp_table_num = gfx_v8_0_cp_jump_table_num, 5522 .resume = gfx_v8_0_rlc_resume, 5523 .stop = gfx_v8_0_rlc_stop, 5524 .reset = gfx_v8_0_rlc_reset, 5525 .start = gfx_v8_0_rlc_start, 5526 .update_spm_vmid = gfx_v8_0_update_spm_vmid 5527 }; 5528 5529 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5530 bool enable) 5531 { 5532 uint32_t temp, data; 5533 5534 /* It is disabled by HW by default */ 5535 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 5536 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5537 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) 5538 /* 1 - RLC memory Light sleep */ 5539 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1); 5540 5541 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) 5542 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1); 5543 } 5544 5545 /* 3 - RLC_CGTT_MGCG_OVERRIDE */ 5546 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5547 if (adev->flags & AMD_IS_APU) 5548 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5549 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5550 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK); 5551 else 5552 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5553 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5554 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5555 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5556 5557 if (temp != data) 5558 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5559 5560 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5561 gfx_v8_0_wait_for_rlc_serdes(adev); 5562 5563 /* 5 - clear mgcg override */ 5564 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5565 5566 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) { 5567 /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */ 5568 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5569 data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK); 5570 data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT); 5571 data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK; 5572 data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK; 5573 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) && 5574 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS)) 5575 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK; 5576 data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK; 5577 data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT); 5578 if (temp != data) 5579 WREG32(mmCGTS_SM_CTRL_REG, data); 5580 } 5581 udelay(50); 5582 5583 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5584 gfx_v8_0_wait_for_rlc_serdes(adev); 5585 } else { 5586 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */ 5587 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5588 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5589 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5590 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5591 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5592 if (temp != data) 5593 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5594 5595 /* 2 - disable MGLS in RLC */ 5596 data = RREG32(mmRLC_MEM_SLP_CNTL); 5597 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 5598 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 5599 WREG32(mmRLC_MEM_SLP_CNTL, data); 5600 } 5601 5602 /* 3 - disable MGLS in CP */ 5603 data = RREG32(mmCP_MEM_SLP_CNTL); 5604 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 5605 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 5606 WREG32(mmCP_MEM_SLP_CNTL, data); 5607 } 5608 5609 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */ 5610 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5611 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK | 5612 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK); 5613 if (temp != data) 5614 WREG32(mmCGTS_SM_CTRL_REG, data); 5615 5616 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5617 gfx_v8_0_wait_for_rlc_serdes(adev); 5618 5619 /* 6 - set mgcg override */ 5620 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5621 5622 udelay(50); 5623 5624 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5625 gfx_v8_0_wait_for_rlc_serdes(adev); 5626 } 5627 } 5628 5629 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5630 bool enable) 5631 { 5632 uint32_t temp, temp1, data, data1; 5633 5634 temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5635 5636 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5637 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5638 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK; 5639 if (temp1 != data1) 5640 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5641 5642 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5643 gfx_v8_0_wait_for_rlc_serdes(adev); 5644 5645 /* 2 - clear cgcg override */ 5646 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5647 5648 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5649 gfx_v8_0_wait_for_rlc_serdes(adev); 5650 5651 /* 3 - write cmd to set CGLS */ 5652 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD); 5653 5654 /* 4 - enable cgcg */ 5655 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5656 5657 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5658 /* enable cgls*/ 5659 data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5660 5661 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5662 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK; 5663 5664 if (temp1 != data1) 5665 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5666 } else { 5667 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5668 } 5669 5670 if (temp != data) 5671 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5672 5673 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/ 5674 * Cmp_busy/GFX_Idle interrupts 5675 */ 5676 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5677 } else { 5678 /* disable cntx_empty_int_enable & GFX Idle interrupt */ 5679 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 5680 5681 /* TEST CGCG */ 5682 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5683 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK | 5684 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK); 5685 if (temp1 != data1) 5686 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5687 5688 /* read gfx register to wake up cgcg */ 5689 RREG32(mmCB_CGTT_SCLK_CTRL); 5690 RREG32(mmCB_CGTT_SCLK_CTRL); 5691 RREG32(mmCB_CGTT_SCLK_CTRL); 5692 RREG32(mmCB_CGTT_SCLK_CTRL); 5693 5694 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5695 gfx_v8_0_wait_for_rlc_serdes(adev); 5696 5697 /* write cmd to Set CGCG Override */ 5698 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5699 5700 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5701 gfx_v8_0_wait_for_rlc_serdes(adev); 5702 5703 /* write cmd to Clear CGLS */ 5704 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD); 5705 5706 /* disable cgcg, cgls should be disabled too. */ 5707 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | 5708 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5709 if (temp != data) 5710 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5711 /* enable interrupts again for PG */ 5712 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5713 } 5714 5715 gfx_v8_0_wait_for_rlc_serdes(adev); 5716 } 5717 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5718 bool enable) 5719 { 5720 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5721 5722 if (enable) { 5723 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS) 5724 * === MGCG + MGLS + TS(CG/LS) === 5725 */ 5726 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5727 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5728 } else { 5729 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS) 5730 * === CGCG + CGLS === 5731 */ 5732 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5733 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5734 } 5735 5736 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5737 return 0; 5738 } 5739 5740 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev, 5741 enum amd_clockgating_state state) 5742 { 5743 uint32_t msg_id, pp_state = 0; 5744 uint32_t pp_support_state = 0; 5745 5746 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5747 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5748 pp_support_state = PP_STATE_SUPPORT_LS; 5749 pp_state = PP_STATE_LS; 5750 } 5751 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5752 pp_support_state |= PP_STATE_SUPPORT_CG; 5753 pp_state |= PP_STATE_CG; 5754 } 5755 if (state == AMD_CG_STATE_UNGATE) 5756 pp_state = 0; 5757 5758 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5759 PP_BLOCK_GFX_CG, 5760 pp_support_state, 5761 pp_state); 5762 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5763 } 5764 5765 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5766 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5767 pp_support_state = PP_STATE_SUPPORT_LS; 5768 pp_state = PP_STATE_LS; 5769 } 5770 5771 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5772 pp_support_state |= PP_STATE_SUPPORT_CG; 5773 pp_state |= PP_STATE_CG; 5774 } 5775 5776 if (state == AMD_CG_STATE_UNGATE) 5777 pp_state = 0; 5778 5779 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5780 PP_BLOCK_GFX_MG, 5781 pp_support_state, 5782 pp_state); 5783 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5784 } 5785 5786 return 0; 5787 } 5788 5789 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, 5790 enum amd_clockgating_state state) 5791 { 5792 5793 uint32_t msg_id, pp_state = 0; 5794 uint32_t pp_support_state = 0; 5795 5796 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5797 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5798 pp_support_state = PP_STATE_SUPPORT_LS; 5799 pp_state = PP_STATE_LS; 5800 } 5801 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5802 pp_support_state |= PP_STATE_SUPPORT_CG; 5803 pp_state |= PP_STATE_CG; 5804 } 5805 if (state == AMD_CG_STATE_UNGATE) 5806 pp_state = 0; 5807 5808 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5809 PP_BLOCK_GFX_CG, 5810 pp_support_state, 5811 pp_state); 5812 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5813 } 5814 5815 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) { 5816 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5817 pp_support_state = PP_STATE_SUPPORT_LS; 5818 pp_state = PP_STATE_LS; 5819 } 5820 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5821 pp_support_state |= PP_STATE_SUPPORT_CG; 5822 pp_state |= PP_STATE_CG; 5823 } 5824 if (state == AMD_CG_STATE_UNGATE) 5825 pp_state = 0; 5826 5827 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5828 PP_BLOCK_GFX_3D, 5829 pp_support_state, 5830 pp_state); 5831 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5832 } 5833 5834 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5835 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5836 pp_support_state = PP_STATE_SUPPORT_LS; 5837 pp_state = PP_STATE_LS; 5838 } 5839 5840 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5841 pp_support_state |= PP_STATE_SUPPORT_CG; 5842 pp_state |= PP_STATE_CG; 5843 } 5844 5845 if (state == AMD_CG_STATE_UNGATE) 5846 pp_state = 0; 5847 5848 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5849 PP_BLOCK_GFX_MG, 5850 pp_support_state, 5851 pp_state); 5852 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5853 } 5854 5855 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 5856 pp_support_state = PP_STATE_SUPPORT_LS; 5857 5858 if (state == AMD_CG_STATE_UNGATE) 5859 pp_state = 0; 5860 else 5861 pp_state = PP_STATE_LS; 5862 5863 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5864 PP_BLOCK_GFX_RLC, 5865 pp_support_state, 5866 pp_state); 5867 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5868 } 5869 5870 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 5871 pp_support_state = PP_STATE_SUPPORT_LS; 5872 5873 if (state == AMD_CG_STATE_UNGATE) 5874 pp_state = 0; 5875 else 5876 pp_state = PP_STATE_LS; 5877 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5878 PP_BLOCK_GFX_CP, 5879 pp_support_state, 5880 pp_state); 5881 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5882 } 5883 5884 return 0; 5885 } 5886 5887 static int gfx_v8_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5888 enum amd_clockgating_state state) 5889 { 5890 struct amdgpu_device *adev = ip_block->adev; 5891 5892 if (amdgpu_sriov_vf(adev)) 5893 return 0; 5894 5895 switch (adev->asic_type) { 5896 case CHIP_FIJI: 5897 case CHIP_CARRIZO: 5898 case CHIP_STONEY: 5899 gfx_v8_0_update_gfx_clock_gating(adev, 5900 state == AMD_CG_STATE_GATE); 5901 break; 5902 case CHIP_TONGA: 5903 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state); 5904 break; 5905 case CHIP_POLARIS10: 5906 case CHIP_POLARIS11: 5907 case CHIP_POLARIS12: 5908 case CHIP_VEGAM: 5909 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state); 5910 break; 5911 default: 5912 break; 5913 } 5914 return 0; 5915 } 5916 5917 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring) 5918 { 5919 return *ring->rptr_cpu_addr; 5920 } 5921 5922 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5923 { 5924 struct amdgpu_device *adev = ring->adev; 5925 5926 if (ring->use_doorbell) 5927 /* XXX check if swapping is necessary on BE */ 5928 return *ring->wptr_cpu_addr; 5929 else 5930 return RREG32(mmCP_RB0_WPTR); 5931 } 5932 5933 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5934 { 5935 struct amdgpu_device *adev = ring->adev; 5936 5937 if (ring->use_doorbell) { 5938 /* XXX check if swapping is necessary on BE */ 5939 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 5940 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 5941 } else { 5942 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5943 (void)RREG32(mmCP_RB0_WPTR); 5944 } 5945 } 5946 5947 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5948 { 5949 u32 ref_and_mask, reg_mem_engine; 5950 struct amdgpu_device *adev = ring->adev; 5951 5952 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5953 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5954 return; 5955 } 5956 5957 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 5958 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 5959 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 5960 WAIT_REG_MEM_FUNCTION(3) | /* == */ 5961 reg_mem_engine)); 5962 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ); 5963 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE); 5964 amdgpu_ring_write(ring, ref_and_mask); 5965 amdgpu_ring_write(ring, ref_and_mask); 5966 amdgpu_ring_write(ring, 0x20); /* poll interval */ 5967 } 5968 5969 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) 5970 { 5971 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 5972 amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) | 5973 EVENT_INDEX(4)); 5974 5975 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 5976 amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) | 5977 EVENT_INDEX(0)); 5978 } 5979 5980 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5981 struct amdgpu_job *job, 5982 struct amdgpu_ib *ib, 5983 uint32_t flags) 5984 { 5985 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5986 u32 header, control = 0; 5987 5988 if (ib->flags & AMDGPU_IB_FLAG_CE) 5989 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5990 else 5991 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5992 5993 control |= ib->length_dw | (vmid << 24); 5994 5995 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5996 control |= INDIRECT_BUFFER_PRE_ENB(1); 5997 5998 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5999 gfx_v8_0_ring_emit_de_meta(ring); 6000 } 6001 6002 amdgpu_ring_write(ring, header); 6003 amdgpu_ring_write(ring, 6004 #ifdef __BIG_ENDIAN 6005 (2 << 0) | 6006 #endif 6007 (ib->gpu_addr & 0xFFFFFFFC)); 6008 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6009 amdgpu_ring_write(ring, control); 6010 } 6011 6012 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 6013 struct amdgpu_job *job, 6014 struct amdgpu_ib *ib, 6015 uint32_t flags) 6016 { 6017 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6018 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 6019 6020 /* Currently, there is a high possibility to get wave ID mismatch 6021 * between ME and GDS, leading to a hw deadlock, because ME generates 6022 * different wave IDs than the GDS expects. This situation happens 6023 * randomly when at least 5 compute pipes use GDS ordered append. 6024 * The wave IDs generated by ME are also wrong after suspend/resume. 6025 * Those are probably bugs somewhere else in the kernel driver. 6026 * 6027 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6028 * GDS to 0 for this ring (me/pipe). 6029 */ 6030 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6031 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6032 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); 6033 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6034 } 6035 6036 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6037 amdgpu_ring_write(ring, 6038 #ifdef __BIG_ENDIAN 6039 (2 << 0) | 6040 #endif 6041 (ib->gpu_addr & 0xFFFFFFFC)); 6042 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6043 amdgpu_ring_write(ring, control); 6044 } 6045 6046 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr, 6047 u64 seq, unsigned flags) 6048 { 6049 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6050 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6051 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC; 6052 6053 /* Workaround for cache flush problems. First send a dummy EOP 6054 * event down the pipe with seq one below. 6055 */ 6056 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6057 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6058 EOP_TC_ACTION_EN | 6059 EOP_TC_WB_ACTION_EN | 6060 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6061 EVENT_INDEX(5))); 6062 amdgpu_ring_write(ring, addr & 0xfffffffc); 6063 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6064 DATA_SEL(1) | INT_SEL(0)); 6065 amdgpu_ring_write(ring, lower_32_bits(seq - 1)); 6066 amdgpu_ring_write(ring, upper_32_bits(seq - 1)); 6067 6068 /* Then send the real EOP event down the pipe: 6069 * EVENT_WRITE_EOP - flush caches, send int */ 6070 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6071 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6072 EOP_TC_ACTION_EN | 6073 EOP_TC_WB_ACTION_EN | 6074 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6075 EVENT_INDEX(5) | 6076 (exec ? EOP_EXEC : 0))); 6077 amdgpu_ring_write(ring, addr & 0xfffffffc); 6078 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6079 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6080 amdgpu_ring_write(ring, lower_32_bits(seq)); 6081 amdgpu_ring_write(ring, upper_32_bits(seq)); 6082 6083 } 6084 6085 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6086 { 6087 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6088 uint32_t seq = ring->fence_drv.sync_seq; 6089 uint64_t addr = ring->fence_drv.gpu_addr; 6090 6091 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6092 amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */ 6093 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 6094 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */ 6095 amdgpu_ring_write(ring, addr & 0xfffffffc); 6096 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); 6097 amdgpu_ring_write(ring, seq); 6098 amdgpu_ring_write(ring, 0xffffffff); 6099 amdgpu_ring_write(ring, 4); /* poll interval */ 6100 } 6101 6102 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6103 unsigned vmid, uint64_t pd_addr) 6104 { 6105 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6106 6107 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6108 6109 /* wait for the invalidate to complete */ 6110 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6111 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 6112 WAIT_REG_MEM_FUNCTION(0) | /* always */ 6113 WAIT_REG_MEM_ENGINE(0))); /* me */ 6114 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); 6115 amdgpu_ring_write(ring, 0); 6116 amdgpu_ring_write(ring, 0); /* ref */ 6117 amdgpu_ring_write(ring, 0); /* mask */ 6118 amdgpu_ring_write(ring, 0x20); /* poll interval */ 6119 6120 /* compute doesn't have PFP */ 6121 if (usepfp) { 6122 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6123 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6124 amdgpu_ring_write(ring, 0x0); 6125 } 6126 } 6127 6128 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 6129 { 6130 return *ring->wptr_cpu_addr; 6131 } 6132 6133 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 6134 { 6135 struct amdgpu_device *adev = ring->adev; 6136 6137 /* XXX check if swapping is necessary on BE */ 6138 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 6139 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 6140 } 6141 6142 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, 6143 u64 addr, u64 seq, 6144 unsigned flags) 6145 { 6146 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6147 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6148 6149 /* RELEASE_MEM - flush caches, send int */ 6150 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 6151 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6152 EOP_TC_ACTION_EN | 6153 EOP_TC_WB_ACTION_EN | 6154 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6155 EVENT_INDEX(5))); 6156 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6157 amdgpu_ring_write(ring, addr & 0xfffffffc); 6158 amdgpu_ring_write(ring, upper_32_bits(addr)); 6159 amdgpu_ring_write(ring, lower_32_bits(seq)); 6160 amdgpu_ring_write(ring, upper_32_bits(seq)); 6161 } 6162 6163 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6164 u64 seq, unsigned int flags) 6165 { 6166 /* write fence seq to the "addr" */ 6167 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6168 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6169 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6170 amdgpu_ring_write(ring, lower_32_bits(addr)); 6171 amdgpu_ring_write(ring, upper_32_bits(addr)); 6172 amdgpu_ring_write(ring, lower_32_bits(seq)); 6173 6174 if (flags & AMDGPU_FENCE_FLAG_INT) { 6175 /* set register to trigger INT */ 6176 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6177 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6178 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6179 amdgpu_ring_write(ring, mmCPC_INT_STATUS); 6180 amdgpu_ring_write(ring, 0); 6181 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6182 } 6183 } 6184 6185 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring) 6186 { 6187 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 6188 amdgpu_ring_write(ring, 0); 6189 } 6190 6191 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 6192 { 6193 uint32_t dw2 = 0; 6194 6195 if (amdgpu_sriov_vf(ring->adev)) 6196 gfx_v8_0_ring_emit_ce_meta(ring); 6197 6198 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6199 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6200 gfx_v8_0_ring_emit_vgt_flush(ring); 6201 /* set load_global_config & load_global_uconfig */ 6202 dw2 |= 0x8001; 6203 /* set load_cs_sh_regs */ 6204 dw2 |= 0x01000000; 6205 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6206 dw2 |= 0x10002; 6207 6208 /* set load_ce_ram if preamble presented */ 6209 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 6210 dw2 |= 0x10000000; 6211 } else { 6212 /* still load_ce_ram if this is the first time preamble presented 6213 * although there is no context switch happens. 6214 */ 6215 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 6216 dw2 |= 0x10000000; 6217 } 6218 6219 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6220 amdgpu_ring_write(ring, dw2); 6221 amdgpu_ring_write(ring, 0); 6222 } 6223 6224 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 6225 uint64_t addr) 6226 { 6227 unsigned ret; 6228 6229 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6230 amdgpu_ring_write(ring, lower_32_bits(addr)); 6231 amdgpu_ring_write(ring, upper_32_bits(addr)); 6232 /* discard following DWs if *cond_exec_gpu_addr==0 */ 6233 amdgpu_ring_write(ring, 0); 6234 ret = ring->wptr & ring->buf_mask; 6235 /* patch dummy value later */ 6236 amdgpu_ring_write(ring, 0); 6237 return ret; 6238 } 6239 6240 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6241 uint32_t reg_val_offs) 6242 { 6243 struct amdgpu_device *adev = ring->adev; 6244 6245 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6246 amdgpu_ring_write(ring, 0 | /* src: register*/ 6247 (5 << 8) | /* dst: memory */ 6248 (1 << 20)); /* write confirm */ 6249 amdgpu_ring_write(ring, reg); 6250 amdgpu_ring_write(ring, 0); 6251 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6252 reg_val_offs * 4)); 6253 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6254 reg_val_offs * 4)); 6255 } 6256 6257 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6258 uint32_t val) 6259 { 6260 uint32_t cmd; 6261 6262 switch (ring->funcs->type) { 6263 case AMDGPU_RING_TYPE_GFX: 6264 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6265 break; 6266 case AMDGPU_RING_TYPE_KIQ: 6267 cmd = 1 << 16; /* no inc addr */ 6268 break; 6269 default: 6270 cmd = WR_CONFIRM; 6271 break; 6272 } 6273 6274 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6275 amdgpu_ring_write(ring, cmd); 6276 amdgpu_ring_write(ring, reg); 6277 amdgpu_ring_write(ring, 0); 6278 amdgpu_ring_write(ring, val); 6279 } 6280 6281 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 6282 { 6283 struct amdgpu_device *adev = ring->adev; 6284 uint32_t value = 0; 6285 6286 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 6287 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 6288 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 6289 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 6290 WREG32(mmSQ_CMD, value); 6291 } 6292 6293 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6294 enum amdgpu_interrupt_state state) 6295 { 6296 WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE, 6297 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6298 } 6299 6300 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6301 int me, int pipe, 6302 enum amdgpu_interrupt_state state) 6303 { 6304 u32 mec_int_cntl, mec_int_cntl_reg; 6305 6306 /* 6307 * amdgpu controls only the first MEC. That's why this function only 6308 * handles the setting of interrupts for this specific MEC. All other 6309 * pipes' interrupts are set by amdkfd. 6310 */ 6311 6312 if (me == 1) { 6313 switch (pipe) { 6314 case 0: 6315 mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL; 6316 break; 6317 case 1: 6318 mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL; 6319 break; 6320 case 2: 6321 mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL; 6322 break; 6323 case 3: 6324 mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL; 6325 break; 6326 default: 6327 DRM_DEBUG("invalid pipe %d\n", pipe); 6328 return; 6329 } 6330 } else { 6331 DRM_DEBUG("invalid me %d\n", me); 6332 return; 6333 } 6334 6335 switch (state) { 6336 case AMDGPU_IRQ_STATE_DISABLE: 6337 mec_int_cntl = RREG32(mec_int_cntl_reg); 6338 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6339 WREG32(mec_int_cntl_reg, mec_int_cntl); 6340 break; 6341 case AMDGPU_IRQ_STATE_ENABLE: 6342 mec_int_cntl = RREG32(mec_int_cntl_reg); 6343 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6344 WREG32(mec_int_cntl_reg, mec_int_cntl); 6345 break; 6346 default: 6347 break; 6348 } 6349 } 6350 6351 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6352 struct amdgpu_irq_src *source, 6353 unsigned type, 6354 enum amdgpu_interrupt_state state) 6355 { 6356 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE, 6357 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6358 6359 return 0; 6360 } 6361 6362 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6363 struct amdgpu_irq_src *source, 6364 unsigned type, 6365 enum amdgpu_interrupt_state state) 6366 { 6367 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE, 6368 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6369 6370 return 0; 6371 } 6372 6373 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6374 struct amdgpu_irq_src *src, 6375 unsigned type, 6376 enum amdgpu_interrupt_state state) 6377 { 6378 switch (type) { 6379 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6380 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state); 6381 break; 6382 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6383 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6384 break; 6385 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6386 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6387 break; 6388 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6389 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6390 break; 6391 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6392 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6393 break; 6394 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 6395 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 6396 break; 6397 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 6398 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 6399 break; 6400 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 6401 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 6402 break; 6403 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 6404 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 6405 break; 6406 default: 6407 break; 6408 } 6409 return 0; 6410 } 6411 6412 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev, 6413 struct amdgpu_irq_src *source, 6414 unsigned int type, 6415 enum amdgpu_interrupt_state state) 6416 { 6417 int enable_flag; 6418 6419 switch (state) { 6420 case AMDGPU_IRQ_STATE_DISABLE: 6421 enable_flag = 0; 6422 break; 6423 6424 case AMDGPU_IRQ_STATE_ENABLE: 6425 enable_flag = 1; 6426 break; 6427 6428 default: 6429 return -EINVAL; 6430 } 6431 6432 WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6433 WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6434 WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6435 WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6436 WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6437 WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6438 enable_flag); 6439 WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6440 enable_flag); 6441 WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6442 enable_flag); 6443 WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6444 enable_flag); 6445 WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6446 enable_flag); 6447 WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6448 enable_flag); 6449 WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6450 enable_flag); 6451 WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6452 enable_flag); 6453 6454 return 0; 6455 } 6456 6457 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev, 6458 struct amdgpu_irq_src *source, 6459 unsigned int type, 6460 enum amdgpu_interrupt_state state) 6461 { 6462 int enable_flag; 6463 6464 switch (state) { 6465 case AMDGPU_IRQ_STATE_DISABLE: 6466 enable_flag = 1; 6467 break; 6468 6469 case AMDGPU_IRQ_STATE_ENABLE: 6470 enable_flag = 0; 6471 break; 6472 6473 default: 6474 return -EINVAL; 6475 } 6476 6477 WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL, 6478 enable_flag); 6479 6480 return 0; 6481 } 6482 6483 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev, 6484 struct amdgpu_irq_src *source, 6485 struct amdgpu_iv_entry *entry) 6486 { 6487 int i; 6488 u8 me_id, pipe_id, queue_id; 6489 struct amdgpu_ring *ring; 6490 6491 DRM_DEBUG("IH: CP EOP\n"); 6492 me_id = (entry->ring_id & 0x0c) >> 2; 6493 pipe_id = (entry->ring_id & 0x03) >> 0; 6494 queue_id = (entry->ring_id & 0x70) >> 4; 6495 6496 switch (me_id) { 6497 case 0: 6498 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 6499 break; 6500 case 1: 6501 case 2: 6502 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6503 ring = &adev->gfx.compute_ring[i]; 6504 /* Per-queue interrupt is supported for MEC starting from VI. 6505 * The interrupt can only be enabled/disabled per pipe instead of per queue. 6506 */ 6507 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 6508 amdgpu_fence_process(ring); 6509 } 6510 break; 6511 } 6512 return 0; 6513 } 6514 6515 static void gfx_v8_0_fault(struct amdgpu_device *adev, 6516 struct amdgpu_iv_entry *entry) 6517 { 6518 u8 me_id, pipe_id, queue_id; 6519 struct amdgpu_ring *ring; 6520 int i; 6521 6522 me_id = (entry->ring_id & 0x0c) >> 2; 6523 pipe_id = (entry->ring_id & 0x03) >> 0; 6524 queue_id = (entry->ring_id & 0x70) >> 4; 6525 6526 switch (me_id) { 6527 case 0: 6528 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 6529 break; 6530 case 1: 6531 case 2: 6532 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6533 ring = &adev->gfx.compute_ring[i]; 6534 if (ring->me == me_id && ring->pipe == pipe_id && 6535 ring->queue == queue_id) 6536 drm_sched_fault(&ring->sched); 6537 } 6538 break; 6539 } 6540 } 6541 6542 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev, 6543 struct amdgpu_irq_src *source, 6544 struct amdgpu_iv_entry *entry) 6545 { 6546 DRM_ERROR("Illegal register access in command stream\n"); 6547 gfx_v8_0_fault(adev, entry); 6548 return 0; 6549 } 6550 6551 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev, 6552 struct amdgpu_irq_src *source, 6553 struct amdgpu_iv_entry *entry) 6554 { 6555 DRM_ERROR("Illegal instruction in command stream\n"); 6556 gfx_v8_0_fault(adev, entry); 6557 return 0; 6558 } 6559 6560 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev, 6561 struct amdgpu_irq_src *source, 6562 struct amdgpu_iv_entry *entry) 6563 { 6564 DRM_ERROR("CP EDC/ECC error detected."); 6565 return 0; 6566 } 6567 6568 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data, 6569 bool from_wq) 6570 { 6571 u32 enc, se_id, sh_id, cu_id; 6572 char type[20]; 6573 int sq_edc_source = -1; 6574 6575 enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING); 6576 se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID); 6577 6578 switch (enc) { 6579 case 0: 6580 drm_info(adev_to_drm(adev), "SQ general purpose intr detected:" 6581 "se_id %d, immed_overflow %d, host_reg_overflow %d," 6582 "host_cmd_overflow %d, cmd_timestamp %d," 6583 "reg_timestamp %d, thread_trace_buff_full %d," 6584 "wlt %d, thread_trace %d.\n", 6585 se_id, 6586 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW), 6587 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW), 6588 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW), 6589 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP), 6590 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP), 6591 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL), 6592 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT), 6593 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE) 6594 ); 6595 break; 6596 case 1: 6597 case 2: 6598 6599 cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID); 6600 sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID); 6601 6602 /* 6603 * This function can be called either directly from ISR 6604 * or from BH in which case we can access SQ_EDC_INFO 6605 * instance 6606 */ 6607 if (from_wq) { 6608 mutex_lock(&adev->grbm_idx_mutex); 6609 gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id, 0); 6610 6611 sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE); 6612 6613 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 6614 mutex_unlock(&adev->grbm_idx_mutex); 6615 } 6616 6617 if (enc == 1) 6618 sprintf(type, "instruction intr"); 6619 else 6620 sprintf(type, "EDC/ECC error"); 6621 6622 drm_info(adev_to_drm(adev), 6623 "SQ %s detected: " 6624 "se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d " 6625 "trap %s, sq_ed_info.source %s.\n", 6626 type, se_id, sh_id, cu_id, 6627 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID), 6628 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID), 6629 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID), 6630 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false", 6631 (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable" 6632 ); 6633 break; 6634 default: 6635 DRM_ERROR("SQ invalid encoding type\n."); 6636 } 6637 } 6638 6639 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work) 6640 { 6641 6642 struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work); 6643 struct sq_work *sq_work = container_of(work, struct sq_work, work); 6644 6645 gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true); 6646 } 6647 6648 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev, 6649 struct amdgpu_irq_src *source, 6650 struct amdgpu_iv_entry *entry) 6651 { 6652 unsigned ih_data = entry->src_data[0]; 6653 6654 /* 6655 * Try to submit work so SQ_EDC_INFO can be accessed from 6656 * BH. If previous work submission hasn't finished yet 6657 * just print whatever info is possible directly from the ISR. 6658 */ 6659 if (work_pending(&adev->gfx.sq_work.work)) { 6660 gfx_v8_0_parse_sq_irq(adev, ih_data, false); 6661 } else { 6662 adev->gfx.sq_work.ih_data = ih_data; 6663 schedule_work(&adev->gfx.sq_work.work); 6664 } 6665 6666 return 0; 6667 } 6668 6669 static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring) 6670 { 6671 amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 6672 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6673 PACKET3_TC_ACTION_ENA | 6674 PACKET3_SH_KCACHE_ACTION_ENA | 6675 PACKET3_SH_ICACHE_ACTION_ENA | 6676 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6677 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6678 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6679 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6680 } 6681 6682 static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring) 6683 { 6684 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6685 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6686 PACKET3_TC_ACTION_ENA | 6687 PACKET3_SH_KCACHE_ACTION_ENA | 6688 PACKET3_SH_ICACHE_ACTION_ENA | 6689 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6690 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6691 amdgpu_ring_write(ring, 0xff); /* CP_COHER_SIZE_HI */ 6692 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6693 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6694 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6695 } 6696 6697 6698 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6699 #define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT 0x0000007f 6700 static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6701 uint32_t pipe, bool enable) 6702 { 6703 uint32_t val; 6704 uint32_t wcl_cs_reg; 6705 6706 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT; 6707 6708 switch (pipe) { 6709 case 0: 6710 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0; 6711 break; 6712 case 1: 6713 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1; 6714 break; 6715 case 2: 6716 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2; 6717 break; 6718 case 3: 6719 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3; 6720 break; 6721 default: 6722 DRM_DEBUG("invalid pipe %d\n", pipe); 6723 return; 6724 } 6725 6726 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6727 6728 } 6729 6730 #define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT 0x07ffffff 6731 static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6732 { 6733 struct amdgpu_device *adev = ring->adev; 6734 uint32_t val; 6735 int i; 6736 6737 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6738 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6739 * around 25% of gpu resources. 6740 */ 6741 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6742 amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val); 6743 6744 /* Restrict waves for normal/low priority compute queues as well 6745 * to get best QoS for high priority compute jobs. 6746 * 6747 * amdgpu controls only 1st ME(0-3 CS pipes). 6748 */ 6749 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6750 if (i != ring->pipe) 6751 gfx_v8_0_emit_wave_limit_cs(ring, i, enable); 6752 6753 } 6754 6755 } 6756 6757 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { 6758 .name = "gfx_v8_0", 6759 .early_init = gfx_v8_0_early_init, 6760 .late_init = gfx_v8_0_late_init, 6761 .sw_init = gfx_v8_0_sw_init, 6762 .sw_fini = gfx_v8_0_sw_fini, 6763 .hw_init = gfx_v8_0_hw_init, 6764 .hw_fini = gfx_v8_0_hw_fini, 6765 .suspend = gfx_v8_0_suspend, 6766 .resume = gfx_v8_0_resume, 6767 .is_idle = gfx_v8_0_is_idle, 6768 .wait_for_idle = gfx_v8_0_wait_for_idle, 6769 .soft_reset = gfx_v8_0_soft_reset, 6770 .set_clockgating_state = gfx_v8_0_set_clockgating_state, 6771 .set_powergating_state = gfx_v8_0_set_powergating_state, 6772 .get_clockgating_state = gfx_v8_0_get_clockgating_state, 6773 }; 6774 6775 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { 6776 .type = AMDGPU_RING_TYPE_GFX, 6777 .align_mask = 0xff, 6778 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6779 .support_64bit_ptrs = false, 6780 .get_rptr = gfx_v8_0_ring_get_rptr, 6781 .get_wptr = gfx_v8_0_ring_get_wptr_gfx, 6782 .set_wptr = gfx_v8_0_ring_set_wptr_gfx, 6783 .emit_frame_size = /* maximum 215dw if count 16 IBs in */ 6784 5 + /* COND_EXEC */ 6785 7 + /* PIPELINE_SYNC */ 6786 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */ 6787 12 + /* FENCE for VM_FLUSH */ 6788 20 + /* GDS switch */ 6789 4 + /* double SWITCH_BUFFER, 6790 the first COND_EXEC jump to the place just 6791 prior to this double SWITCH_BUFFER */ 6792 5 + /* COND_EXEC */ 6793 7 + /* HDP_flush */ 6794 4 + /* VGT_flush */ 6795 14 + /* CE_META */ 6796 31 + /* DE_META */ 6797 3 + /* CNTX_CTRL */ 6798 5 + /* HDP_INVL */ 6799 12 + 12 + /* FENCE x2 */ 6800 2 + /* SWITCH_BUFFER */ 6801 5, /* SURFACE_SYNC */ 6802 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */ 6803 .emit_ib = gfx_v8_0_ring_emit_ib_gfx, 6804 .emit_fence = gfx_v8_0_ring_emit_fence_gfx, 6805 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6806 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6807 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6808 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6809 .test_ring = gfx_v8_0_ring_test_ring, 6810 .test_ib = gfx_v8_0_ring_test_ib, 6811 .insert_nop = amdgpu_ring_insert_nop, 6812 .pad_ib = amdgpu_ring_generic_pad_ib, 6813 .emit_switch_buffer = gfx_v8_ring_emit_sb, 6814 .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl, 6815 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, 6816 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6817 .soft_recovery = gfx_v8_0_ring_soft_recovery, 6818 .emit_mem_sync = gfx_v8_0_emit_mem_sync, 6819 }; 6820 6821 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { 6822 .type = AMDGPU_RING_TYPE_COMPUTE, 6823 .align_mask = 0xff, 6824 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6825 .support_64bit_ptrs = false, 6826 .get_rptr = gfx_v8_0_ring_get_rptr, 6827 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6828 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6829 .emit_frame_size = 6830 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_ib_schedule) */ 6831 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6832 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6833 5 + /* hdp_invalidate */ 6834 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6835 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_vm_flush) */ 6836 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ 6837 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ 6838 7 + /* gfx_v8_0_emit_mem_sync_compute */ 6839 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6840 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6841 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6842 .emit_ib = gfx_v8_0_ring_emit_ib_compute, 6843 .emit_fence = gfx_v8_0_ring_emit_fence_compute, 6844 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6845 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6846 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6847 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6848 .test_ring = gfx_v8_0_ring_test_ring, 6849 .test_ib = gfx_v8_0_ring_test_ib, 6850 .insert_nop = amdgpu_ring_insert_nop, 6851 .pad_ib = amdgpu_ring_generic_pad_ib, 6852 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6853 .soft_recovery = gfx_v8_0_ring_soft_recovery, 6854 .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute, 6855 .emit_wave_limit = gfx_v8_0_emit_wave_limit, 6856 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, 6857 }; 6858 6859 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { 6860 .type = AMDGPU_RING_TYPE_KIQ, 6861 .align_mask = 0xff, 6862 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6863 .support_64bit_ptrs = false, 6864 .get_rptr = gfx_v8_0_ring_get_rptr, 6865 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6866 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6867 .emit_frame_size = 6868 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6869 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6870 5 + /* hdp_invalidate */ 6871 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6872 17 + /* gfx_v8_0_ring_emit_vm_flush */ 6873 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6874 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6875 .emit_fence = gfx_v8_0_ring_emit_fence_kiq, 6876 .test_ring = gfx_v8_0_ring_test_ring, 6877 .insert_nop = amdgpu_ring_insert_nop, 6878 .pad_ib = amdgpu_ring_generic_pad_ib, 6879 .emit_rreg = gfx_v8_0_ring_emit_rreg, 6880 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6881 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6882 }; 6883 6884 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev) 6885 { 6886 int i; 6887 6888 adev->gfx.kiq[0].ring.funcs = &gfx_v8_0_ring_funcs_kiq; 6889 6890 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6891 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx; 6892 6893 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6894 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute; 6895 } 6896 6897 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = { 6898 .set = gfx_v8_0_set_eop_interrupt_state, 6899 .process = gfx_v8_0_eop_irq, 6900 }; 6901 6902 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = { 6903 .set = gfx_v8_0_set_priv_reg_fault_state, 6904 .process = gfx_v8_0_priv_reg_irq, 6905 }; 6906 6907 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = { 6908 .set = gfx_v8_0_set_priv_inst_fault_state, 6909 .process = gfx_v8_0_priv_inst_irq, 6910 }; 6911 6912 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = { 6913 .set = gfx_v8_0_set_cp_ecc_int_state, 6914 .process = gfx_v8_0_cp_ecc_error_irq, 6915 }; 6916 6917 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = { 6918 .set = gfx_v8_0_set_sq_int_state, 6919 .process = gfx_v8_0_sq_irq, 6920 }; 6921 6922 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev) 6923 { 6924 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6925 adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs; 6926 6927 adev->gfx.priv_reg_irq.num_types = 1; 6928 adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs; 6929 6930 adev->gfx.priv_inst_irq.num_types = 1; 6931 adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs; 6932 6933 adev->gfx.cp_ecc_error_irq.num_types = 1; 6934 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs; 6935 6936 adev->gfx.sq_irq.num_types = 1; 6937 adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs; 6938 } 6939 6940 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev) 6941 { 6942 adev->gfx.rlc.funcs = &iceland_rlc_funcs; 6943 } 6944 6945 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev) 6946 { 6947 /* init asci gds info */ 6948 adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE); 6949 adev->gds.gws_size = 64; 6950 adev->gds.oa_size = 16; 6951 adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); 6952 } 6953 6954 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6955 u32 bitmap) 6956 { 6957 u32 data; 6958 6959 if (!bitmap) 6960 return; 6961 6962 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6963 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6964 6965 WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data); 6966 } 6967 6968 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6969 { 6970 u32 data, mask; 6971 6972 data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) | 6973 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG); 6974 6975 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6976 6977 return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask; 6978 } 6979 6980 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev) 6981 { 6982 int i, j, k, counter, active_cu_number = 0; 6983 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6984 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 6985 unsigned disable_masks[4 * 2]; 6986 u32 ao_cu_num; 6987 6988 memset(cu_info, 0, sizeof(*cu_info)); 6989 6990 if (adev->flags & AMD_IS_APU) 6991 ao_cu_num = 2; 6992 else 6993 ao_cu_num = adev->gfx.config.max_cu_per_sh; 6994 6995 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 4, 2); 6996 6997 mutex_lock(&adev->grbm_idx_mutex); 6998 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6999 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7000 mask = 1; 7001 ao_bitmap = 0; 7002 counter = 0; 7003 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7004 if (i < 4 && j < 2) 7005 gfx_v8_0_set_user_cu_inactive_bitmap( 7006 adev, disable_masks[i * 2 + j]); 7007 bitmap = gfx_v8_0_get_cu_active_bitmap(adev); 7008 cu_info->bitmap[0][i][j] = bitmap; 7009 7010 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7011 if (bitmap & mask) { 7012 if (counter < ao_cu_num) 7013 ao_bitmap |= mask; 7014 counter ++; 7015 } 7016 mask <<= 1; 7017 } 7018 active_cu_number += counter; 7019 if (i < 2 && j < 2) 7020 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7021 cu_info->ao_cu_bitmap[i][j] = ao_bitmap; 7022 } 7023 } 7024 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7025 mutex_unlock(&adev->grbm_idx_mutex); 7026 7027 cu_info->number = active_cu_number; 7028 cu_info->ao_cu_mask = ao_cu_mask; 7029 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7030 cu_info->max_waves_per_simd = 10; 7031 cu_info->max_scratch_slots_per_cu = 32; 7032 cu_info->wave_front_size = 64; 7033 cu_info->lds_size = 64; 7034 } 7035 7036 const struct amdgpu_ip_block_version gfx_v8_0_ip_block = 7037 { 7038 .type = AMD_IP_BLOCK_TYPE_GFX, 7039 .major = 8, 7040 .minor = 0, 7041 .rev = 0, 7042 .funcs = &gfx_v8_0_ip_funcs, 7043 }; 7044 7045 const struct amdgpu_ip_block_version gfx_v8_1_ip_block = 7046 { 7047 .type = AMD_IP_BLOCK_TYPE_GFX, 7048 .major = 8, 7049 .minor = 1, 7050 .rev = 0, 7051 .funcs = &gfx_v8_0_ip_funcs, 7052 }; 7053 7054 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 7055 { 7056 uint64_t ce_payload_addr; 7057 int cnt_ce; 7058 union { 7059 struct vi_ce_ib_state regular; 7060 struct vi_ce_ib_state_chained_ib chained; 7061 } ce_payload = {}; 7062 7063 if (ring->adev->virt.chained_ib_support) { 7064 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7065 offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload); 7066 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2; 7067 } else { 7068 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7069 offsetof(struct vi_gfx_meta_data, ce_payload); 7070 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2; 7071 } 7072 7073 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce)); 7074 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 7075 WRITE_DATA_DST_SEL(8) | 7076 WR_CONFIRM) | 7077 WRITE_DATA_CACHE_POLICY(0)); 7078 amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr)); 7079 amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr)); 7080 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2); 7081 } 7082 7083 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring) 7084 { 7085 uint64_t de_payload_addr, gds_addr, csa_addr; 7086 int cnt_de; 7087 union { 7088 struct vi_de_ib_state regular; 7089 struct vi_de_ib_state_chained_ib chained; 7090 } de_payload = {}; 7091 7092 csa_addr = amdgpu_csa_vaddr(ring->adev); 7093 gds_addr = csa_addr + 4096; 7094 if (ring->adev->virt.chained_ib_support) { 7095 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr); 7096 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr); 7097 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload); 7098 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2; 7099 } else { 7100 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr); 7101 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr); 7102 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload); 7103 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2; 7104 } 7105 7106 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de)); 7107 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 7108 WRITE_DATA_DST_SEL(8) | 7109 WR_CONFIRM) | 7110 WRITE_DATA_CACHE_POLICY(0)); 7111 amdgpu_ring_write(ring, lower_32_bits(de_payload_addr)); 7112 amdgpu_ring_write(ring, upper_32_bits(de_payload_addr)); 7113 amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2); 7114 } 7115