1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "amdgpu_ring.h" 33 #include "vi.h" 34 #include "vi_structs.h" 35 #include "vid.h" 36 #include "amdgpu_ucode.h" 37 #include "amdgpu_atombios.h" 38 #include "atombios_i2c.h" 39 #include "clearstate_vi.h" 40 41 #include "gmc/gmc_8_2_d.h" 42 #include "gmc/gmc_8_2_sh_mask.h" 43 44 #include "oss/oss_3_0_d.h" 45 #include "oss/oss_3_0_sh_mask.h" 46 47 #include "bif/bif_5_0_d.h" 48 #include "bif/bif_5_0_sh_mask.h" 49 #include "gca/gfx_8_0_d.h" 50 #include "gca/gfx_8_0_enum.h" 51 #include "gca/gfx_8_0_sh_mask.h" 52 53 #include "dce/dce_10_0_d.h" 54 #include "dce/dce_10_0_sh_mask.h" 55 56 #include "smu/smu_7_1_3_d.h" 57 58 #include "ivsrcid/ivsrcid_vislands30.h" 59 60 #define GFX8_NUM_GFX_RINGS 1 61 #define GFX8_MEC_HPD_SIZE 4096 62 63 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001 64 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001 65 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002 66 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003 67 68 #define ARRAY_MODE(x) ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT) 69 #define PIPE_CONFIG(x) ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT) 70 #define TILE_SPLIT(x) ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT) 71 #define MICRO_TILE_MODE_NEW(x) ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT) 72 #define SAMPLE_SPLIT(x) ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT) 73 #define BANK_WIDTH(x) ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT) 74 #define BANK_HEIGHT(x) ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT) 75 #define MACRO_TILE_ASPECT(x) ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT) 76 #define NUM_BANKS(x) ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT) 77 78 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK 0x00000001L 79 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK 0x00000002L 80 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK 0x00000004L 81 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK 0x00000008L 82 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK 0x00000010L 83 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK 0x00000020L 84 85 /* BPM SERDES CMD */ 86 #define SET_BPM_SERDES_CMD 1 87 #define CLE_BPM_SERDES_CMD 0 88 89 /* BPM Register Address*/ 90 enum { 91 BPM_REG_CGLS_EN = 0, /* Enable/Disable CGLS */ 92 BPM_REG_CGLS_ON, /* ON/OFF CGLS: shall be controlled by RLC FW */ 93 BPM_REG_CGCG_OVERRIDE, /* Set/Clear CGCG Override */ 94 BPM_REG_MGCG_OVERRIDE, /* Set/Clear MGCG Override */ 95 BPM_REG_FGCG_OVERRIDE, /* Set/Clear FGCG Override */ 96 BPM_REG_FGCG_MAX 97 }; 98 99 #define RLC_FormatDirectRegListLength 14 100 101 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin"); 102 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin"); 103 MODULE_FIRMWARE("amdgpu/carrizo_me.bin"); 104 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin"); 105 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin"); 106 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin"); 107 108 MODULE_FIRMWARE("amdgpu/stoney_ce.bin"); 109 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin"); 110 MODULE_FIRMWARE("amdgpu/stoney_me.bin"); 111 MODULE_FIRMWARE("amdgpu/stoney_mec.bin"); 112 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin"); 113 114 MODULE_FIRMWARE("amdgpu/tonga_ce.bin"); 115 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin"); 116 MODULE_FIRMWARE("amdgpu/tonga_me.bin"); 117 MODULE_FIRMWARE("amdgpu/tonga_mec.bin"); 118 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin"); 119 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin"); 120 121 MODULE_FIRMWARE("amdgpu/topaz_ce.bin"); 122 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin"); 123 MODULE_FIRMWARE("amdgpu/topaz_me.bin"); 124 MODULE_FIRMWARE("amdgpu/topaz_mec.bin"); 125 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin"); 126 127 MODULE_FIRMWARE("amdgpu/fiji_ce.bin"); 128 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin"); 129 MODULE_FIRMWARE("amdgpu/fiji_me.bin"); 130 MODULE_FIRMWARE("amdgpu/fiji_mec.bin"); 131 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin"); 132 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin"); 133 134 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin"); 135 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin"); 136 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin"); 137 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin"); 138 MODULE_FIRMWARE("amdgpu/polaris10_me.bin"); 139 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin"); 140 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin"); 141 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin"); 142 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin"); 143 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin"); 144 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin"); 145 146 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin"); 147 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin"); 148 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin"); 149 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin"); 150 MODULE_FIRMWARE("amdgpu/polaris11_me.bin"); 151 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin"); 152 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin"); 153 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin"); 154 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin"); 155 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin"); 156 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin"); 157 158 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin"); 159 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin"); 160 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin"); 161 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin"); 162 MODULE_FIRMWARE("amdgpu/polaris12_me.bin"); 163 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin"); 164 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin"); 165 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin"); 166 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin"); 167 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin"); 168 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin"); 169 170 MODULE_FIRMWARE("amdgpu/vegam_ce.bin"); 171 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin"); 172 MODULE_FIRMWARE("amdgpu/vegam_me.bin"); 173 MODULE_FIRMWARE("amdgpu/vegam_mec.bin"); 174 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin"); 175 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin"); 176 177 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] = 178 { 179 {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0}, 180 {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1}, 181 {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2}, 182 {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3}, 183 {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4}, 184 {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5}, 185 {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6}, 186 {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7}, 187 {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8}, 188 {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9}, 189 {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10}, 190 {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11}, 191 {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12}, 192 {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13}, 193 {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14}, 194 {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15} 195 }; 196 197 static const u32 golden_settings_tonga_a11[] = 198 { 199 mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208, 200 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 201 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 202 mmGB_GPU_ID, 0x0000000f, 0x00000000, 203 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 204 mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc, 205 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 206 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 207 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 208 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 209 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 210 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 211 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb, 212 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b, 213 mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876, 214 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 215 }; 216 217 static const u32 tonga_golden_common_all[] = 218 { 219 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 220 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 221 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 222 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 223 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 224 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 225 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 226 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 227 }; 228 229 static const u32 tonga_mgcg_cgcg_init[] = 230 { 231 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 232 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 233 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 234 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 235 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 236 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 237 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 238 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 239 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 240 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 241 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 242 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 243 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 244 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 245 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 246 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 247 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 248 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 249 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 250 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 251 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 252 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 253 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 254 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 255 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 256 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 257 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 258 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 259 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 260 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 261 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 262 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 263 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 264 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 265 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 266 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 267 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 268 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 269 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 270 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 271 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 272 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 273 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 274 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 275 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 276 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 277 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 278 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 279 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 280 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 281 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 282 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 283 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 284 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 285 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 286 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 287 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 288 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 289 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 290 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 291 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 292 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 293 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 294 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 295 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 296 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 297 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 298 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 299 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 300 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 301 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 302 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 303 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 304 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 305 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 306 }; 307 308 static const u32 golden_settings_vegam_a11[] = 309 { 310 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 311 mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000, 312 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 313 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 314 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 315 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 316 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a, 317 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e, 318 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 319 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 320 mmSQ_CONFIG, 0x07f80000, 0x01180000, 321 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 322 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 323 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 324 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 325 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054, 326 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 327 }; 328 329 static const u32 vegam_golden_common_all[] = 330 { 331 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 332 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 333 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 334 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 335 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 336 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 337 }; 338 339 static const u32 golden_settings_polaris11_a11[] = 340 { 341 mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208, 342 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 343 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 344 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 345 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 346 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 347 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 348 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 349 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 350 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 351 mmSQ_CONFIG, 0x07f80000, 0x01180000, 352 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 353 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 354 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3, 355 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 356 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210, 357 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 358 }; 359 360 static const u32 polaris11_golden_common_all[] = 361 { 362 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 363 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002, 364 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 365 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 366 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 367 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 368 }; 369 370 static const u32 golden_settings_polaris10_a11[] = 371 { 372 mmATC_MISC_CG, 0x000c0fc0, 0x000c0200, 373 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 374 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 375 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 376 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 377 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 378 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 379 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 380 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a, 381 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 382 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 383 mmSQ_CONFIG, 0x07f80000, 0x07180000, 384 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 385 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 386 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 387 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 388 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 389 }; 390 391 static const u32 polaris10_golden_common_all[] = 392 { 393 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 394 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 395 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 396 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 397 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 398 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 399 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 400 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 401 }; 402 403 static const u32 fiji_golden_common_all[] = 404 { 405 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 406 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a, 407 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e, 408 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 409 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 410 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 411 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 412 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 413 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 414 mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, 415 }; 416 417 static const u32 golden_settings_fiji_a10[] = 418 { 419 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 420 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 421 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 422 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 423 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 424 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 425 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 426 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 427 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 428 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff, 429 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 430 }; 431 432 static const u32 fiji_mgcg_cgcg_init[] = 433 { 434 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 435 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 436 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 437 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 438 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 439 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 440 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 441 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 442 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 443 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 444 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 445 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 446 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 447 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 448 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 449 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 450 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 451 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 452 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 453 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 454 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 455 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 456 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 457 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 458 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 459 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 460 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 461 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 462 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 463 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 464 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 465 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 466 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 467 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 468 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 469 }; 470 471 static const u32 golden_settings_iceland_a11[] = 472 { 473 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 474 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 475 mmDB_DEBUG3, 0xc0000000, 0xc0000000, 476 mmGB_GPU_ID, 0x0000000f, 0x00000000, 477 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 478 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 479 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002, 480 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 481 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 482 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 483 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 484 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 485 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 486 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1, 487 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 488 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010, 489 }; 490 491 static const u32 iceland_golden_common_all[] = 492 { 493 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 494 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 495 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 496 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 497 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 498 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 499 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 500 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 501 }; 502 503 static const u32 iceland_mgcg_cgcg_init[] = 504 { 505 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 506 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 507 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 508 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 509 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100, 510 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100, 511 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100, 512 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 513 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 514 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 515 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 516 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 517 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 518 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 519 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 520 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 521 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 522 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 523 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 524 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 525 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 526 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 527 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100, 528 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 529 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 530 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 531 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 532 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 533 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 534 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 535 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 536 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 537 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 538 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 539 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 540 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 541 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 542 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 543 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 544 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 545 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 546 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 547 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 548 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 549 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 550 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 551 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 552 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 553 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 554 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 555 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 556 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 557 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 558 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 559 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 560 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 561 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 562 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 563 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 564 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 565 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 566 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 567 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 568 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 569 }; 570 571 static const u32 cz_golden_settings_a11[] = 572 { 573 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 574 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 575 mmGB_GPU_ID, 0x0000000f, 0x00000000, 576 mmPA_SC_ENHANCE, 0xffffffff, 0x00000001, 577 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 578 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 579 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 580 mmTA_CNTL_AUX, 0x000f000f, 0x00010000, 581 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 582 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 583 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3, 584 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302 585 }; 586 587 static const u32 cz_golden_common_all[] = 588 { 589 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 590 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 591 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 592 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 593 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 594 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 595 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 596 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 597 }; 598 599 static const u32 cz_mgcg_cgcg_init[] = 600 { 601 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 602 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 603 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 604 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 605 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 606 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 607 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100, 608 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 609 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 610 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 611 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 612 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 613 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 614 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 615 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 616 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 617 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 618 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 619 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 620 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 621 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 622 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 623 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 624 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 625 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 626 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 627 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 628 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 629 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 630 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 631 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 632 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 633 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 634 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 635 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 636 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 637 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 638 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 639 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 640 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 641 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 642 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 643 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 644 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 645 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 646 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 647 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 648 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 649 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 650 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 651 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 652 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 653 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 654 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 655 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 656 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 657 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 658 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 659 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 660 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 661 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 662 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 663 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 664 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 665 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 666 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 667 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 668 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 669 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 670 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 671 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 672 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 673 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 674 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 675 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 676 }; 677 678 static const u32 stoney_golden_settings_a11[] = 679 { 680 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 681 mmGB_GPU_ID, 0x0000000f, 0x00000000, 682 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 683 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 684 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 685 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 686 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 687 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 688 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1, 689 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010, 690 }; 691 692 static const u32 stoney_golden_common_all[] = 693 { 694 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 695 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000, 696 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 697 mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001, 698 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 699 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 700 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 701 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 702 }; 703 704 static const u32 stoney_mgcg_cgcg_init[] = 705 { 706 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 707 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 708 mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 709 mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 710 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200, 711 }; 712 713 714 static const char * const sq_edc_source_names[] = { 715 "SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred", 716 "SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch", 717 "SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return", 718 "SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR", 719 "SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS", 720 "SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS", 721 "SQ_EDC_INFO_SOURCE_TA: EDC source is TA", 722 }; 723 724 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev); 725 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev); 726 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev); 727 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev); 728 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev); 729 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev); 730 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring); 731 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring); 732 733 #define CG_ACLK_CNTL__ACLK_DIVIDER_MASK 0x0000007fL 734 #define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT 0x00000000L 735 736 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev) 737 { 738 uint32_t data; 739 740 switch (adev->asic_type) { 741 case CHIP_TOPAZ: 742 amdgpu_device_program_register_sequence(adev, 743 iceland_mgcg_cgcg_init, 744 ARRAY_SIZE(iceland_mgcg_cgcg_init)); 745 amdgpu_device_program_register_sequence(adev, 746 golden_settings_iceland_a11, 747 ARRAY_SIZE(golden_settings_iceland_a11)); 748 amdgpu_device_program_register_sequence(adev, 749 iceland_golden_common_all, 750 ARRAY_SIZE(iceland_golden_common_all)); 751 break; 752 case CHIP_FIJI: 753 amdgpu_device_program_register_sequence(adev, 754 fiji_mgcg_cgcg_init, 755 ARRAY_SIZE(fiji_mgcg_cgcg_init)); 756 amdgpu_device_program_register_sequence(adev, 757 golden_settings_fiji_a10, 758 ARRAY_SIZE(golden_settings_fiji_a10)); 759 amdgpu_device_program_register_sequence(adev, 760 fiji_golden_common_all, 761 ARRAY_SIZE(fiji_golden_common_all)); 762 break; 763 764 case CHIP_TONGA: 765 amdgpu_device_program_register_sequence(adev, 766 tonga_mgcg_cgcg_init, 767 ARRAY_SIZE(tonga_mgcg_cgcg_init)); 768 amdgpu_device_program_register_sequence(adev, 769 golden_settings_tonga_a11, 770 ARRAY_SIZE(golden_settings_tonga_a11)); 771 amdgpu_device_program_register_sequence(adev, 772 tonga_golden_common_all, 773 ARRAY_SIZE(tonga_golden_common_all)); 774 break; 775 case CHIP_VEGAM: 776 amdgpu_device_program_register_sequence(adev, 777 golden_settings_vegam_a11, 778 ARRAY_SIZE(golden_settings_vegam_a11)); 779 amdgpu_device_program_register_sequence(adev, 780 vegam_golden_common_all, 781 ARRAY_SIZE(vegam_golden_common_all)); 782 break; 783 case CHIP_POLARIS11: 784 case CHIP_POLARIS12: 785 amdgpu_device_program_register_sequence(adev, 786 golden_settings_polaris11_a11, 787 ARRAY_SIZE(golden_settings_polaris11_a11)); 788 amdgpu_device_program_register_sequence(adev, 789 polaris11_golden_common_all, 790 ARRAY_SIZE(polaris11_golden_common_all)); 791 break; 792 case CHIP_POLARIS10: 793 amdgpu_device_program_register_sequence(adev, 794 golden_settings_polaris10_a11, 795 ARRAY_SIZE(golden_settings_polaris10_a11)); 796 amdgpu_device_program_register_sequence(adev, 797 polaris10_golden_common_all, 798 ARRAY_SIZE(polaris10_golden_common_all)); 799 data = RREG32_SMC(ixCG_ACLK_CNTL); 800 data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK; 801 data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT; 802 WREG32_SMC(ixCG_ACLK_CNTL, data); 803 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) && 804 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) || 805 (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) || 806 (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) { 807 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD); 808 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0); 809 } 810 break; 811 case CHIP_CARRIZO: 812 amdgpu_device_program_register_sequence(adev, 813 cz_mgcg_cgcg_init, 814 ARRAY_SIZE(cz_mgcg_cgcg_init)); 815 amdgpu_device_program_register_sequence(adev, 816 cz_golden_settings_a11, 817 ARRAY_SIZE(cz_golden_settings_a11)); 818 amdgpu_device_program_register_sequence(adev, 819 cz_golden_common_all, 820 ARRAY_SIZE(cz_golden_common_all)); 821 break; 822 case CHIP_STONEY: 823 amdgpu_device_program_register_sequence(adev, 824 stoney_mgcg_cgcg_init, 825 ARRAY_SIZE(stoney_mgcg_cgcg_init)); 826 amdgpu_device_program_register_sequence(adev, 827 stoney_golden_settings_a11, 828 ARRAY_SIZE(stoney_golden_settings_a11)); 829 amdgpu_device_program_register_sequence(adev, 830 stoney_golden_common_all, 831 ARRAY_SIZE(stoney_golden_common_all)); 832 break; 833 default: 834 break; 835 } 836 } 837 838 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring) 839 { 840 struct amdgpu_device *adev = ring->adev; 841 uint32_t tmp = 0; 842 unsigned i; 843 int r; 844 845 WREG32(mmSCRATCH_REG0, 0xCAFEDEAD); 846 r = amdgpu_ring_alloc(ring, 3); 847 if (r) 848 return r; 849 850 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 851 amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START); 852 amdgpu_ring_write(ring, 0xDEADBEEF); 853 amdgpu_ring_commit(ring); 854 855 for (i = 0; i < adev->usec_timeout; i++) { 856 tmp = RREG32(mmSCRATCH_REG0); 857 if (tmp == 0xDEADBEEF) 858 break; 859 udelay(1); 860 } 861 862 if (i >= adev->usec_timeout) 863 r = -ETIMEDOUT; 864 865 return r; 866 } 867 868 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 869 { 870 struct amdgpu_device *adev = ring->adev; 871 struct amdgpu_job *job; 872 struct amdgpu_ib *ib; 873 struct dma_fence *f = NULL; 874 unsigned int index; 875 uint64_t gpu_addr; 876 uint32_t tmp; 877 long r; 878 879 r = amdgpu_wb_get(adev, &index); 880 if (r) 881 return r; 882 883 gpu_addr = adev->wb.gpu_addr + (index * 4); 884 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 885 886 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, 20, 887 AMDGPU_IB_POOL_DIRECT, 888 AMDGPU_KERNEL_JOB_ID_GFX_RING_TEST, 889 &job); 890 if (r) 891 goto err1; 892 893 ib = &job->ibs[0]; 894 ib->ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 895 ib->ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 896 ib->ptr[2] = lower_32_bits(gpu_addr); 897 ib->ptr[3] = upper_32_bits(gpu_addr); 898 ib->ptr[4] = 0xDEADBEEF; 899 ib->length_dw = 5; 900 901 r = amdgpu_job_submit_direct(job, ring, &f); 902 if (r) { 903 amdgpu_job_free(job); 904 goto err2; 905 } 906 907 r = dma_fence_wait_timeout(f, false, timeout); 908 if (r == 0) { 909 r = -ETIMEDOUT; 910 goto err2; 911 } else if (r < 0) { 912 goto err2; 913 } 914 915 tmp = adev->wb.wb[index]; 916 if (tmp == 0xDEADBEEF) 917 r = 0; 918 else 919 r = -EINVAL; 920 921 err2: 922 dma_fence_put(f); 923 err1: 924 amdgpu_wb_free(adev, index); 925 return r; 926 } 927 928 929 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev) 930 { 931 amdgpu_ucode_release(&adev->gfx.pfp_fw); 932 amdgpu_ucode_release(&adev->gfx.me_fw); 933 amdgpu_ucode_release(&adev->gfx.ce_fw); 934 amdgpu_ucode_release(&adev->gfx.rlc_fw); 935 amdgpu_ucode_release(&adev->gfx.mec_fw); 936 if ((adev->asic_type != CHIP_STONEY) && 937 (adev->asic_type != CHIP_TOPAZ)) 938 amdgpu_ucode_release(&adev->gfx.mec2_fw); 939 940 kfree(adev->gfx.rlc.register_list_format); 941 } 942 943 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev) 944 { 945 const char *chip_name; 946 int err; 947 struct amdgpu_firmware_info *info = NULL; 948 const struct common_firmware_header *header = NULL; 949 const struct gfx_firmware_header_v1_0 *cp_hdr; 950 const struct rlc_firmware_header_v2_0 *rlc_hdr; 951 unsigned int *tmp = NULL, i; 952 953 DRM_DEBUG("\n"); 954 955 switch (adev->asic_type) { 956 case CHIP_TOPAZ: 957 chip_name = "topaz"; 958 break; 959 case CHIP_TONGA: 960 chip_name = "tonga"; 961 break; 962 case CHIP_CARRIZO: 963 chip_name = "carrizo"; 964 break; 965 case CHIP_FIJI: 966 chip_name = "fiji"; 967 break; 968 case CHIP_STONEY: 969 chip_name = "stoney"; 970 break; 971 case CHIP_POLARIS10: 972 chip_name = "polaris10"; 973 break; 974 case CHIP_POLARIS11: 975 chip_name = "polaris11"; 976 break; 977 case CHIP_POLARIS12: 978 chip_name = "polaris12"; 979 break; 980 case CHIP_VEGAM: 981 chip_name = "vegam"; 982 break; 983 default: 984 dev_warn(adev->dev, "Unsupported asic_type 0x%08x\n", adev->asic_type); 985 return -EINVAL; 986 } 987 988 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 989 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 990 AMDGPU_UCODE_OPTIONAL, 991 "amdgpu/%s_pfp_2.bin", chip_name); 992 if (err == -ENODEV) { 993 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 994 AMDGPU_UCODE_REQUIRED, 995 "amdgpu/%s_pfp.bin", chip_name); 996 } 997 } else { 998 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 999 AMDGPU_UCODE_REQUIRED, 1000 "amdgpu/%s_pfp.bin", chip_name); 1001 } 1002 if (err) 1003 goto out; 1004 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1005 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1006 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1007 1008 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1009 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1010 AMDGPU_UCODE_OPTIONAL, 1011 "amdgpu/%s_me_2.bin", chip_name); 1012 if (err == -ENODEV) { 1013 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1014 AMDGPU_UCODE_REQUIRED, 1015 "amdgpu/%s_me.bin", chip_name); 1016 } 1017 } else { 1018 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1019 AMDGPU_UCODE_REQUIRED, 1020 "amdgpu/%s_me.bin", chip_name); 1021 } 1022 if (err) 1023 goto out; 1024 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1025 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1026 1027 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1028 1029 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1030 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1031 AMDGPU_UCODE_OPTIONAL, 1032 "amdgpu/%s_ce_2.bin", chip_name); 1033 if (err == -ENODEV) { 1034 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1035 AMDGPU_UCODE_REQUIRED, 1036 "amdgpu/%s_ce.bin", chip_name); 1037 } 1038 } else { 1039 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1040 AMDGPU_UCODE_REQUIRED, 1041 "amdgpu/%s_ce.bin", chip_name); 1042 } 1043 if (err) 1044 goto out; 1045 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1046 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1047 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1048 1049 /* 1050 * Support for MCBP/Virtualization in combination with chained IBs is 1051 * formal released on feature version #46 1052 */ 1053 if (adev->gfx.ce_feature_version >= 46 && 1054 adev->gfx.pfp_feature_version >= 46) { 1055 adev->virt.chained_ib_support = true; 1056 drm_info(adev_to_drm(adev), "Chained IB support enabled!\n"); 1057 } else 1058 adev->virt.chained_ib_support = false; 1059 1060 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1061 AMDGPU_UCODE_REQUIRED, 1062 "amdgpu/%s_rlc.bin", chip_name); 1063 if (err) 1064 goto out; 1065 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1066 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1067 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1068 1069 adev->gfx.rlc.save_and_restore_offset = 1070 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1071 adev->gfx.rlc.clear_state_descriptor_offset = 1072 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1073 adev->gfx.rlc.avail_scratch_ram_locations = 1074 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1075 adev->gfx.rlc.reg_restore_list_size = 1076 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1077 adev->gfx.rlc.reg_list_format_start = 1078 le32_to_cpu(rlc_hdr->reg_list_format_start); 1079 adev->gfx.rlc.reg_list_format_separate_start = 1080 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1081 adev->gfx.rlc.starting_offsets_start = 1082 le32_to_cpu(rlc_hdr->starting_offsets_start); 1083 adev->gfx.rlc.reg_list_format_size_bytes = 1084 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1085 adev->gfx.rlc.reg_list_size_bytes = 1086 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1087 1088 adev->gfx.rlc.register_list_format = 1089 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1090 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1091 1092 if (!adev->gfx.rlc.register_list_format) { 1093 err = -ENOMEM; 1094 goto out; 1095 } 1096 1097 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1098 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1099 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1100 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1101 1102 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1103 1104 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1105 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1106 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1107 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1108 1109 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1110 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1111 AMDGPU_UCODE_OPTIONAL, 1112 "amdgpu/%s_mec_2.bin", chip_name); 1113 if (err == -ENODEV) { 1114 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1115 AMDGPU_UCODE_REQUIRED, 1116 "amdgpu/%s_mec.bin", chip_name); 1117 } 1118 } else { 1119 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1120 AMDGPU_UCODE_REQUIRED, 1121 "amdgpu/%s_mec.bin", chip_name); 1122 } 1123 if (err) 1124 goto out; 1125 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1126 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1127 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1128 1129 if ((adev->asic_type != CHIP_STONEY) && 1130 (adev->asic_type != CHIP_TOPAZ)) { 1131 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1132 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1133 AMDGPU_UCODE_OPTIONAL, 1134 "amdgpu/%s_mec2_2.bin", chip_name); 1135 if (err == -ENODEV) { 1136 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1137 AMDGPU_UCODE_REQUIRED, 1138 "amdgpu/%s_mec2.bin", chip_name); 1139 } 1140 } else { 1141 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1142 AMDGPU_UCODE_REQUIRED, 1143 "amdgpu/%s_mec2.bin", chip_name); 1144 } 1145 if (!err) { 1146 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1147 adev->gfx.mec2_fw->data; 1148 adev->gfx.mec2_fw_version = 1149 le32_to_cpu(cp_hdr->header.ucode_version); 1150 adev->gfx.mec2_feature_version = 1151 le32_to_cpu(cp_hdr->ucode_feature_version); 1152 } else { 1153 err = 0; 1154 adev->gfx.mec2_fw = NULL; 1155 } 1156 } 1157 1158 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1159 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1160 info->fw = adev->gfx.pfp_fw; 1161 header = (const struct common_firmware_header *)info->fw->data; 1162 adev->firmware.fw_size += 1163 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1164 1165 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1166 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1167 info->fw = adev->gfx.me_fw; 1168 header = (const struct common_firmware_header *)info->fw->data; 1169 adev->firmware.fw_size += 1170 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1171 1172 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1173 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1174 info->fw = adev->gfx.ce_fw; 1175 header = (const struct common_firmware_header *)info->fw->data; 1176 adev->firmware.fw_size += 1177 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1178 1179 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1180 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1181 info->fw = adev->gfx.rlc_fw; 1182 header = (const struct common_firmware_header *)info->fw->data; 1183 adev->firmware.fw_size += 1184 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1185 1186 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1187 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1188 info->fw = adev->gfx.mec_fw; 1189 header = (const struct common_firmware_header *)info->fw->data; 1190 adev->firmware.fw_size += 1191 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1192 1193 /* we need account JT in */ 1194 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1195 adev->firmware.fw_size += 1196 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE); 1197 1198 if (amdgpu_sriov_vf(adev)) { 1199 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE]; 1200 info->ucode_id = AMDGPU_UCODE_ID_STORAGE; 1201 info->fw = adev->gfx.mec_fw; 1202 adev->firmware.fw_size += 1203 ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE); 1204 } 1205 1206 if (adev->gfx.mec2_fw) { 1207 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1208 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1209 info->fw = adev->gfx.mec2_fw; 1210 header = (const struct common_firmware_header *)info->fw->data; 1211 adev->firmware.fw_size += 1212 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1213 } 1214 1215 out: 1216 if (err) { 1217 dev_err(adev->dev, "gfx8: Failed to load firmware %s gfx firmware\n", chip_name); 1218 gfx_v8_0_free_microcode(adev); 1219 } 1220 return err; 1221 } 1222 1223 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 1224 { 1225 u32 count = 0; 1226 1227 if (adev->gfx.rlc.cs_data == NULL) 1228 return; 1229 if (buffer == NULL) 1230 return; 1231 1232 count = amdgpu_gfx_csb_preamble_start(buffer); 1233 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 1234 1235 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 1236 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 1237 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config); 1238 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1); 1239 1240 amdgpu_gfx_csb_preamble_end(buffer, count); 1241 } 1242 1243 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev) 1244 { 1245 if (adev->asic_type == CHIP_CARRIZO) 1246 return 5; 1247 else 1248 return 4; 1249 } 1250 1251 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev) 1252 { 1253 const struct cs_section_def *cs_data; 1254 int r; 1255 1256 adev->gfx.rlc.cs_data = vi_cs_data; 1257 1258 cs_data = adev->gfx.rlc.cs_data; 1259 1260 if (cs_data) { 1261 /* init clear state block */ 1262 r = amdgpu_gfx_rlc_init_csb(adev); 1263 if (r) 1264 return r; 1265 } 1266 1267 if ((adev->asic_type == CHIP_CARRIZO) || 1268 (adev->asic_type == CHIP_STONEY)) { 1269 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1270 r = amdgpu_gfx_rlc_init_cpt(adev); 1271 if (r) 1272 return r; 1273 } 1274 1275 /* init spm vmid with 0xf */ 1276 if (adev->gfx.rlc.funcs->update_spm_vmid) 1277 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf); 1278 1279 return 0; 1280 } 1281 1282 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev) 1283 { 1284 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1285 } 1286 1287 static int gfx_v8_0_mec_init(struct amdgpu_device *adev) 1288 { 1289 int r; 1290 u32 *hpd; 1291 size_t mec_hpd_size; 1292 1293 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1294 1295 /* take ownership of the relevant compute queues */ 1296 amdgpu_gfx_compute_queue_acquire(adev); 1297 1298 mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE; 1299 if (mec_hpd_size) { 1300 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1301 AMDGPU_GEM_DOMAIN_VRAM | 1302 AMDGPU_GEM_DOMAIN_GTT, 1303 &adev->gfx.mec.hpd_eop_obj, 1304 &adev->gfx.mec.hpd_eop_gpu_addr, 1305 (void **)&hpd); 1306 if (r) { 1307 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1308 return r; 1309 } 1310 1311 memset(hpd, 0, mec_hpd_size); 1312 1313 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1314 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1315 } 1316 1317 return 0; 1318 } 1319 1320 static const u32 vgpr_init_compute_shader[] = 1321 { 1322 0x7e000209, 0x7e020208, 1323 0x7e040207, 0x7e060206, 1324 0x7e080205, 0x7e0a0204, 1325 0x7e0c0203, 0x7e0e0202, 1326 0x7e100201, 0x7e120200, 1327 0x7e140209, 0x7e160208, 1328 0x7e180207, 0x7e1a0206, 1329 0x7e1c0205, 0x7e1e0204, 1330 0x7e200203, 0x7e220202, 1331 0x7e240201, 0x7e260200, 1332 0x7e280209, 0x7e2a0208, 1333 0x7e2c0207, 0x7e2e0206, 1334 0x7e300205, 0x7e320204, 1335 0x7e340203, 0x7e360202, 1336 0x7e380201, 0x7e3a0200, 1337 0x7e3c0209, 0x7e3e0208, 1338 0x7e400207, 0x7e420206, 1339 0x7e440205, 0x7e460204, 1340 0x7e480203, 0x7e4a0202, 1341 0x7e4c0201, 0x7e4e0200, 1342 0x7e500209, 0x7e520208, 1343 0x7e540207, 0x7e560206, 1344 0x7e580205, 0x7e5a0204, 1345 0x7e5c0203, 0x7e5e0202, 1346 0x7e600201, 0x7e620200, 1347 0x7e640209, 0x7e660208, 1348 0x7e680207, 0x7e6a0206, 1349 0x7e6c0205, 0x7e6e0204, 1350 0x7e700203, 0x7e720202, 1351 0x7e740201, 0x7e760200, 1352 0x7e780209, 0x7e7a0208, 1353 0x7e7c0207, 0x7e7e0206, 1354 0xbf8a0000, 0xbf810000, 1355 }; 1356 1357 static const u32 sgpr_init_compute_shader[] = 1358 { 1359 0xbe8a0100, 0xbe8c0102, 1360 0xbe8e0104, 0xbe900106, 1361 0xbe920108, 0xbe940100, 1362 0xbe960102, 0xbe980104, 1363 0xbe9a0106, 0xbe9c0108, 1364 0xbe9e0100, 0xbea00102, 1365 0xbea20104, 0xbea40106, 1366 0xbea60108, 0xbea80100, 1367 0xbeaa0102, 0xbeac0104, 1368 0xbeae0106, 0xbeb00108, 1369 0xbeb20100, 0xbeb40102, 1370 0xbeb60104, 0xbeb80106, 1371 0xbeba0108, 0xbebc0100, 1372 0xbebe0102, 0xbec00104, 1373 0xbec20106, 0xbec40108, 1374 0xbec60100, 0xbec80102, 1375 0xbee60004, 0xbee70005, 1376 0xbeea0006, 0xbeeb0007, 1377 0xbee80008, 0xbee90009, 1378 0xbefc0000, 0xbf8a0000, 1379 0xbf810000, 0x00000000, 1380 }; 1381 1382 static const u32 vgpr_init_regs[] = 1383 { 1384 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff, 1385 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1386 mmCOMPUTE_NUM_THREAD_X, 256*4, 1387 mmCOMPUTE_NUM_THREAD_Y, 1, 1388 mmCOMPUTE_NUM_THREAD_Z, 1, 1389 mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */ 1390 mmCOMPUTE_PGM_RSRC2, 20, 1391 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1392 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1393 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1394 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1395 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1396 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1397 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1398 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1399 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1400 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1401 }; 1402 1403 static const u32 sgpr1_init_regs[] = 1404 { 1405 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f, 1406 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1407 mmCOMPUTE_NUM_THREAD_X, 256*5, 1408 mmCOMPUTE_NUM_THREAD_Y, 1, 1409 mmCOMPUTE_NUM_THREAD_Z, 1, 1410 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1411 mmCOMPUTE_PGM_RSRC2, 20, 1412 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1413 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1414 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1415 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1416 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1417 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1418 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1419 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1420 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1421 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1422 }; 1423 1424 static const u32 sgpr2_init_regs[] = 1425 { 1426 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0, 1427 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, 1428 mmCOMPUTE_NUM_THREAD_X, 256*5, 1429 mmCOMPUTE_NUM_THREAD_Y, 1, 1430 mmCOMPUTE_NUM_THREAD_Z, 1, 1431 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1432 mmCOMPUTE_PGM_RSRC2, 20, 1433 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1434 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1435 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1436 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1437 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1438 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1439 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1440 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1441 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1442 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1443 }; 1444 1445 static const u32 sec_ded_counter_registers[] = 1446 { 1447 mmCPC_EDC_ATC_CNT, 1448 mmCPC_EDC_SCRATCH_CNT, 1449 mmCPC_EDC_UCODE_CNT, 1450 mmCPF_EDC_ATC_CNT, 1451 mmCPF_EDC_ROQ_CNT, 1452 mmCPF_EDC_TAG_CNT, 1453 mmCPG_EDC_ATC_CNT, 1454 mmCPG_EDC_DMA_CNT, 1455 mmCPG_EDC_TAG_CNT, 1456 mmDC_EDC_CSINVOC_CNT, 1457 mmDC_EDC_RESTORE_CNT, 1458 mmDC_EDC_STATE_CNT, 1459 mmGDS_EDC_CNT, 1460 mmGDS_EDC_GRBM_CNT, 1461 mmGDS_EDC_OA_DED, 1462 mmSPI_EDC_CNT, 1463 mmSQC_ATC_EDC_GATCL1_CNT, 1464 mmSQC_EDC_CNT, 1465 mmSQ_EDC_DED_CNT, 1466 mmSQ_EDC_INFO, 1467 mmSQ_EDC_SEC_CNT, 1468 mmTCC_EDC_CNT, 1469 mmTCP_ATC_EDC_GATCL1_CNT, 1470 mmTCP_EDC_CNT, 1471 mmTD_EDC_CNT 1472 }; 1473 1474 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 1475 { 1476 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 1477 struct amdgpu_job *job; 1478 struct amdgpu_ib *ib; 1479 struct dma_fence *f = NULL; 1480 int r, i; 1481 u32 tmp; 1482 unsigned total_size, vgpr_offset, sgpr_offset; 1483 u64 gpu_addr; 1484 1485 /* only supported on CZ */ 1486 if (adev->asic_type != CHIP_CARRIZO) 1487 return 0; 1488 1489 /* bail if the compute ring is not ready */ 1490 if (!ring->sched.ready) 1491 return -EBUSY; 1492 1493 if (amdgpu_in_reset(adev)) { 1494 /* Set preempt condition to execute IB */ 1495 amdgpu_ring_set_preempt_cond_exec(ring, true); 1496 /* Flush HDP cache so the GPU can see the updated COND_EXEC value */ 1497 amdgpu_device_flush_hdp(adev, NULL); 1498 } 1499 1500 tmp = RREG32(mmGB_EDC_MODE); 1501 WREG32(mmGB_EDC_MODE, 0); 1502 1503 total_size = 1504 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1505 total_size += 1506 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1507 total_size += 1508 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1509 total_size = ALIGN(total_size, 256); 1510 vgpr_offset = total_size; 1511 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256); 1512 sgpr_offset = total_size; 1513 total_size += sizeof(sgpr_init_compute_shader); 1514 1515 /* allocate an indirect buffer to put the commands in */ 1516 r = amdgpu_job_alloc_with_ib(ring->adev, NULL, NULL, total_size, 1517 AMDGPU_IB_POOL_DIRECT, 1518 AMDGPU_KERNEL_JOB_ID_RUN_SHADER, 1519 &job); 1520 if (r) { 1521 drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r); 1522 return r; 1523 } 1524 ib = &job->ibs[0]; 1525 1526 /* load the compute shaders */ 1527 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++) 1528 ib->ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i]; 1529 1530 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 1531 ib->ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 1532 1533 /* init the ib length to 0 */ 1534 ib->length_dw = 0; 1535 1536 /* VGPR */ 1537 /* write the register state for the compute dispatch */ 1538 for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) { 1539 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1540 ib->ptr[ib->length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START; 1541 ib->ptr[ib->length_dw++] = vgpr_init_regs[i + 1]; 1542 } 1543 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1544 gpu_addr = (ib->gpu_addr + (u64)vgpr_offset) >> 8; 1545 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1546 ib->ptr[ib->length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1547 ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr); 1548 ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr); 1549 1550 /* write dispatch packet */ 1551 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1552 ib->ptr[ib->length_dw++] = 8; /* x */ 1553 ib->ptr[ib->length_dw++] = 1; /* y */ 1554 ib->ptr[ib->length_dw++] = 1; /* z */ 1555 ib->ptr[ib->length_dw++] = 1556 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1557 1558 /* write CS partial flush packet */ 1559 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1560 ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1561 1562 /* SGPR1 */ 1563 /* write the register state for the compute dispatch */ 1564 for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) { 1565 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1566 ib->ptr[ib->length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START; 1567 ib->ptr[ib->length_dw++] = sgpr1_init_regs[i + 1]; 1568 } 1569 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1570 gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8; 1571 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1572 ib->ptr[ib->length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1573 ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr); 1574 ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr); 1575 1576 /* write dispatch packet */ 1577 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1578 ib->ptr[ib->length_dw++] = 8; /* x */ 1579 ib->ptr[ib->length_dw++] = 1; /* y */ 1580 ib->ptr[ib->length_dw++] = 1; /* z */ 1581 ib->ptr[ib->length_dw++] = 1582 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1583 1584 /* write CS partial flush packet */ 1585 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1586 ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1587 1588 /* SGPR2 */ 1589 /* write the register state for the compute dispatch */ 1590 for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) { 1591 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1592 ib->ptr[ib->length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START; 1593 ib->ptr[ib->length_dw++] = sgpr2_init_regs[i + 1]; 1594 } 1595 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1596 gpu_addr = (ib->gpu_addr + (u64)sgpr_offset) >> 8; 1597 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1598 ib->ptr[ib->length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1599 ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr); 1600 ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr); 1601 1602 /* write dispatch packet */ 1603 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1604 ib->ptr[ib->length_dw++] = 8; /* x */ 1605 ib->ptr[ib->length_dw++] = 1; /* y */ 1606 ib->ptr[ib->length_dw++] = 1; /* z */ 1607 ib->ptr[ib->length_dw++] = 1608 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1609 1610 /* write CS partial flush packet */ 1611 ib->ptr[ib->length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1612 ib->ptr[ib->length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1613 1614 /* schedule the ib on the ring */ 1615 r = amdgpu_job_submit_direct(job, ring, &f); 1616 if (r) { 1617 drm_err(adev_to_drm(adev), "ib submit failed (%d).\n", r); 1618 amdgpu_job_free(job); 1619 goto fail; 1620 } 1621 1622 /* wait for the GPU to finish processing the IB */ 1623 r = dma_fence_wait(f, false); 1624 if (r) { 1625 drm_err(adev_to_drm(adev), "fence wait failed (%d).\n", r); 1626 goto fail; 1627 } 1628 1629 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2); 1630 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1); 1631 WREG32(mmGB_EDC_MODE, tmp); 1632 1633 tmp = RREG32(mmCC_GC_EDC_CONFIG); 1634 tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1; 1635 WREG32(mmCC_GC_EDC_CONFIG, tmp); 1636 1637 1638 /* read back registers to clear the counters */ 1639 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) 1640 RREG32(sec_ded_counter_registers[i]); 1641 1642 fail: 1643 dma_fence_put(f); 1644 1645 return r; 1646 } 1647 1648 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev) 1649 { 1650 u32 gb_addr_config; 1651 u32 mc_arb_ramcfg; 1652 u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map; 1653 u32 tmp; 1654 int ret; 1655 1656 switch (adev->asic_type) { 1657 case CHIP_TOPAZ: 1658 adev->gfx.config.max_shader_engines = 1; 1659 adev->gfx.config.max_tile_pipes = 2; 1660 adev->gfx.config.max_cu_per_sh = 6; 1661 adev->gfx.config.max_sh_per_se = 1; 1662 adev->gfx.config.max_backends_per_se = 2; 1663 adev->gfx.config.max_texture_channel_caches = 2; 1664 adev->gfx.config.max_gprs = 256; 1665 adev->gfx.config.max_gs_threads = 32; 1666 adev->gfx.config.max_hw_contexts = 8; 1667 1668 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1669 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1670 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1671 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1672 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN; 1673 break; 1674 case CHIP_FIJI: 1675 adev->gfx.config.max_shader_engines = 4; 1676 adev->gfx.config.max_tile_pipes = 16; 1677 adev->gfx.config.max_cu_per_sh = 16; 1678 adev->gfx.config.max_sh_per_se = 1; 1679 adev->gfx.config.max_backends_per_se = 4; 1680 adev->gfx.config.max_texture_channel_caches = 16; 1681 adev->gfx.config.max_gprs = 256; 1682 adev->gfx.config.max_gs_threads = 32; 1683 adev->gfx.config.max_hw_contexts = 8; 1684 1685 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1686 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1687 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1688 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1689 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1690 break; 1691 case CHIP_POLARIS11: 1692 case CHIP_POLARIS12: 1693 ret = amdgpu_atombios_get_gfx_info(adev); 1694 if (ret) 1695 return ret; 1696 adev->gfx.config.max_gprs = 256; 1697 adev->gfx.config.max_gs_threads = 32; 1698 adev->gfx.config.max_hw_contexts = 8; 1699 1700 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1701 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1702 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1703 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1704 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN; 1705 break; 1706 case CHIP_POLARIS10: 1707 case CHIP_VEGAM: 1708 ret = amdgpu_atombios_get_gfx_info(adev); 1709 if (ret) 1710 return ret; 1711 adev->gfx.config.max_gprs = 256; 1712 adev->gfx.config.max_gs_threads = 32; 1713 adev->gfx.config.max_hw_contexts = 8; 1714 1715 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1716 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1717 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1718 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1719 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1720 break; 1721 case CHIP_TONGA: 1722 adev->gfx.config.max_shader_engines = 4; 1723 adev->gfx.config.max_tile_pipes = 8; 1724 adev->gfx.config.max_cu_per_sh = 8; 1725 adev->gfx.config.max_sh_per_se = 1; 1726 adev->gfx.config.max_backends_per_se = 2; 1727 adev->gfx.config.max_texture_channel_caches = 8; 1728 adev->gfx.config.max_gprs = 256; 1729 adev->gfx.config.max_gs_threads = 32; 1730 adev->gfx.config.max_hw_contexts = 8; 1731 1732 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1733 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1734 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1735 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1736 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1737 break; 1738 case CHIP_CARRIZO: 1739 adev->gfx.config.max_shader_engines = 1; 1740 adev->gfx.config.max_tile_pipes = 2; 1741 adev->gfx.config.max_sh_per_se = 1; 1742 adev->gfx.config.max_backends_per_se = 2; 1743 adev->gfx.config.max_cu_per_sh = 8; 1744 adev->gfx.config.max_texture_channel_caches = 2; 1745 adev->gfx.config.max_gprs = 256; 1746 adev->gfx.config.max_gs_threads = 32; 1747 adev->gfx.config.max_hw_contexts = 8; 1748 1749 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1750 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1751 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1752 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1753 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1754 break; 1755 case CHIP_STONEY: 1756 adev->gfx.config.max_shader_engines = 1; 1757 adev->gfx.config.max_tile_pipes = 2; 1758 adev->gfx.config.max_sh_per_se = 1; 1759 adev->gfx.config.max_backends_per_se = 1; 1760 adev->gfx.config.max_cu_per_sh = 3; 1761 adev->gfx.config.max_texture_channel_caches = 2; 1762 adev->gfx.config.max_gprs = 256; 1763 adev->gfx.config.max_gs_threads = 16; 1764 adev->gfx.config.max_hw_contexts = 8; 1765 1766 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1767 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1768 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1769 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1770 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1771 break; 1772 default: 1773 adev->gfx.config.max_shader_engines = 2; 1774 adev->gfx.config.max_tile_pipes = 4; 1775 adev->gfx.config.max_cu_per_sh = 2; 1776 adev->gfx.config.max_sh_per_se = 1; 1777 adev->gfx.config.max_backends_per_se = 2; 1778 adev->gfx.config.max_texture_channel_caches = 4; 1779 adev->gfx.config.max_gprs = 256; 1780 adev->gfx.config.max_gs_threads = 32; 1781 adev->gfx.config.max_hw_contexts = 8; 1782 1783 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1784 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1785 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1786 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1787 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1788 break; 1789 } 1790 1791 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG); 1792 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg; 1793 1794 adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg, 1795 MC_ARB_RAMCFG, NOOFBANK); 1796 adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg, 1797 MC_ARB_RAMCFG, NOOFRANKS); 1798 1799 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; 1800 adev->gfx.config.mem_max_burst_length_bytes = 256; 1801 if (adev->flags & AMD_IS_APU) { 1802 /* Get memory bank mapping mode. */ 1803 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING); 1804 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1805 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1806 1807 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING); 1808 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1809 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1810 1811 /* Validate settings in case only one DIMM installed. */ 1812 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12)) 1813 dimm00_addr_map = 0; 1814 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12)) 1815 dimm01_addr_map = 0; 1816 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12)) 1817 dimm10_addr_map = 0; 1818 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12)) 1819 dimm11_addr_map = 0; 1820 1821 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */ 1822 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */ 1823 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11)) 1824 adev->gfx.config.mem_row_size_in_kb = 2; 1825 else 1826 adev->gfx.config.mem_row_size_in_kb = 1; 1827 } else { 1828 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS); 1829 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 1830 if (adev->gfx.config.mem_row_size_in_kb > 4) 1831 adev->gfx.config.mem_row_size_in_kb = 4; 1832 } 1833 1834 adev->gfx.config.shader_engine_tile_size = 32; 1835 adev->gfx.config.num_gpus = 1; 1836 adev->gfx.config.multi_gpu_tile_size = 64; 1837 1838 /* fix up row size */ 1839 switch (adev->gfx.config.mem_row_size_in_kb) { 1840 case 1: 1841 default: 1842 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0); 1843 break; 1844 case 2: 1845 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1); 1846 break; 1847 case 4: 1848 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2); 1849 break; 1850 } 1851 adev->gfx.config.gb_addr_config = gb_addr_config; 1852 1853 return 0; 1854 } 1855 1856 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1857 int mec, int pipe, int queue) 1858 { 1859 int r; 1860 unsigned irq_type; 1861 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 1862 unsigned int hw_prio; 1863 1864 ring = &adev->gfx.compute_ring[ring_id]; 1865 1866 /* mec0 is me1 */ 1867 ring->me = mec + 1; 1868 ring->pipe = pipe; 1869 ring->queue = queue; 1870 1871 ring->ring_obj = NULL; 1872 ring->use_doorbell = true; 1873 ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id; 1874 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1875 + (ring_id * GFX8_MEC_HPD_SIZE); 1876 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1877 1878 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1879 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1880 + ring->pipe; 1881 1882 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1883 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT; 1884 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1885 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1886 hw_prio, NULL); 1887 if (r) 1888 return r; 1889 1890 1891 return 0; 1892 } 1893 1894 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work); 1895 1896 static int gfx_v8_0_sw_init(struct amdgpu_ip_block *ip_block) 1897 { 1898 int i, j, k, r, ring_id; 1899 int xcc_id = 0; 1900 struct amdgpu_ring *ring; 1901 struct amdgpu_device *adev = ip_block->adev; 1902 1903 switch (adev->asic_type) { 1904 case CHIP_TONGA: 1905 case CHIP_CARRIZO: 1906 case CHIP_FIJI: 1907 case CHIP_POLARIS10: 1908 case CHIP_POLARIS11: 1909 case CHIP_POLARIS12: 1910 case CHIP_VEGAM: 1911 adev->gfx.mec.num_mec = 2; 1912 break; 1913 case CHIP_TOPAZ: 1914 case CHIP_STONEY: 1915 default: 1916 adev->gfx.mec.num_mec = 1; 1917 break; 1918 } 1919 1920 adev->gfx.mec.num_pipe_per_mec = 4; 1921 adev->gfx.mec.num_queue_per_pipe = 8; 1922 1923 /* EOP Event */ 1924 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq); 1925 if (r) 1926 return r; 1927 1928 /* Privileged reg */ 1929 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT, 1930 &adev->gfx.priv_reg_irq); 1931 if (r) 1932 return r; 1933 1934 /* Privileged inst */ 1935 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT, 1936 &adev->gfx.priv_inst_irq); 1937 if (r) 1938 return r; 1939 1940 /* Add CP EDC/ECC irq */ 1941 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR, 1942 &adev->gfx.cp_ecc_error_irq); 1943 if (r) 1944 return r; 1945 1946 /* SQ interrupts. */ 1947 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG, 1948 &adev->gfx.sq_irq); 1949 if (r) { 1950 DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r); 1951 return r; 1952 } 1953 1954 INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func); 1955 1956 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1957 1958 r = gfx_v8_0_init_microcode(adev); 1959 if (r) { 1960 DRM_ERROR("Failed to load gfx firmware!\n"); 1961 return r; 1962 } 1963 1964 r = adev->gfx.rlc.funcs->init(adev); 1965 if (r) { 1966 DRM_ERROR("Failed to init rlc BOs!\n"); 1967 return r; 1968 } 1969 1970 r = gfx_v8_0_mec_init(adev); 1971 if (r) { 1972 DRM_ERROR("Failed to init MEC BOs!\n"); 1973 return r; 1974 } 1975 1976 /* set up the gfx ring */ 1977 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 1978 ring = &adev->gfx.gfx_ring[i]; 1979 ring->ring_obj = NULL; 1980 sprintf(ring->name, "gfx"); 1981 /* no gfx doorbells on iceland */ 1982 if (adev->asic_type != CHIP_TOPAZ) { 1983 ring->use_doorbell = true; 1984 ring->doorbell_index = adev->doorbell_index.gfx_ring0; 1985 } 1986 1987 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 1988 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 1989 AMDGPU_RING_PRIO_DEFAULT, NULL); 1990 if (r) 1991 return r; 1992 } 1993 1994 1995 /* set up the compute queues - allocate horizontally across pipes */ 1996 ring_id = 0; 1997 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1998 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1999 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2000 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 2001 k, j)) 2002 continue; 2003 2004 r = gfx_v8_0_compute_ring_init(adev, 2005 ring_id, 2006 i, k, j); 2007 if (r) 2008 return r; 2009 2010 ring_id++; 2011 } 2012 } 2013 } 2014 2015 r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE, 0); 2016 if (r) { 2017 DRM_ERROR("Failed to init KIQ BOs!\n"); 2018 return r; 2019 } 2020 2021 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 2022 if (r) 2023 return r; 2024 2025 /* create MQD for all compute queues as well as KIQ for SRIOV case */ 2026 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation), 0); 2027 if (r) 2028 return r; 2029 2030 adev->gfx.ce_ram_size = 0x8000; 2031 2032 r = gfx_v8_0_gpu_early_init(adev); 2033 if (r) 2034 return r; 2035 2036 adev->gfx.gfx_supported_reset = 2037 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 2038 adev->gfx.compute_supported_reset = 2039 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 2040 2041 if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_ip_block_soft_reset) { 2042 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; 2043 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; 2044 } 2045 2046 return 0; 2047 } 2048 2049 static int gfx_v8_0_sw_fini(struct amdgpu_ip_block *ip_block) 2050 { 2051 struct amdgpu_device *adev = ip_block->adev; 2052 int i; 2053 2054 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2055 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2056 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2057 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2058 2059 amdgpu_gfx_mqd_sw_fini(adev, 0); 2060 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 2061 amdgpu_gfx_kiq_fini(adev, 0); 2062 2063 gfx_v8_0_mec_fini(adev); 2064 amdgpu_gfx_rlc_fini(adev); 2065 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2066 &adev->gfx.rlc.clear_state_gpu_addr, 2067 (void **)&adev->gfx.rlc.cs_ptr); 2068 if ((adev->asic_type == CHIP_CARRIZO) || 2069 (adev->asic_type == CHIP_STONEY)) { 2070 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2071 &adev->gfx.rlc.cp_table_gpu_addr, 2072 (void **)&adev->gfx.rlc.cp_table_ptr); 2073 } 2074 gfx_v8_0_free_microcode(adev); 2075 2076 return 0; 2077 } 2078 2079 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev) 2080 { 2081 uint32_t *modearray, *mod2array; 2082 const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array); 2083 const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 2084 u32 reg_offset; 2085 2086 modearray = adev->gfx.config.tile_mode_array; 2087 mod2array = adev->gfx.config.macrotile_mode_array; 2088 2089 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2090 modearray[reg_offset] = 0; 2091 2092 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2093 mod2array[reg_offset] = 0; 2094 2095 switch (adev->asic_type) { 2096 case CHIP_TOPAZ: 2097 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2098 PIPE_CONFIG(ADDR_SURF_P2) | 2099 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2100 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2101 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2102 PIPE_CONFIG(ADDR_SURF_P2) | 2103 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2104 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2105 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2106 PIPE_CONFIG(ADDR_SURF_P2) | 2107 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2108 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2109 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2110 PIPE_CONFIG(ADDR_SURF_P2) | 2111 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2112 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2113 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2114 PIPE_CONFIG(ADDR_SURF_P2) | 2115 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2116 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2117 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2118 PIPE_CONFIG(ADDR_SURF_P2) | 2119 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2120 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2121 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2122 PIPE_CONFIG(ADDR_SURF_P2) | 2123 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2124 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2125 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2126 PIPE_CONFIG(ADDR_SURF_P2)); 2127 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2128 PIPE_CONFIG(ADDR_SURF_P2) | 2129 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2130 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2131 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2132 PIPE_CONFIG(ADDR_SURF_P2) | 2133 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2134 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2135 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2136 PIPE_CONFIG(ADDR_SURF_P2) | 2137 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2138 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2139 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2140 PIPE_CONFIG(ADDR_SURF_P2) | 2141 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2142 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2143 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2144 PIPE_CONFIG(ADDR_SURF_P2) | 2145 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2146 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2147 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2148 PIPE_CONFIG(ADDR_SURF_P2) | 2149 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2150 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2151 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2152 PIPE_CONFIG(ADDR_SURF_P2) | 2153 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2154 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2155 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2156 PIPE_CONFIG(ADDR_SURF_P2) | 2157 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2158 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2159 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2160 PIPE_CONFIG(ADDR_SURF_P2) | 2161 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2162 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2163 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2164 PIPE_CONFIG(ADDR_SURF_P2) | 2165 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2166 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2167 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2168 PIPE_CONFIG(ADDR_SURF_P2) | 2169 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2170 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2171 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2172 PIPE_CONFIG(ADDR_SURF_P2) | 2173 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2174 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2175 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2176 PIPE_CONFIG(ADDR_SURF_P2) | 2177 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2178 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2179 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2180 PIPE_CONFIG(ADDR_SURF_P2) | 2181 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2182 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2183 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2184 PIPE_CONFIG(ADDR_SURF_P2) | 2185 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2186 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2187 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2188 PIPE_CONFIG(ADDR_SURF_P2) | 2189 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2190 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2191 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2192 PIPE_CONFIG(ADDR_SURF_P2) | 2193 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2194 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2195 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2196 PIPE_CONFIG(ADDR_SURF_P2) | 2197 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2198 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2199 2200 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2201 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2202 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2203 NUM_BANKS(ADDR_SURF_8_BANK)); 2204 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2205 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2206 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2207 NUM_BANKS(ADDR_SURF_8_BANK)); 2208 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2209 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2210 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2211 NUM_BANKS(ADDR_SURF_8_BANK)); 2212 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2213 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2214 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2215 NUM_BANKS(ADDR_SURF_8_BANK)); 2216 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2217 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2218 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2219 NUM_BANKS(ADDR_SURF_8_BANK)); 2220 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2221 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2222 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2223 NUM_BANKS(ADDR_SURF_8_BANK)); 2224 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2225 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2226 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2227 NUM_BANKS(ADDR_SURF_8_BANK)); 2228 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2229 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2230 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2231 NUM_BANKS(ADDR_SURF_16_BANK)); 2232 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2233 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2234 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2235 NUM_BANKS(ADDR_SURF_16_BANK)); 2236 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2237 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2238 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2239 NUM_BANKS(ADDR_SURF_16_BANK)); 2240 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2241 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2242 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2243 NUM_BANKS(ADDR_SURF_16_BANK)); 2244 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2245 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2246 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2247 NUM_BANKS(ADDR_SURF_16_BANK)); 2248 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2249 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2250 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2251 NUM_BANKS(ADDR_SURF_16_BANK)); 2252 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2253 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2254 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2255 NUM_BANKS(ADDR_SURF_8_BANK)); 2256 2257 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2258 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 2259 reg_offset != 23) 2260 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2261 2262 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2263 if (reg_offset != 7) 2264 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2265 2266 break; 2267 case CHIP_FIJI: 2268 case CHIP_VEGAM: 2269 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2270 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2271 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2272 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2273 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2274 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2275 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2276 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2277 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2278 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2279 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2280 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2281 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2282 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2283 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2284 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2285 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2286 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2287 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2288 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2289 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2290 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2291 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2292 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2293 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2294 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2295 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2296 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2297 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2298 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2299 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2300 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2301 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2302 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2303 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2304 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2305 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2306 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2307 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2308 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2309 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2310 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2311 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2312 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2313 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2314 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2315 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2316 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2317 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2318 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2319 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2320 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2321 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2322 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2323 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2324 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2325 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2326 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2327 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2328 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2329 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2330 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2331 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2332 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2333 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2334 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2335 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2336 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2337 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2338 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2339 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2340 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2341 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2342 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2343 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2344 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2345 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2346 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2347 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2348 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2349 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2350 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2351 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2352 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2353 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2354 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2355 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2356 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2357 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2358 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2359 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2360 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2361 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2362 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2363 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2364 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2365 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2366 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2367 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2368 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2369 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2370 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2371 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2372 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2373 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2374 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2375 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2376 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2377 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2378 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2379 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2380 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2381 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2382 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2383 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2384 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2385 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2386 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2387 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2388 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2389 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2390 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2391 2392 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2393 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2394 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2395 NUM_BANKS(ADDR_SURF_8_BANK)); 2396 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2399 NUM_BANKS(ADDR_SURF_8_BANK)); 2400 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2401 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2402 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2403 NUM_BANKS(ADDR_SURF_8_BANK)); 2404 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2405 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2406 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2407 NUM_BANKS(ADDR_SURF_8_BANK)); 2408 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2409 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2410 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2411 NUM_BANKS(ADDR_SURF_8_BANK)); 2412 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2413 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2414 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2415 NUM_BANKS(ADDR_SURF_8_BANK)); 2416 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2417 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2418 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2419 NUM_BANKS(ADDR_SURF_8_BANK)); 2420 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2421 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2422 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2423 NUM_BANKS(ADDR_SURF_8_BANK)); 2424 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2425 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2426 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2427 NUM_BANKS(ADDR_SURF_8_BANK)); 2428 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2429 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2430 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2431 NUM_BANKS(ADDR_SURF_8_BANK)); 2432 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2433 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2434 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2435 NUM_BANKS(ADDR_SURF_8_BANK)); 2436 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2437 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2438 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2439 NUM_BANKS(ADDR_SURF_8_BANK)); 2440 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2441 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2442 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2443 NUM_BANKS(ADDR_SURF_8_BANK)); 2444 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2445 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2446 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2447 NUM_BANKS(ADDR_SURF_4_BANK)); 2448 2449 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2450 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2451 2452 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2453 if (reg_offset != 7) 2454 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2455 2456 break; 2457 case CHIP_TONGA: 2458 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2459 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2460 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2461 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2462 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2463 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2465 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2466 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2467 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2468 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2469 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2470 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2471 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2472 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2473 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2474 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2475 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2476 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2477 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2478 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2479 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2480 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2481 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2482 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2483 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2484 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2485 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2486 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2487 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2488 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2489 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2490 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2491 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2492 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2493 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2494 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2495 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2496 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2497 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2498 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2499 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2500 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2501 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2502 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2503 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2504 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2505 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2506 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2507 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2508 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2509 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2510 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2511 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2512 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2513 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2514 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2515 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2516 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2517 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2518 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2519 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2520 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2521 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2522 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2523 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2524 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2525 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2526 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2527 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2528 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2529 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2530 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2531 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2532 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2533 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2534 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2535 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2536 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2537 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2538 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2539 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2540 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2541 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2542 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2543 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2544 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2545 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2546 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2547 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2548 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2549 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2550 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2551 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2552 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2553 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2554 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2555 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2556 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2557 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2558 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2559 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2560 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2562 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2563 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2564 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2565 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2566 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2567 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2568 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2569 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2570 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2571 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2572 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2573 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2574 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2575 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2576 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2577 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2578 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2579 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2580 2581 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2582 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2583 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2584 NUM_BANKS(ADDR_SURF_16_BANK)); 2585 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2586 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2587 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2588 NUM_BANKS(ADDR_SURF_16_BANK)); 2589 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2590 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2591 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2592 NUM_BANKS(ADDR_SURF_16_BANK)); 2593 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2594 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2595 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2596 NUM_BANKS(ADDR_SURF_16_BANK)); 2597 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2598 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2599 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2600 NUM_BANKS(ADDR_SURF_16_BANK)); 2601 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2602 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2603 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2604 NUM_BANKS(ADDR_SURF_16_BANK)); 2605 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2606 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2607 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2608 NUM_BANKS(ADDR_SURF_16_BANK)); 2609 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2610 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2611 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2612 NUM_BANKS(ADDR_SURF_16_BANK)); 2613 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2614 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2615 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2616 NUM_BANKS(ADDR_SURF_16_BANK)); 2617 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2618 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2619 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2620 NUM_BANKS(ADDR_SURF_16_BANK)); 2621 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2622 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2623 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2624 NUM_BANKS(ADDR_SURF_16_BANK)); 2625 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2626 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2627 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2628 NUM_BANKS(ADDR_SURF_8_BANK)); 2629 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2630 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2631 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2632 NUM_BANKS(ADDR_SURF_4_BANK)); 2633 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2634 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2635 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2636 NUM_BANKS(ADDR_SURF_4_BANK)); 2637 2638 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2639 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2640 2641 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2642 if (reg_offset != 7) 2643 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2644 2645 break; 2646 case CHIP_POLARIS11: 2647 case CHIP_POLARIS12: 2648 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2649 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2650 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2651 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2652 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2653 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2654 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2655 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2656 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2657 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2658 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2659 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2660 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2661 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2662 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2663 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2664 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2665 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2666 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2667 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2668 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2669 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2670 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2671 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2672 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2673 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2674 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2675 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2676 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2677 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2678 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2679 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2680 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2681 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2682 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2683 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2684 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2685 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2686 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2687 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2688 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2689 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2690 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2691 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2692 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2693 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2694 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2695 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2696 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2697 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2698 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2699 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2700 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2701 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2702 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2703 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2704 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2705 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2706 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2707 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2708 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2709 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2710 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2711 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2712 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2713 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2714 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2715 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2716 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2717 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2718 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2719 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2720 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2721 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2722 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2723 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2724 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2725 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2726 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2727 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2728 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2729 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2730 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2731 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2732 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2733 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2734 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2735 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2736 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2737 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2738 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2739 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2740 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2741 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2742 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2743 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2744 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2745 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2746 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2747 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2748 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2749 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2750 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2751 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2752 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2753 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2754 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2755 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2756 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2757 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2758 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2759 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2760 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2761 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2762 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2763 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2764 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2765 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2766 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2767 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2768 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2769 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2770 2771 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2772 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2773 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2774 NUM_BANKS(ADDR_SURF_16_BANK)); 2775 2776 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2777 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2778 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2779 NUM_BANKS(ADDR_SURF_16_BANK)); 2780 2781 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2782 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2783 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2784 NUM_BANKS(ADDR_SURF_16_BANK)); 2785 2786 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2787 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2788 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2789 NUM_BANKS(ADDR_SURF_16_BANK)); 2790 2791 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2792 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2793 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2794 NUM_BANKS(ADDR_SURF_16_BANK)); 2795 2796 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2797 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2798 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2799 NUM_BANKS(ADDR_SURF_16_BANK)); 2800 2801 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2802 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2803 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2804 NUM_BANKS(ADDR_SURF_16_BANK)); 2805 2806 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2809 NUM_BANKS(ADDR_SURF_16_BANK)); 2810 2811 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2812 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2813 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2814 NUM_BANKS(ADDR_SURF_16_BANK)); 2815 2816 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2817 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2818 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2819 NUM_BANKS(ADDR_SURF_16_BANK)); 2820 2821 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2822 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2823 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2824 NUM_BANKS(ADDR_SURF_16_BANK)); 2825 2826 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2827 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2828 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2829 NUM_BANKS(ADDR_SURF_16_BANK)); 2830 2831 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2832 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2833 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2834 NUM_BANKS(ADDR_SURF_8_BANK)); 2835 2836 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2837 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2838 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2839 NUM_BANKS(ADDR_SURF_4_BANK)); 2840 2841 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2842 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2843 2844 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2845 if (reg_offset != 7) 2846 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2847 2848 break; 2849 case CHIP_POLARIS10: 2850 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2851 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2852 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2853 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2854 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2855 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2856 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2857 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2858 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2859 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2860 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2861 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2862 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2863 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2864 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2865 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2866 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2867 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2868 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2869 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2870 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2871 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2872 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2873 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2874 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2875 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2876 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2877 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2878 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2879 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2880 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2881 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2882 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2883 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2884 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2885 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2886 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2887 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2888 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2889 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2890 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2891 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2892 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2893 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2894 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2895 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2896 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2897 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2898 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2899 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2900 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2901 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2902 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2903 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2904 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2905 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2906 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2907 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2908 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2909 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2910 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2911 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2912 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2913 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2914 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2915 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2916 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2917 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2918 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2919 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2920 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2921 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2922 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2923 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2924 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2925 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2926 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2927 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2928 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2929 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2930 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2931 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2932 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2933 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2934 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2935 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2936 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2937 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2938 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2939 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2940 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2941 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2942 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2943 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2944 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2945 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2946 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2947 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2948 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2949 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2950 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2951 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2952 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2953 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2954 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2955 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2956 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2957 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2958 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2959 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2960 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2961 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2962 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2963 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2964 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2965 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2966 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2967 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2968 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2969 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2970 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2971 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2972 2973 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2974 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2975 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2976 NUM_BANKS(ADDR_SURF_16_BANK)); 2977 2978 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2979 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2980 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2981 NUM_BANKS(ADDR_SURF_16_BANK)); 2982 2983 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2984 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2985 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2986 NUM_BANKS(ADDR_SURF_16_BANK)); 2987 2988 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2989 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2990 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2991 NUM_BANKS(ADDR_SURF_16_BANK)); 2992 2993 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2994 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2995 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2996 NUM_BANKS(ADDR_SURF_16_BANK)); 2997 2998 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2999 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3000 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3001 NUM_BANKS(ADDR_SURF_16_BANK)); 3002 3003 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3004 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3005 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3006 NUM_BANKS(ADDR_SURF_16_BANK)); 3007 3008 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3009 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3010 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3011 NUM_BANKS(ADDR_SURF_16_BANK)); 3012 3013 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3014 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3015 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3016 NUM_BANKS(ADDR_SURF_16_BANK)); 3017 3018 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3019 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3020 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3021 NUM_BANKS(ADDR_SURF_16_BANK)); 3022 3023 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3024 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3025 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3026 NUM_BANKS(ADDR_SURF_16_BANK)); 3027 3028 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3029 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3030 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3031 NUM_BANKS(ADDR_SURF_8_BANK)); 3032 3033 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3034 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3035 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3036 NUM_BANKS(ADDR_SURF_4_BANK)); 3037 3038 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3039 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3040 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3041 NUM_BANKS(ADDR_SURF_4_BANK)); 3042 3043 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3044 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3045 3046 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3047 if (reg_offset != 7) 3048 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3049 3050 break; 3051 case CHIP_STONEY: 3052 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3053 PIPE_CONFIG(ADDR_SURF_P2) | 3054 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3055 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3056 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3057 PIPE_CONFIG(ADDR_SURF_P2) | 3058 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3059 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3060 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3061 PIPE_CONFIG(ADDR_SURF_P2) | 3062 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3063 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3064 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3065 PIPE_CONFIG(ADDR_SURF_P2) | 3066 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3067 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3068 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3069 PIPE_CONFIG(ADDR_SURF_P2) | 3070 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3071 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3072 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3073 PIPE_CONFIG(ADDR_SURF_P2) | 3074 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3075 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3076 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3077 PIPE_CONFIG(ADDR_SURF_P2) | 3078 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3079 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3080 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3081 PIPE_CONFIG(ADDR_SURF_P2)); 3082 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3083 PIPE_CONFIG(ADDR_SURF_P2) | 3084 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3085 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3086 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3087 PIPE_CONFIG(ADDR_SURF_P2) | 3088 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3089 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3090 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3091 PIPE_CONFIG(ADDR_SURF_P2) | 3092 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3093 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3094 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3095 PIPE_CONFIG(ADDR_SURF_P2) | 3096 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3097 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3098 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3099 PIPE_CONFIG(ADDR_SURF_P2) | 3100 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3102 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3103 PIPE_CONFIG(ADDR_SURF_P2) | 3104 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3105 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3106 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3107 PIPE_CONFIG(ADDR_SURF_P2) | 3108 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3109 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3110 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3111 PIPE_CONFIG(ADDR_SURF_P2) | 3112 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3113 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3114 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3115 PIPE_CONFIG(ADDR_SURF_P2) | 3116 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3117 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3118 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3119 PIPE_CONFIG(ADDR_SURF_P2) | 3120 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3121 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3122 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3123 PIPE_CONFIG(ADDR_SURF_P2) | 3124 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3125 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3126 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3127 PIPE_CONFIG(ADDR_SURF_P2) | 3128 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3129 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3130 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3131 PIPE_CONFIG(ADDR_SURF_P2) | 3132 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3133 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3134 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3135 PIPE_CONFIG(ADDR_SURF_P2) | 3136 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3137 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3138 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3139 PIPE_CONFIG(ADDR_SURF_P2) | 3140 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3141 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3142 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3143 PIPE_CONFIG(ADDR_SURF_P2) | 3144 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3145 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3146 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3147 PIPE_CONFIG(ADDR_SURF_P2) | 3148 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3149 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3150 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3151 PIPE_CONFIG(ADDR_SURF_P2) | 3152 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3153 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3154 3155 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3156 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3157 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3158 NUM_BANKS(ADDR_SURF_8_BANK)); 3159 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3160 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3161 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3162 NUM_BANKS(ADDR_SURF_8_BANK)); 3163 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3164 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3165 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3166 NUM_BANKS(ADDR_SURF_8_BANK)); 3167 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3168 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3169 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3170 NUM_BANKS(ADDR_SURF_8_BANK)); 3171 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3172 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3173 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3174 NUM_BANKS(ADDR_SURF_8_BANK)); 3175 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3176 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3177 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3178 NUM_BANKS(ADDR_SURF_8_BANK)); 3179 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3180 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3181 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3182 NUM_BANKS(ADDR_SURF_8_BANK)); 3183 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3184 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3185 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3186 NUM_BANKS(ADDR_SURF_16_BANK)); 3187 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3188 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3189 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3190 NUM_BANKS(ADDR_SURF_16_BANK)); 3191 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3192 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3193 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3194 NUM_BANKS(ADDR_SURF_16_BANK)); 3195 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3196 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3197 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3198 NUM_BANKS(ADDR_SURF_16_BANK)); 3199 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3200 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3201 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3202 NUM_BANKS(ADDR_SURF_16_BANK)); 3203 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3204 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3205 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3206 NUM_BANKS(ADDR_SURF_16_BANK)); 3207 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3208 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3209 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3210 NUM_BANKS(ADDR_SURF_8_BANK)); 3211 3212 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3213 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3214 reg_offset != 23) 3215 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3216 3217 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3218 if (reg_offset != 7) 3219 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3220 3221 break; 3222 default: 3223 dev_warn(adev->dev, 3224 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n", 3225 adev->asic_type); 3226 fallthrough; 3227 3228 case CHIP_CARRIZO: 3229 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3230 PIPE_CONFIG(ADDR_SURF_P2) | 3231 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3232 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3233 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3234 PIPE_CONFIG(ADDR_SURF_P2) | 3235 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3236 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3237 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3238 PIPE_CONFIG(ADDR_SURF_P2) | 3239 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3240 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3241 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3242 PIPE_CONFIG(ADDR_SURF_P2) | 3243 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3244 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3245 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3246 PIPE_CONFIG(ADDR_SURF_P2) | 3247 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3248 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3249 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3250 PIPE_CONFIG(ADDR_SURF_P2) | 3251 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3252 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3253 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3254 PIPE_CONFIG(ADDR_SURF_P2) | 3255 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3256 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3257 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3258 PIPE_CONFIG(ADDR_SURF_P2)); 3259 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3260 PIPE_CONFIG(ADDR_SURF_P2) | 3261 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3262 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3263 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3264 PIPE_CONFIG(ADDR_SURF_P2) | 3265 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3266 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3267 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3268 PIPE_CONFIG(ADDR_SURF_P2) | 3269 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3270 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3271 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3272 PIPE_CONFIG(ADDR_SURF_P2) | 3273 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3274 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3275 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3276 PIPE_CONFIG(ADDR_SURF_P2) | 3277 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3278 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3279 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3280 PIPE_CONFIG(ADDR_SURF_P2) | 3281 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3282 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3283 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3284 PIPE_CONFIG(ADDR_SURF_P2) | 3285 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3286 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3287 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3288 PIPE_CONFIG(ADDR_SURF_P2) | 3289 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3290 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3291 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3292 PIPE_CONFIG(ADDR_SURF_P2) | 3293 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3294 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3295 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3296 PIPE_CONFIG(ADDR_SURF_P2) | 3297 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3299 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3300 PIPE_CONFIG(ADDR_SURF_P2) | 3301 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3302 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3303 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3304 PIPE_CONFIG(ADDR_SURF_P2) | 3305 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3306 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3307 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3308 PIPE_CONFIG(ADDR_SURF_P2) | 3309 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3310 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3311 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3312 PIPE_CONFIG(ADDR_SURF_P2) | 3313 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3314 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3315 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3316 PIPE_CONFIG(ADDR_SURF_P2) | 3317 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3318 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3319 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3320 PIPE_CONFIG(ADDR_SURF_P2) | 3321 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3322 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3323 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3324 PIPE_CONFIG(ADDR_SURF_P2) | 3325 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3326 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3327 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3328 PIPE_CONFIG(ADDR_SURF_P2) | 3329 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3330 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3331 3332 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3333 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3334 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3335 NUM_BANKS(ADDR_SURF_8_BANK)); 3336 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3337 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3338 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3339 NUM_BANKS(ADDR_SURF_8_BANK)); 3340 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3341 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3342 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3343 NUM_BANKS(ADDR_SURF_8_BANK)); 3344 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3345 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3346 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3347 NUM_BANKS(ADDR_SURF_8_BANK)); 3348 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3349 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3350 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3351 NUM_BANKS(ADDR_SURF_8_BANK)); 3352 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3353 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3354 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3355 NUM_BANKS(ADDR_SURF_8_BANK)); 3356 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3357 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3358 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3359 NUM_BANKS(ADDR_SURF_8_BANK)); 3360 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3361 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3362 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3363 NUM_BANKS(ADDR_SURF_16_BANK)); 3364 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3365 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3366 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3367 NUM_BANKS(ADDR_SURF_16_BANK)); 3368 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3369 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3370 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3371 NUM_BANKS(ADDR_SURF_16_BANK)); 3372 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3373 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3374 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3375 NUM_BANKS(ADDR_SURF_16_BANK)); 3376 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3377 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3378 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3379 NUM_BANKS(ADDR_SURF_16_BANK)); 3380 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3381 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3382 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3383 NUM_BANKS(ADDR_SURF_16_BANK)); 3384 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3385 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3386 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3387 NUM_BANKS(ADDR_SURF_8_BANK)); 3388 3389 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3390 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3391 reg_offset != 23) 3392 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3393 3394 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3395 if (reg_offset != 7) 3396 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3397 3398 break; 3399 } 3400 } 3401 3402 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, 3403 u32 se_num, u32 sh_num, u32 instance, 3404 int xcc_id) 3405 { 3406 u32 data; 3407 3408 if (instance == 0xffffffff) 3409 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 3410 else 3411 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 3412 3413 if (se_num == 0xffffffff) 3414 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 3415 else 3416 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 3417 3418 if (sh_num == 0xffffffff) 3419 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 3420 else 3421 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 3422 3423 WREG32(mmGRBM_GFX_INDEX, data); 3424 } 3425 3426 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev, 3427 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 3428 { 3429 vi_srbm_select(adev, me, pipe, q, vm); 3430 } 3431 3432 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev) 3433 { 3434 u32 data, mask; 3435 3436 data = RREG32(mmCC_RB_BACKEND_DISABLE) | 3437 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3438 3439 data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE); 3440 3441 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 3442 adev->gfx.config.max_sh_per_se); 3443 3444 return (~data) & mask; 3445 } 3446 3447 static void 3448 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1) 3449 { 3450 switch (adev->asic_type) { 3451 case CHIP_FIJI: 3452 case CHIP_VEGAM: 3453 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) | 3454 RB_XSEL2(1) | PKR_MAP(2) | 3455 PKR_XSEL(1) | PKR_YSEL(1) | 3456 SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3); 3457 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) | 3458 SE_PAIR_YSEL(2); 3459 break; 3460 case CHIP_TONGA: 3461 case CHIP_POLARIS10: 3462 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3463 SE_XSEL(1) | SE_YSEL(1); 3464 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) | 3465 SE_PAIR_YSEL(2); 3466 break; 3467 case CHIP_TOPAZ: 3468 case CHIP_CARRIZO: 3469 *rconf |= RB_MAP_PKR0(2); 3470 *rconf1 |= 0x0; 3471 break; 3472 case CHIP_POLARIS11: 3473 case CHIP_POLARIS12: 3474 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3475 SE_XSEL(1) | SE_YSEL(1); 3476 *rconf1 |= 0x0; 3477 break; 3478 case CHIP_STONEY: 3479 *rconf |= 0x0; 3480 *rconf1 |= 0x0; 3481 break; 3482 default: 3483 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type); 3484 break; 3485 } 3486 } 3487 3488 static void 3489 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev, 3490 u32 raster_config, u32 raster_config_1, 3491 unsigned rb_mask, unsigned num_rb) 3492 { 3493 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1); 3494 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1); 3495 unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2); 3496 unsigned rb_per_se = num_rb / num_se; 3497 unsigned se_mask[4]; 3498 unsigned se; 3499 3500 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask; 3501 se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask; 3502 se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask; 3503 se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask; 3504 3505 WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4)); 3506 WARN_ON(!(sh_per_se == 1 || sh_per_se == 2)); 3507 WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2)); 3508 3509 if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) || 3510 (!se_mask[2] && !se_mask[3]))) { 3511 raster_config_1 &= ~SE_PAIR_MAP_MASK; 3512 3513 if (!se_mask[0] && !se_mask[1]) { 3514 raster_config_1 |= 3515 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3); 3516 } else { 3517 raster_config_1 |= 3518 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0); 3519 } 3520 } 3521 3522 for (se = 0; se < num_se; se++) { 3523 unsigned raster_config_se = raster_config; 3524 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se); 3525 unsigned pkr1_mask = pkr0_mask << rb_per_pkr; 3526 int idx = (se / 2) * 2; 3527 3528 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) { 3529 raster_config_se &= ~SE_MAP_MASK; 3530 3531 if (!se_mask[idx]) { 3532 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3); 3533 } else { 3534 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0); 3535 } 3536 } 3537 3538 pkr0_mask &= rb_mask; 3539 pkr1_mask &= rb_mask; 3540 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) { 3541 raster_config_se &= ~PKR_MAP_MASK; 3542 3543 if (!pkr0_mask) { 3544 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3); 3545 } else { 3546 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0); 3547 } 3548 } 3549 3550 if (rb_per_se >= 2) { 3551 unsigned rb0_mask = 1 << (se * rb_per_se); 3552 unsigned rb1_mask = rb0_mask << 1; 3553 3554 rb0_mask &= rb_mask; 3555 rb1_mask &= rb_mask; 3556 if (!rb0_mask || !rb1_mask) { 3557 raster_config_se &= ~RB_MAP_PKR0_MASK; 3558 3559 if (!rb0_mask) { 3560 raster_config_se |= 3561 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3); 3562 } else { 3563 raster_config_se |= 3564 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0); 3565 } 3566 } 3567 3568 if (rb_per_se > 2) { 3569 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr); 3570 rb1_mask = rb0_mask << 1; 3571 rb0_mask &= rb_mask; 3572 rb1_mask &= rb_mask; 3573 if (!rb0_mask || !rb1_mask) { 3574 raster_config_se &= ~RB_MAP_PKR1_MASK; 3575 3576 if (!rb0_mask) { 3577 raster_config_se |= 3578 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3); 3579 } else { 3580 raster_config_se |= 3581 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0); 3582 } 3583 } 3584 } 3585 } 3586 3587 /* GRBM_GFX_INDEX has a different offset on VI */ 3588 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff, 0); 3589 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se); 3590 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3591 } 3592 3593 /* GRBM_GFX_INDEX has a different offset on VI */ 3594 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3595 } 3596 3597 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev) 3598 { 3599 int i, j; 3600 u32 data; 3601 u32 raster_config = 0, raster_config_1 = 0; 3602 u32 active_rbs = 0; 3603 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 3604 adev->gfx.config.max_sh_per_se; 3605 unsigned num_rb_pipes; 3606 3607 mutex_lock(&adev->grbm_idx_mutex); 3608 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3609 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3610 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3611 data = gfx_v8_0_get_rb_active_bitmap(adev); 3612 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 3613 rb_bitmap_width_per_sh); 3614 } 3615 } 3616 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3617 3618 adev->gfx.config.backend_enable_mask = active_rbs; 3619 adev->gfx.config.num_rbs = hweight32(active_rbs); 3620 3621 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se * 3622 adev->gfx.config.max_shader_engines, 16); 3623 3624 gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1); 3625 3626 if (!adev->gfx.config.backend_enable_mask || 3627 adev->gfx.config.num_rbs >= num_rb_pipes) { 3628 WREG32(mmPA_SC_RASTER_CONFIG, raster_config); 3629 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3630 } else { 3631 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1, 3632 adev->gfx.config.backend_enable_mask, 3633 num_rb_pipes); 3634 } 3635 3636 /* cache the values for userspace */ 3637 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3638 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3639 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3640 adev->gfx.config.rb_config[i][j].rb_backend_disable = 3641 RREG32(mmCC_RB_BACKEND_DISABLE); 3642 adev->gfx.config.rb_config[i][j].user_rb_backend_disable = 3643 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3644 adev->gfx.config.rb_config[i][j].raster_config = 3645 RREG32(mmPA_SC_RASTER_CONFIG); 3646 adev->gfx.config.rb_config[i][j].raster_config_1 = 3647 RREG32(mmPA_SC_RASTER_CONFIG_1); 3648 } 3649 } 3650 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3651 mutex_unlock(&adev->grbm_idx_mutex); 3652 } 3653 3654 #define DEFAULT_SH_MEM_BASES (0x6000) 3655 /** 3656 * gfx_v8_0_init_compute_vmid - gart enable 3657 * 3658 * @adev: amdgpu_device pointer 3659 * 3660 * Initialize compute vmid sh_mem registers 3661 * 3662 */ 3663 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev) 3664 { 3665 int i; 3666 uint32_t sh_mem_config; 3667 uint32_t sh_mem_bases; 3668 3669 /* 3670 * Configure apertures: 3671 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 3672 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 3673 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 3674 */ 3675 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 3676 3677 sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 << 3678 SH_MEM_CONFIG__ADDRESS_MODE__SHIFT | 3679 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 3680 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT | 3681 MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | 3682 SH_MEM_CONFIG__PRIVATE_ATC_MASK; 3683 3684 mutex_lock(&adev->srbm_mutex); 3685 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3686 vi_srbm_select(adev, 0, 0, 0, i); 3687 /* CP and shaders */ 3688 WREG32(mmSH_MEM_CONFIG, sh_mem_config); 3689 WREG32(mmSH_MEM_APE1_BASE, 1); 3690 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3691 WREG32(mmSH_MEM_BASES, sh_mem_bases); 3692 } 3693 vi_srbm_select(adev, 0, 0, 0, 0); 3694 mutex_unlock(&adev->srbm_mutex); 3695 3696 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 3697 access. These should be enabled by FW for target VMIDs. */ 3698 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3699 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0); 3700 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0); 3701 WREG32(amdgpu_gds_reg_offset[i].gws, 0); 3702 WREG32(amdgpu_gds_reg_offset[i].oa, 0); 3703 } 3704 } 3705 3706 static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev) 3707 { 3708 int vmid; 3709 3710 /* 3711 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 3712 * access. Compute VMIDs should be enabled by FW for target VMIDs, 3713 * the driver can enable them for graphics. VMID0 should maintain 3714 * access so that HWS firmware can save/restore entries. 3715 */ 3716 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 3717 WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0); 3718 WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0); 3719 WREG32(amdgpu_gds_reg_offset[vmid].gws, 0); 3720 WREG32(amdgpu_gds_reg_offset[vmid].oa, 0); 3721 } 3722 } 3723 3724 static void gfx_v8_0_config_init(struct amdgpu_device *adev) 3725 { 3726 switch (adev->asic_type) { 3727 default: 3728 adev->gfx.config.double_offchip_lds_buf = 1; 3729 break; 3730 case CHIP_CARRIZO: 3731 case CHIP_STONEY: 3732 adev->gfx.config.double_offchip_lds_buf = 0; 3733 break; 3734 } 3735 } 3736 3737 static void gfx_v8_0_constants_init(struct amdgpu_device *adev) 3738 { 3739 u32 tmp, sh_static_mem_cfg; 3740 int i; 3741 3742 WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF); 3743 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3744 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3745 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config); 3746 3747 gfx_v8_0_tiling_mode_table_init(adev); 3748 gfx_v8_0_setup_rb(adev); 3749 gfx_v8_0_get_cu_info(adev); 3750 gfx_v8_0_config_init(adev); 3751 3752 /* XXX SH_MEM regs */ 3753 /* where to put LDS, scratch, GPUVM in FSA64 space */ 3754 sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG, 3755 SWIZZLE_ENABLE, 1); 3756 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3757 ELEMENT_SIZE, 1); 3758 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3759 INDEX_STRIDE, 3); 3760 WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); 3761 3762 mutex_lock(&adev->srbm_mutex); 3763 for_each_vmid_and_zero(i, adev, 0) { 3764 vi_srbm_select(adev, 0, 0, 0, i); 3765 /* CP and shaders */ 3766 if (i == 0) { 3767 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC); 3768 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3769 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3770 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3771 WREG32(mmSH_MEM_CONFIG, tmp); 3772 WREG32(mmSH_MEM_BASES, 0); 3773 } else { 3774 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC); 3775 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3776 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3777 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3778 WREG32(mmSH_MEM_CONFIG, tmp); 3779 tmp = adev->gmc.shared_aperture_start >> 48; 3780 WREG32(mmSH_MEM_BASES, tmp); 3781 } 3782 3783 WREG32(mmSH_MEM_APE1_BASE, 1); 3784 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3785 } 3786 vi_srbm_select(adev, 0, 0, 0, 0); 3787 mutex_unlock(&adev->srbm_mutex); 3788 3789 gfx_v8_0_init_compute_vmid(adev); 3790 gfx_v8_0_init_gds_vmid(adev); 3791 3792 mutex_lock(&adev->grbm_idx_mutex); 3793 /* 3794 * making sure that the following register writes will be broadcasted 3795 * to all the shaders 3796 */ 3797 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3798 3799 WREG32(mmPA_SC_FIFO_SIZE, 3800 (adev->gfx.config.sc_prim_fifo_size_frontend << 3801 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) | 3802 (adev->gfx.config.sc_prim_fifo_size_backend << 3803 PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) | 3804 (adev->gfx.config.sc_hiz_tile_fifo_size << 3805 PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) | 3806 (adev->gfx.config.sc_earlyz_tile_fifo_size << 3807 PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)); 3808 3809 tmp = RREG32(mmSPI_ARB_PRIORITY); 3810 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2); 3811 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2); 3812 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2); 3813 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2); 3814 WREG32(mmSPI_ARB_PRIORITY, tmp); 3815 3816 mutex_unlock(&adev->grbm_idx_mutex); 3817 3818 } 3819 3820 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 3821 { 3822 u32 i, j, k; 3823 u32 mask; 3824 3825 mutex_lock(&adev->grbm_idx_mutex); 3826 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3827 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3828 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3829 for (k = 0; k < adev->usec_timeout; k++) { 3830 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0) 3831 break; 3832 udelay(1); 3833 } 3834 if (k == adev->usec_timeout) { 3835 gfx_v8_0_select_se_sh(adev, 0xffffffff, 3836 0xffffffff, 0xffffffff, 0); 3837 mutex_unlock(&adev->grbm_idx_mutex); 3838 drm_info(adev_to_drm(adev), "Timeout wait for RLC serdes %u,%u\n", 3839 i, j); 3840 return; 3841 } 3842 } 3843 } 3844 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3845 mutex_unlock(&adev->grbm_idx_mutex); 3846 3847 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 3848 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 3849 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 3850 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 3851 for (k = 0; k < adev->usec_timeout; k++) { 3852 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 3853 break; 3854 udelay(1); 3855 } 3856 } 3857 3858 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 3859 bool enable) 3860 { 3861 u32 tmp = RREG32(mmCP_INT_CNTL_RING0); 3862 3863 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 3864 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 3865 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 3866 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 3867 3868 WREG32(mmCP_INT_CNTL_RING0, tmp); 3869 } 3870 3871 static void gfx_v8_0_init_csb(struct amdgpu_device *adev) 3872 { 3873 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 3874 /* csib */ 3875 WREG32(mmRLC_CSIB_ADDR_HI, 3876 adev->gfx.rlc.clear_state_gpu_addr >> 32); 3877 WREG32(mmRLC_CSIB_ADDR_LO, 3878 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 3879 WREG32(mmRLC_CSIB_LENGTH, 3880 adev->gfx.rlc.clear_state_size); 3881 } 3882 3883 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format, 3884 int ind_offset, 3885 int list_size, 3886 int *unique_indices, 3887 int *indices_count, 3888 int max_indices, 3889 int *ind_start_offsets, 3890 int *offset_count, 3891 int max_offset) 3892 { 3893 int indices; 3894 bool new_entry = true; 3895 3896 for (; ind_offset < list_size; ind_offset++) { 3897 3898 if (new_entry) { 3899 new_entry = false; 3900 ind_start_offsets[*offset_count] = ind_offset; 3901 *offset_count = *offset_count + 1; 3902 BUG_ON(*offset_count >= max_offset); 3903 } 3904 3905 if (register_list_format[ind_offset] == 0xFFFFFFFF) { 3906 new_entry = true; 3907 continue; 3908 } 3909 3910 ind_offset += 2; 3911 3912 /* look for the matching indice */ 3913 for (indices = 0; 3914 indices < *indices_count; 3915 indices++) { 3916 if (unique_indices[indices] == 3917 register_list_format[ind_offset]) 3918 break; 3919 } 3920 3921 if (indices >= *indices_count) { 3922 unique_indices[*indices_count] = 3923 register_list_format[ind_offset]; 3924 indices = *indices_count; 3925 *indices_count = *indices_count + 1; 3926 BUG_ON(*indices_count >= max_indices); 3927 } 3928 3929 register_list_format[ind_offset] = indices; 3930 } 3931 } 3932 3933 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev) 3934 { 3935 int i, temp, data; 3936 int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0}; 3937 int indices_count = 0; 3938 int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 3939 int offset_count = 0; 3940 3941 int list_size; 3942 unsigned int *register_list_format = 3943 kmemdup(adev->gfx.rlc.register_list_format, 3944 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 3945 if (!register_list_format) 3946 return -ENOMEM; 3947 3948 gfx_v8_0_parse_ind_reg_list(register_list_format, 3949 RLC_FormatDirectRegListLength, 3950 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 3951 unique_indices, 3952 &indices_count, 3953 ARRAY_SIZE(unique_indices), 3954 indirect_start_offsets, 3955 &offset_count, 3956 ARRAY_SIZE(indirect_start_offsets)); 3957 3958 /* save and restore list */ 3959 WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1); 3960 3961 WREG32(mmRLC_SRM_ARAM_ADDR, 0); 3962 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 3963 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]); 3964 3965 /* indirect list */ 3966 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start); 3967 for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++) 3968 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]); 3969 3970 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 3971 list_size = list_size >> 1; 3972 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size); 3973 WREG32(mmRLC_GPM_SCRATCH_DATA, list_size); 3974 3975 /* starting offsets starts */ 3976 WREG32(mmRLC_GPM_SCRATCH_ADDR, 3977 adev->gfx.rlc.starting_offsets_start); 3978 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 3979 WREG32(mmRLC_GPM_SCRATCH_DATA, 3980 indirect_start_offsets[i]); 3981 3982 /* unique indices */ 3983 temp = mmRLC_SRM_INDEX_CNTL_ADDR_0; 3984 data = mmRLC_SRM_INDEX_CNTL_DATA_0; 3985 for (i = 0; i < ARRAY_SIZE(unique_indices); i++) { 3986 if (unique_indices[i] != 0) { 3987 WREG32(temp + i, unique_indices[i] & 0x3FFFF); 3988 WREG32(data + i, unique_indices[i] >> 20); 3989 } 3990 } 3991 kfree(register_list_format); 3992 3993 return 0; 3994 } 3995 3996 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev) 3997 { 3998 WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1); 3999 } 4000 4001 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev) 4002 { 4003 uint32_t data; 4004 4005 WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60); 4006 4007 data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10); 4008 data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10); 4009 data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10); 4010 data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10); 4011 WREG32(mmRLC_PG_DELAY, data); 4012 4013 WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3); 4014 WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0); 4015 4016 } 4017 4018 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 4019 bool enable) 4020 { 4021 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0); 4022 } 4023 4024 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 4025 bool enable) 4026 { 4027 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0); 4028 } 4029 4030 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable) 4031 { 4032 WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1); 4033 } 4034 4035 static void gfx_v8_0_init_pg(struct amdgpu_device *adev) 4036 { 4037 if ((adev->asic_type == CHIP_CARRIZO) || 4038 (adev->asic_type == CHIP_STONEY)) { 4039 gfx_v8_0_init_csb(adev); 4040 gfx_v8_0_init_save_restore_list(adev); 4041 gfx_v8_0_enable_save_restore_machine(adev); 4042 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8); 4043 gfx_v8_0_init_power_gating(adev); 4044 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask); 4045 } else if ((adev->asic_type == CHIP_POLARIS11) || 4046 (adev->asic_type == CHIP_POLARIS12) || 4047 (adev->asic_type == CHIP_VEGAM)) { 4048 gfx_v8_0_init_csb(adev); 4049 gfx_v8_0_init_save_restore_list(adev); 4050 gfx_v8_0_enable_save_restore_machine(adev); 4051 gfx_v8_0_init_power_gating(adev); 4052 } 4053 4054 } 4055 4056 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev) 4057 { 4058 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0); 4059 4060 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4061 gfx_v8_0_wait_for_rlc_serdes(adev); 4062 } 4063 4064 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev) 4065 { 4066 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4067 udelay(50); 4068 4069 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 4070 udelay(50); 4071 } 4072 4073 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev) 4074 { 4075 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1); 4076 4077 /* carrizo do enable cp interrupt after cp inited */ 4078 if (!(adev->flags & AMD_IS_APU)) 4079 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4080 4081 udelay(50); 4082 } 4083 4084 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev) 4085 { 4086 if (amdgpu_sriov_vf(adev)) { 4087 gfx_v8_0_init_csb(adev); 4088 return 0; 4089 } 4090 4091 adev->gfx.rlc.funcs->stop(adev); 4092 adev->gfx.rlc.funcs->reset(adev); 4093 gfx_v8_0_init_pg(adev); 4094 adev->gfx.rlc.funcs->start(adev); 4095 4096 return 0; 4097 } 4098 4099 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 4100 { 4101 u32 tmp = RREG32(mmCP_ME_CNTL); 4102 4103 if (enable) { 4104 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0); 4105 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0); 4106 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0); 4107 } else { 4108 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1); 4109 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1); 4110 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1); 4111 } 4112 WREG32(mmCP_ME_CNTL, tmp); 4113 udelay(50); 4114 } 4115 4116 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev) 4117 { 4118 u32 count = 0; 4119 const struct cs_section_def *sect = NULL; 4120 const struct cs_extent_def *ext = NULL; 4121 4122 /* begin clear state */ 4123 count += 2; 4124 /* context control state */ 4125 count += 3; 4126 4127 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4128 for (ext = sect->section; ext->extent != NULL; ++ext) { 4129 if (sect->id == SECT_CONTEXT) 4130 count += 2 + ext->reg_count; 4131 else 4132 return 0; 4133 } 4134 } 4135 /* pa_sc_raster_config/pa_sc_raster_config1 */ 4136 count += 4; 4137 /* end clear state */ 4138 count += 2; 4139 /* clear state */ 4140 count += 2; 4141 4142 return count; 4143 } 4144 4145 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev) 4146 { 4147 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 4148 const struct cs_section_def *sect = NULL; 4149 const struct cs_extent_def *ext = NULL; 4150 int r, i; 4151 4152 /* init the CP */ 4153 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 4154 WREG32(mmCP_ENDIAN_SWAP, 0); 4155 WREG32(mmCP_DEVICE_ID, 1); 4156 4157 gfx_v8_0_cp_gfx_enable(adev, true); 4158 4159 r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4); 4160 if (r) { 4161 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 4162 return r; 4163 } 4164 4165 /* clear state buffer */ 4166 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4167 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4168 4169 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4170 amdgpu_ring_write(ring, 0x80000000); 4171 amdgpu_ring_write(ring, 0x80000000); 4172 4173 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4174 for (ext = sect->section; ext->extent != NULL; ++ext) { 4175 if (sect->id == SECT_CONTEXT) { 4176 amdgpu_ring_write(ring, 4177 PACKET3(PACKET3_SET_CONTEXT_REG, 4178 ext->reg_count)); 4179 amdgpu_ring_write(ring, 4180 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 4181 for (i = 0; i < ext->reg_count; i++) 4182 amdgpu_ring_write(ring, ext->extent[i]); 4183 } 4184 } 4185 } 4186 4187 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4188 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 4189 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config); 4190 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1); 4191 4192 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4193 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4194 4195 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4196 amdgpu_ring_write(ring, 0); 4197 4198 /* init the CE partitions */ 4199 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4200 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4201 amdgpu_ring_write(ring, 0x8000); 4202 amdgpu_ring_write(ring, 0x8000); 4203 4204 amdgpu_ring_commit(ring); 4205 4206 return 0; 4207 } 4208 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring) 4209 { 4210 u32 tmp; 4211 /* no gfx doorbells on iceland */ 4212 if (adev->asic_type == CHIP_TOPAZ) 4213 return; 4214 4215 tmp = RREG32(mmCP_RB_DOORBELL_CONTROL); 4216 4217 if (ring->use_doorbell) { 4218 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4219 DOORBELL_OFFSET, ring->doorbell_index); 4220 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4221 DOORBELL_HIT, 0); 4222 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4223 DOORBELL_EN, 1); 4224 } else { 4225 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 4226 } 4227 4228 WREG32(mmCP_RB_DOORBELL_CONTROL, tmp); 4229 4230 if (adev->flags & AMD_IS_APU) 4231 return; 4232 4233 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 4234 DOORBELL_RANGE_LOWER, 4235 adev->doorbell_index.gfx_ring0); 4236 WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 4237 4238 WREG32(mmCP_RB_DOORBELL_RANGE_UPPER, 4239 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 4240 } 4241 4242 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev) 4243 { 4244 struct amdgpu_ring *ring; 4245 u32 tmp; 4246 u32 rb_bufsz; 4247 u64 rb_addr, rptr_addr, wptr_gpu_addr; 4248 4249 /* Set the write pointer delay */ 4250 WREG32(mmCP_RB_WPTR_DELAY, 0); 4251 4252 /* set the RB to use vmid 0 */ 4253 WREG32(mmCP_RB_VMID, 0); 4254 4255 /* Set ring buffer size */ 4256 ring = &adev->gfx.gfx_ring[0]; 4257 rb_bufsz = order_base_2(ring->ring_size / 8); 4258 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 4259 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 4260 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3); 4261 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1); 4262 #ifdef __BIG_ENDIAN 4263 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 4264 #endif 4265 WREG32(mmCP_RB0_CNTL, tmp); 4266 4267 /* Initialize the ring buffer's read and write pointers */ 4268 WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK); 4269 ring->wptr = 0; 4270 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 4271 4272 /* set the wb address whether it's enabled or not */ 4273 rptr_addr = ring->rptr_gpu_addr; 4274 WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 4275 WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF); 4276 4277 wptr_gpu_addr = ring->wptr_gpu_addr; 4278 WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 4279 WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 4280 mdelay(1); 4281 WREG32(mmCP_RB0_CNTL, tmp); 4282 4283 rb_addr = ring->gpu_addr >> 8; 4284 WREG32(mmCP_RB0_BASE, rb_addr); 4285 WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4286 4287 gfx_v8_0_set_cpg_door_bell(adev, ring); 4288 /* start the ring */ 4289 amdgpu_ring_clear_ring(ring); 4290 gfx_v8_0_cp_gfx_start(adev); 4291 4292 return 0; 4293 } 4294 4295 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 4296 { 4297 if (enable) { 4298 WREG32(mmCP_MEC_CNTL, 0); 4299 } else { 4300 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 4301 adev->gfx.kiq[0].ring.sched.ready = false; 4302 } 4303 udelay(50); 4304 } 4305 4306 /* KIQ functions */ 4307 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring) 4308 { 4309 uint32_t tmp; 4310 struct amdgpu_device *adev = ring->adev; 4311 4312 /* tell RLC which is KIQ queue */ 4313 tmp = RREG32(mmRLC_CP_SCHEDULERS); 4314 tmp &= 0xffffff00; 4315 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4316 WREG32(mmRLC_CP_SCHEDULERS, tmp | 0x80); 4317 } 4318 4319 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev) 4320 { 4321 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 4322 uint64_t queue_mask = 0; 4323 int r, i; 4324 4325 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 4326 if (!test_bit(i, adev->gfx.mec_bitmap[0].queue_bitmap)) 4327 continue; 4328 4329 /* This situation may be hit in the future if a new HW 4330 * generation exposes more than 64 queues. If so, the 4331 * definition of queue_mask needs updating */ 4332 if (WARN_ON(i >= (sizeof(queue_mask)*8))) { 4333 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 4334 break; 4335 } 4336 4337 queue_mask |= (1ull << i); 4338 } 4339 4340 r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8); 4341 if (r) { 4342 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4343 return r; 4344 } 4345 /* set resources */ 4346 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 4347 amdgpu_ring_write(kiq_ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */ 4348 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 4349 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 4350 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 4351 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 4352 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 4353 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 4354 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4355 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4356 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 4357 uint64_t wptr_addr = ring->wptr_gpu_addr; 4358 4359 /* map queues */ 4360 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 4361 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 4362 amdgpu_ring_write(kiq_ring, 4363 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 4364 amdgpu_ring_write(kiq_ring, 4365 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) | 4366 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 4367 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 4368 PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */ 4369 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 4370 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 4371 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 4372 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 4373 } 4374 4375 amdgpu_ring_commit(kiq_ring); 4376 4377 return 0; 4378 } 4379 4380 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req) 4381 { 4382 int i, r = 0; 4383 4384 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) { 4385 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req); 4386 for (i = 0; i < adev->usec_timeout; i++) { 4387 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK)) 4388 break; 4389 udelay(1); 4390 } 4391 if (i == adev->usec_timeout) 4392 r = -ETIMEDOUT; 4393 } 4394 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0); 4395 WREG32(mmCP_HQD_PQ_RPTR, 0); 4396 WREG32(mmCP_HQD_PQ_WPTR, 0); 4397 4398 return r; 4399 } 4400 4401 static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd) 4402 { 4403 struct amdgpu_device *adev = ring->adev; 4404 4405 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 4406 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 4407 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 4408 mqd->cp_hqd_queue_priority = 4409 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 4410 } 4411 } 4412 } 4413 4414 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring) 4415 { 4416 struct amdgpu_device *adev = ring->adev; 4417 struct vi_mqd *mqd = ring->mqd_ptr; 4418 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4419 uint32_t tmp; 4420 4421 mqd->header = 0xC0310800; 4422 mqd->compute_pipelinestat_enable = 0x00000001; 4423 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4424 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4425 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4426 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4427 mqd->compute_misc_reserved = 0x00000003; 4428 mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr 4429 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4430 mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr 4431 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4432 eop_base_addr = ring->eop_gpu_addr >> 8; 4433 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4434 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4435 4436 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4437 tmp = RREG32(mmCP_HQD_EOP_CONTROL); 4438 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4439 (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1)); 4440 4441 mqd->cp_hqd_eop_control = tmp; 4442 4443 /* enable doorbell? */ 4444 tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL), 4445 CP_HQD_PQ_DOORBELL_CONTROL, 4446 DOORBELL_EN, 4447 ring->use_doorbell ? 1 : 0); 4448 4449 mqd->cp_hqd_pq_doorbell_control = tmp; 4450 4451 /* set the pointer to the MQD */ 4452 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 4453 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 4454 4455 /* set MQD vmid to 0 */ 4456 tmp = RREG32(mmCP_MQD_CONTROL); 4457 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4458 mqd->cp_mqd_control = tmp; 4459 4460 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4461 hqd_gpu_addr = ring->gpu_addr >> 8; 4462 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4463 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4464 4465 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4466 tmp = RREG32(mmCP_HQD_PQ_CONTROL); 4467 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4468 (order_base_2(ring->ring_size / 4) - 1)); 4469 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4470 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4471 #ifdef __BIG_ENDIAN 4472 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 4473 #endif 4474 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 4475 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 4476 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4477 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4478 mqd->cp_hqd_pq_control = tmp; 4479 4480 /* set the wb address whether it's enabled or not */ 4481 wb_gpu_addr = ring->rptr_gpu_addr; 4482 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4483 mqd->cp_hqd_pq_rptr_report_addr_hi = 4484 upper_32_bits(wb_gpu_addr) & 0xffff; 4485 4486 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4487 wb_gpu_addr = ring->wptr_gpu_addr; 4488 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4489 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4490 4491 tmp = 0; 4492 /* enable the doorbell if requested */ 4493 if (ring->use_doorbell) { 4494 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL); 4495 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4496 DOORBELL_OFFSET, ring->doorbell_index); 4497 4498 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4499 DOORBELL_EN, 1); 4500 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4501 DOORBELL_SOURCE, 0); 4502 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4503 DOORBELL_HIT, 0); 4504 } 4505 4506 mqd->cp_hqd_pq_doorbell_control = tmp; 4507 4508 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4509 ring->wptr = 0; 4510 mqd->cp_hqd_pq_wptr = ring->wptr; 4511 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); 4512 4513 /* set the vmid for the queue */ 4514 mqd->cp_hqd_vmid = 0; 4515 4516 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE); 4517 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 4518 mqd->cp_hqd_persistent_state = tmp; 4519 4520 /* set MTYPE */ 4521 tmp = RREG32(mmCP_HQD_IB_CONTROL); 4522 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4523 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3); 4524 mqd->cp_hqd_ib_control = tmp; 4525 4526 tmp = RREG32(mmCP_HQD_IQ_TIMER); 4527 tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3); 4528 mqd->cp_hqd_iq_timer = tmp; 4529 4530 tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL); 4531 tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3); 4532 mqd->cp_hqd_ctx_save_control = tmp; 4533 4534 /* defaults */ 4535 mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR); 4536 mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR); 4537 mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO); 4538 mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI); 4539 mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET); 4540 mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE); 4541 mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET); 4542 mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE); 4543 mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS); 4544 mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR); 4545 mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM); 4546 mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES); 4547 4548 /* set static priority for a queue/ring */ 4549 gfx_v8_0_mqd_set_priority(ring, mqd); 4550 tmp = RREG32(mmCP_HQD_QUANTUM); 4551 if (ring != &adev->gfx.kiq[0].ring) { 4552 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); 4553 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 4554 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10); 4555 } 4556 mqd->cp_hqd_quantum = tmp; 4557 4558 /* map_queues packet doesn't need activate the queue, 4559 * so only kiq need set this field. 4560 */ 4561 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 4562 mqd->cp_hqd_active = 1; 4563 4564 return 0; 4565 } 4566 4567 static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev, 4568 struct vi_mqd *mqd) 4569 { 4570 uint32_t mqd_reg; 4571 uint32_t *mqd_data; 4572 4573 /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */ 4574 mqd_data = &mqd->cp_mqd_base_addr_lo; 4575 4576 /* disable wptr polling */ 4577 WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0); 4578 4579 /* program all HQD registers */ 4580 for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++) 4581 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4582 4583 /* Tonga errata: EOP RPTR/WPTR should be left unmodified. 4584 * This is safe since EOP RPTR==WPTR for any inactive HQD 4585 * on ASICs that do not support context-save. 4586 * EOP writes/reads can start anywhere in the ring. 4587 */ 4588 if (adev->asic_type != CHIP_TONGA) { 4589 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr); 4590 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr); 4591 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem); 4592 } 4593 4594 for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++) 4595 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4596 4597 /* activate the HQD */ 4598 for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++) 4599 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4600 4601 return 0; 4602 } 4603 4604 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) 4605 { 4606 struct amdgpu_device *adev = ring->adev; 4607 struct vi_mqd *mqd = ring->mqd_ptr; 4608 4609 gfx_v8_0_kiq_setting(ring); 4610 4611 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4612 /* reset MQD to a clean status */ 4613 if (adev->gfx.kiq[0].mqd_backup) 4614 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation)); 4615 4616 /* reset ring buffer */ 4617 ring->wptr = 0; 4618 amdgpu_ring_clear_ring(ring); 4619 mutex_lock(&adev->srbm_mutex); 4620 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4621 gfx_v8_0_mqd_commit(adev, mqd); 4622 vi_srbm_select(adev, 0, 0, 0, 0); 4623 mutex_unlock(&adev->srbm_mutex); 4624 } else { 4625 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4626 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4627 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4628 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4629 amdgpu_ring_clear_ring(ring); 4630 mutex_lock(&adev->srbm_mutex); 4631 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4632 gfx_v8_0_mqd_init(ring); 4633 gfx_v8_0_mqd_commit(adev, mqd); 4634 vi_srbm_select(adev, 0, 0, 0, 0); 4635 mutex_unlock(&adev->srbm_mutex); 4636 4637 if (adev->gfx.kiq[0].mqd_backup) 4638 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation)); 4639 } 4640 4641 return 0; 4642 } 4643 4644 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) 4645 { 4646 struct amdgpu_device *adev = ring->adev; 4647 struct vi_mqd *mqd = ring->mqd_ptr; 4648 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4649 4650 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4651 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4652 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4653 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4654 mutex_lock(&adev->srbm_mutex); 4655 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4656 gfx_v8_0_mqd_init(ring); 4657 vi_srbm_select(adev, 0, 0, 0, 0); 4658 mutex_unlock(&adev->srbm_mutex); 4659 4660 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4661 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); 4662 } else { 4663 /* restore MQD to a clean status */ 4664 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4665 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); 4666 /* reset ring buffer */ 4667 ring->wptr = 0; 4668 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4669 amdgpu_ring_clear_ring(ring); 4670 } 4671 return 0; 4672 } 4673 4674 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev) 4675 { 4676 if (adev->asic_type > CHIP_TONGA) { 4677 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2); 4678 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2); 4679 } 4680 /* enable doorbells */ 4681 WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4682 } 4683 4684 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev) 4685 { 4686 gfx_v8_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 4687 return 0; 4688 } 4689 4690 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev) 4691 { 4692 int i, r; 4693 4694 gfx_v8_0_cp_compute_enable(adev, true); 4695 4696 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4697 r = gfx_v8_0_kcq_init_queue(&adev->gfx.compute_ring[i]); 4698 if (r) 4699 return r; 4700 } 4701 4702 gfx_v8_0_set_mec_doorbell_range(adev); 4703 4704 return gfx_v8_0_kiq_kcq_enable(adev); 4705 } 4706 4707 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev) 4708 { 4709 int r, i; 4710 struct amdgpu_ring *ring; 4711 4712 /* collect all the ring_tests here, gfx, kiq, compute */ 4713 ring = &adev->gfx.gfx_ring[0]; 4714 r = amdgpu_ring_test_helper(ring); 4715 if (r) 4716 return r; 4717 4718 ring = &adev->gfx.kiq[0].ring; 4719 r = amdgpu_ring_test_helper(ring); 4720 if (r) 4721 return r; 4722 4723 r = 0; 4724 4725 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4726 ring = &adev->gfx.compute_ring[i]; 4727 r |= amdgpu_ring_test_helper(ring); 4728 } 4729 4730 return r; 4731 } 4732 4733 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev) 4734 { 4735 int r; 4736 4737 if (!(adev->flags & AMD_IS_APU)) 4738 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4739 4740 r = gfx_v8_0_kiq_resume(adev); 4741 if (r) 4742 return r; 4743 4744 r = gfx_v8_0_cp_gfx_resume(adev); 4745 if (r) 4746 return r; 4747 4748 r = gfx_v8_0_kcq_resume(adev); 4749 if (r) 4750 return r; 4751 4752 r = gfx_v8_0_cp_test_all_rings(adev); 4753 if (r) 4754 return r; 4755 4756 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4757 4758 return 0; 4759 } 4760 4761 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable) 4762 { 4763 gfx_v8_0_cp_gfx_enable(adev, enable); 4764 gfx_v8_0_cp_compute_enable(adev, enable); 4765 } 4766 4767 static int gfx_v8_0_hw_init(struct amdgpu_ip_block *ip_block) 4768 { 4769 int r; 4770 struct amdgpu_device *adev = ip_block->adev; 4771 4772 gfx_v8_0_init_golden_registers(adev); 4773 gfx_v8_0_constants_init(adev); 4774 4775 r = adev->gfx.rlc.funcs->resume(adev); 4776 if (r) 4777 return r; 4778 4779 r = gfx_v8_0_cp_resume(adev); 4780 4781 return r; 4782 } 4783 4784 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev) 4785 { 4786 int r, i; 4787 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 4788 4789 r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings); 4790 if (r) 4791 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4792 4793 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4794 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4795 4796 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 4797 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 4798 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */ 4799 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 4800 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) | 4801 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 4802 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 4803 amdgpu_ring_write(kiq_ring, 0); 4804 amdgpu_ring_write(kiq_ring, 0); 4805 amdgpu_ring_write(kiq_ring, 0); 4806 } 4807 /* Submit unmap queue packet */ 4808 amdgpu_ring_commit(kiq_ring); 4809 /* 4810 * Ring test will do a basic scratch register change check. Just run 4811 * this to ensure that unmap queues that is submitted before got 4812 * processed successfully before returning. 4813 */ 4814 r = amdgpu_ring_test_helper(kiq_ring); 4815 if (r) 4816 DRM_ERROR("KCQ disable failed\n"); 4817 4818 return r; 4819 } 4820 4821 static bool gfx_v8_0_is_idle(struct amdgpu_ip_block *ip_block) 4822 { 4823 struct amdgpu_device *adev = ip_block->adev; 4824 4825 if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE) 4826 || RREG32(mmGRBM_STATUS2) != 0x8) 4827 return false; 4828 else 4829 return true; 4830 } 4831 4832 static bool gfx_v8_0_rlc_is_idle(void *handle) 4833 { 4834 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4835 4836 if (RREG32(mmGRBM_STATUS2) != 0x8) 4837 return false; 4838 else 4839 return true; 4840 } 4841 4842 static int gfx_v8_0_wait_for_rlc_idle(void *handle) 4843 { 4844 unsigned int i; 4845 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4846 4847 for (i = 0; i < adev->usec_timeout; i++) { 4848 if (gfx_v8_0_rlc_is_idle(handle)) 4849 return 0; 4850 4851 udelay(1); 4852 } 4853 return -ETIMEDOUT; 4854 } 4855 4856 static int gfx_v8_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 4857 { 4858 unsigned int i; 4859 struct amdgpu_device *adev = ip_block->adev; 4860 4861 for (i = 0; i < adev->usec_timeout; i++) { 4862 if (gfx_v8_0_is_idle(ip_block)) 4863 return 0; 4864 4865 udelay(1); 4866 } 4867 return -ETIMEDOUT; 4868 } 4869 4870 static int gfx_v8_0_hw_fini(struct amdgpu_ip_block *ip_block) 4871 { 4872 struct amdgpu_device *adev = ip_block->adev; 4873 4874 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4875 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4876 4877 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 4878 4879 amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0); 4880 4881 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4882 gfx_v8_0_kcq_disable(adev); 4883 4884 if (amdgpu_sriov_vf(adev)) { 4885 pr_debug("For SRIOV client, shouldn't do anything.\n"); 4886 return 0; 4887 } 4888 4889 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4890 if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_idle(ip_block)) 4891 pr_err("cp is busy, skip halt cp\n"); 4892 if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_rlc_idle(adev)) 4893 pr_err("rlc is busy\n"); 4894 gfx_v8_0_cp_enable(adev, false); 4895 adev->gfx.rlc.funcs->stop(adev); 4896 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4897 4898 return 0; 4899 } 4900 4901 static int gfx_v8_0_suspend(struct amdgpu_ip_block *ip_block) 4902 { 4903 return gfx_v8_0_hw_fini(ip_block); 4904 } 4905 4906 static int gfx_v8_0_resume(struct amdgpu_ip_block *ip_block) 4907 { 4908 return gfx_v8_0_hw_init(ip_block); 4909 } 4910 4911 static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) 4912 { 4913 struct amdgpu_device *adev = ip_block->adev; 4914 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 4915 u32 tmp; 4916 int i; 4917 int r; 4918 4919 grbm_soft_reset = 4920 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1) | 4921 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_GFX, 1) | 4922 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CP, 1) | 4923 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPF, 1) | 4924 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPC, 1) | 4925 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPG, 1); 4926 4927 srbm_soft_reset = 4928 REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1) | 4929 REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_SEM, 1); 4930 4931 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4932 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4933 4934 mutex_lock(&adev->srbm_mutex); 4935 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4936 gfx_v8_0_deactivate_hqd(adev, 2); 4937 vi_srbm_select(adev, 0, 0, 0, 0); 4938 mutex_unlock(&adev->srbm_mutex); 4939 4940 udelay(50); 4941 } 4942 4943 ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE); 4944 ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_UNGATE); 4945 ip_block->version->funcs->suspend(ip_block); 4946 4947 if (grbm_soft_reset || srbm_soft_reset) { 4948 tmp = RREG32(mmGMCON_DEBUG); 4949 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1); 4950 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1); 4951 WREG32(mmGMCON_DEBUG, tmp); 4952 4953 udelay(100); 4954 } 4955 4956 if (grbm_soft_reset) { 4957 tmp = RREG32(mmGRBM_SOFT_RESET); 4958 tmp |= grbm_soft_reset; 4959 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4960 WREG32(mmGRBM_SOFT_RESET, tmp); 4961 tmp = RREG32(mmGRBM_SOFT_RESET); 4962 4963 udelay(100); 4964 4965 tmp &= ~grbm_soft_reset; 4966 WREG32(mmGRBM_SOFT_RESET, tmp); 4967 tmp = RREG32(mmGRBM_SOFT_RESET); 4968 4969 udelay(100); 4970 } 4971 4972 if (srbm_soft_reset) { 4973 tmp = RREG32(mmSRBM_SOFT_RESET); 4974 tmp |= srbm_soft_reset; 4975 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 4976 WREG32(mmSRBM_SOFT_RESET, tmp); 4977 tmp = RREG32(mmSRBM_SOFT_RESET); 4978 4979 udelay(100); 4980 4981 tmp &= ~srbm_soft_reset; 4982 WREG32(mmSRBM_SOFT_RESET, tmp); 4983 tmp = RREG32(mmSRBM_SOFT_RESET); 4984 4985 udelay(100); 4986 } 4987 4988 if (grbm_soft_reset || srbm_soft_reset) { 4989 tmp = RREG32(mmGMCON_DEBUG); 4990 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0); 4991 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0); 4992 WREG32(mmGMCON_DEBUG, tmp); 4993 } 4994 4995 /* Wait a little for things to settle down */ 4996 udelay(100); 4997 4998 r = ip_block->version->funcs->resume(ip_block); 4999 r |= ip_block->version->funcs->late_init(ip_block); 5000 if (r) 5001 return r; 5002 5003 ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_GATE); 5004 ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_GATE); 5005 5006 return 0; 5007 } 5008 5009 /** 5010 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot 5011 * 5012 * @adev: amdgpu_device pointer 5013 * 5014 * Fetches a GPU clock counter snapshot. 5015 * Returns the 64 bit clock counter snapshot. 5016 */ 5017 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5018 { 5019 uint64_t clock; 5020 5021 mutex_lock(&adev->gfx.gpu_clock_mutex); 5022 WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 5023 clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) | 5024 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 5025 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5026 return clock; 5027 } 5028 5029 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5030 uint32_t vmid, 5031 uint32_t gds_base, uint32_t gds_size, 5032 uint32_t gws_base, uint32_t gws_size, 5033 uint32_t oa_base, uint32_t oa_size) 5034 { 5035 /* GDS Base */ 5036 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5037 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5038 WRITE_DATA_DST_SEL(0))); 5039 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base); 5040 amdgpu_ring_write(ring, 0); 5041 amdgpu_ring_write(ring, gds_base); 5042 5043 /* GDS Size */ 5044 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5045 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5046 WRITE_DATA_DST_SEL(0))); 5047 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size); 5048 amdgpu_ring_write(ring, 0); 5049 amdgpu_ring_write(ring, gds_size); 5050 5051 /* GWS */ 5052 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5053 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5054 WRITE_DATA_DST_SEL(0))); 5055 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws); 5056 amdgpu_ring_write(ring, 0); 5057 amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5058 5059 /* OA */ 5060 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5061 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5062 WRITE_DATA_DST_SEL(0))); 5063 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa); 5064 amdgpu_ring_write(ring, 0); 5065 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); 5066 } 5067 5068 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 5069 { 5070 WREG32(mmSQ_IND_INDEX, 5071 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5072 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5073 (address << SQ_IND_INDEX__INDEX__SHIFT) | 5074 (SQ_IND_INDEX__FORCE_READ_MASK)); 5075 return RREG32(mmSQ_IND_DATA); 5076 } 5077 5078 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 5079 uint32_t wave, uint32_t thread, 5080 uint32_t regno, uint32_t num, uint32_t *out) 5081 { 5082 WREG32(mmSQ_IND_INDEX, 5083 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5084 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5085 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 5086 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 5087 (SQ_IND_INDEX__FORCE_READ_MASK) | 5088 (SQ_IND_INDEX__AUTO_INCR_MASK)); 5089 while (num--) 5090 *(out++) = RREG32(mmSQ_IND_DATA); 5091 } 5092 5093 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 5094 { 5095 /* type 0 wave data */ 5096 dst[(*no_fields)++] = 0; 5097 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 5098 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 5099 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 5100 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 5101 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 5102 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 5103 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 5104 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 5105 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 5106 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 5107 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 5108 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 5109 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO); 5110 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI); 5111 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO); 5112 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI); 5113 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 5114 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 5115 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 5116 } 5117 5118 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 5119 uint32_t wave, uint32_t start, 5120 uint32_t size, uint32_t *dst) 5121 { 5122 wave_read_regs( 5123 adev, simd, wave, 0, 5124 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 5125 } 5126 5127 /** 5128 * gfx_v8_0_get_hdp_flush_mask - get the reference and mask for HDP flush 5129 * 5130 * @ring: amdgpu_ring structure holding ring information 5131 * @ref_and_mask: pointer to store the reference and mask 5132 * @reg_mem_engine: pointer to store the register memory engine 5133 * 5134 * Calculates the reference and mask for HDP flush based on the ring type and me. 5135 */ 5136 static void gfx_v8_0_get_hdp_flush_mask(struct amdgpu_ring *ring, 5137 uint32_t *ref_and_mask, uint32_t *reg_mem_engine) 5138 { 5139 if (!ring || !ref_and_mask || !reg_mem_engine) { 5140 DRM_INFO("%s:invalid params\n", __func__); 5141 return; 5142 } 5143 5144 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) || 5145 (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) { 5146 switch (ring->me) { 5147 case 1: 5148 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe; 5149 break; 5150 case 2: 5151 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe; 5152 break; 5153 default: 5154 return; 5155 } 5156 *reg_mem_engine = 0; 5157 } else { 5158 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK; 5159 *reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */ 5160 } 5161 } 5162 5163 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = { 5164 .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter, 5165 .select_se_sh = &gfx_v8_0_select_se_sh, 5166 .read_wave_data = &gfx_v8_0_read_wave_data, 5167 .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs, 5168 .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q, 5169 .get_hdp_flush_mask = &gfx_v8_0_get_hdp_flush_mask, 5170 }; 5171 5172 static int gfx_v8_0_early_init(struct amdgpu_ip_block *ip_block) 5173 { 5174 struct amdgpu_device *adev = ip_block->adev; 5175 5176 adev->gfx.xcc_mask = 1; 5177 adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; 5178 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5179 AMDGPU_MAX_COMPUTE_RINGS); 5180 adev->gfx.funcs = &gfx_v8_0_gfx_funcs; 5181 gfx_v8_0_set_ring_funcs(adev); 5182 gfx_v8_0_set_irq_funcs(adev); 5183 gfx_v8_0_set_gds_init(adev); 5184 gfx_v8_0_set_rlc_funcs(adev); 5185 5186 return 0; 5187 } 5188 5189 static int gfx_v8_0_late_init(struct amdgpu_ip_block *ip_block) 5190 { 5191 struct amdgpu_device *adev = ip_block->adev; 5192 int r; 5193 5194 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5195 if (r) 5196 return r; 5197 5198 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5199 if (r) 5200 return r; 5201 5202 /* requires IBs so do in late init after IB pool is initialized */ 5203 r = gfx_v8_0_do_edc_gpr_workarounds(adev); 5204 if (r) 5205 return r; 5206 5207 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 5208 if (r) { 5209 DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r); 5210 return r; 5211 } 5212 5213 r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0); 5214 if (r) { 5215 DRM_ERROR( 5216 "amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n", 5217 r); 5218 return r; 5219 } 5220 5221 return 0; 5222 } 5223 5224 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 5225 bool enable) 5226 { 5227 if ((adev->asic_type == CHIP_POLARIS11) || 5228 (adev->asic_type == CHIP_POLARIS12) || 5229 (adev->asic_type == CHIP_VEGAM)) 5230 /* Send msg to SMU via Powerplay */ 5231 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable, 0); 5232 5233 WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0); 5234 } 5235 5236 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 5237 bool enable) 5238 { 5239 WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0); 5240 } 5241 5242 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev, 5243 bool enable) 5244 { 5245 WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0); 5246 } 5247 5248 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 5249 bool enable) 5250 { 5251 WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0); 5252 } 5253 5254 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev, 5255 bool enable) 5256 { 5257 WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0); 5258 5259 /* Read any GFX register to wake up GFX. */ 5260 if (!enable) 5261 RREG32(mmDB_RENDER_CONTROL); 5262 } 5263 5264 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev, 5265 bool enable) 5266 { 5267 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 5268 cz_enable_gfx_cg_power_gating(adev, true); 5269 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 5270 cz_enable_gfx_pipeline_power_gating(adev, true); 5271 } else { 5272 cz_enable_gfx_cg_power_gating(adev, false); 5273 cz_enable_gfx_pipeline_power_gating(adev, false); 5274 } 5275 } 5276 5277 static int gfx_v8_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5278 enum amd_powergating_state state) 5279 { 5280 struct amdgpu_device *adev = ip_block->adev; 5281 bool enable = (state == AMD_PG_STATE_GATE); 5282 5283 if (amdgpu_sriov_vf(adev)) 5284 return 0; 5285 5286 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5287 AMD_PG_SUPPORT_RLC_SMU_HS | 5288 AMD_PG_SUPPORT_CP | 5289 AMD_PG_SUPPORT_GFX_DMG)) 5290 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5291 switch (adev->asic_type) { 5292 case CHIP_CARRIZO: 5293 case CHIP_STONEY: 5294 5295 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5296 cz_enable_sck_slow_down_on_power_up(adev, true); 5297 cz_enable_sck_slow_down_on_power_down(adev, true); 5298 } else { 5299 cz_enable_sck_slow_down_on_power_up(adev, false); 5300 cz_enable_sck_slow_down_on_power_down(adev, false); 5301 } 5302 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5303 cz_enable_cp_power_gating(adev, true); 5304 else 5305 cz_enable_cp_power_gating(adev, false); 5306 5307 cz_update_gfx_cg_power_gating(adev, enable); 5308 5309 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5310 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5311 else 5312 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5313 5314 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5315 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5316 else 5317 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5318 break; 5319 case CHIP_POLARIS11: 5320 case CHIP_POLARIS12: 5321 case CHIP_VEGAM: 5322 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5323 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5324 else 5325 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5326 5327 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5328 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5329 else 5330 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5331 5332 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable) 5333 polaris11_enable_gfx_quick_mg_power_gating(adev, true); 5334 else 5335 polaris11_enable_gfx_quick_mg_power_gating(adev, false); 5336 break; 5337 default: 5338 break; 5339 } 5340 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5341 AMD_PG_SUPPORT_RLC_SMU_HS | 5342 AMD_PG_SUPPORT_CP | 5343 AMD_PG_SUPPORT_GFX_DMG)) 5344 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5345 return 0; 5346 } 5347 5348 static void gfx_v8_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5349 { 5350 struct amdgpu_device *adev = ip_block->adev; 5351 int data; 5352 5353 if (amdgpu_sriov_vf(adev)) 5354 *flags = 0; 5355 5356 /* AMD_CG_SUPPORT_GFX_MGCG */ 5357 data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5358 if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK)) 5359 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5360 5361 /* AMD_CG_SUPPORT_GFX_CGLG */ 5362 data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5363 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5364 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5365 5366 /* AMD_CG_SUPPORT_GFX_CGLS */ 5367 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5368 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5369 5370 /* AMD_CG_SUPPORT_GFX_CGTS */ 5371 data = RREG32(mmCGTS_SM_CTRL_REG); 5372 if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK)) 5373 *flags |= AMD_CG_SUPPORT_GFX_CGTS; 5374 5375 /* AMD_CG_SUPPORT_GFX_CGTS_LS */ 5376 if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK)) 5377 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS; 5378 5379 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5380 data = RREG32(mmRLC_MEM_SLP_CNTL); 5381 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5382 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5383 5384 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5385 data = RREG32(mmCP_MEM_SLP_CNTL); 5386 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5387 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5388 } 5389 5390 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev, 5391 uint32_t reg_addr, uint32_t cmd) 5392 { 5393 uint32_t data; 5394 5395 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 5396 5397 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 5398 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 5399 5400 data = RREG32(mmRLC_SERDES_WR_CTRL); 5401 if (adev->asic_type == CHIP_STONEY) 5402 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5403 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5404 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5405 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5406 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5407 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5408 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5409 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5410 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5411 else 5412 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5413 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5414 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5415 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5416 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5417 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5418 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5419 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5420 RLC_SERDES_WR_CTRL__BPM_DATA_MASK | 5421 RLC_SERDES_WR_CTRL__REG_ADDR_MASK | 5422 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5423 data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK | 5424 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) | 5425 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) | 5426 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT)); 5427 5428 WREG32(mmRLC_SERDES_WR_CTRL, data); 5429 } 5430 5431 #define MSG_ENTER_RLC_SAFE_MODE 1 5432 #define MSG_EXIT_RLC_SAFE_MODE 0 5433 #define RLC_GPR_REG2__REQ_MASK 0x00000001 5434 #define RLC_GPR_REG2__REQ__SHIFT 0 5435 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001 5436 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e 5437 5438 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev) 5439 { 5440 uint32_t rlc_setting; 5441 5442 rlc_setting = RREG32(mmRLC_CNTL); 5443 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 5444 return false; 5445 5446 return true; 5447 } 5448 5449 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5450 { 5451 uint32_t data; 5452 unsigned i; 5453 data = RREG32(mmRLC_CNTL); 5454 data |= RLC_SAFE_MODE__CMD_MASK; 5455 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5456 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5457 WREG32(mmRLC_SAFE_MODE, data); 5458 5459 /* wait for RLC_SAFE_MODE */ 5460 for (i = 0; i < adev->usec_timeout; i++) { 5461 if ((RREG32(mmRLC_GPM_STAT) & 5462 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5463 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) == 5464 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5465 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) 5466 break; 5467 udelay(1); 5468 } 5469 for (i = 0; i < adev->usec_timeout; i++) { 5470 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5471 break; 5472 udelay(1); 5473 } 5474 } 5475 5476 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5477 { 5478 uint32_t data; 5479 unsigned i; 5480 5481 data = RREG32(mmRLC_CNTL); 5482 data |= RLC_SAFE_MODE__CMD_MASK; 5483 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5484 WREG32(mmRLC_SAFE_MODE, data); 5485 5486 for (i = 0; i < adev->usec_timeout; i++) { 5487 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5488 break; 5489 udelay(1); 5490 } 5491 } 5492 5493 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5494 struct amdgpu_ring *ring, unsigned vmid) 5495 { 5496 u32 data; 5497 5498 amdgpu_gfx_off_ctrl(adev, false); 5499 5500 if (amdgpu_sriov_is_pp_one_vf(adev)) 5501 data = RREG32_NO_KIQ(mmRLC_SPM_VMID); 5502 else 5503 data = RREG32(mmRLC_SPM_VMID); 5504 5505 data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK; 5506 data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT; 5507 5508 if (amdgpu_sriov_is_pp_one_vf(adev)) 5509 WREG32_NO_KIQ(mmRLC_SPM_VMID, data); 5510 else 5511 WREG32(mmRLC_SPM_VMID, data); 5512 5513 amdgpu_gfx_off_ctrl(adev, true); 5514 } 5515 5516 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = { 5517 .is_rlc_enabled = gfx_v8_0_is_rlc_enabled, 5518 .set_safe_mode = gfx_v8_0_set_safe_mode, 5519 .unset_safe_mode = gfx_v8_0_unset_safe_mode, 5520 .init = gfx_v8_0_rlc_init, 5521 .get_csb_size = gfx_v8_0_get_csb_size, 5522 .get_csb_buffer = gfx_v8_0_get_csb_buffer, 5523 .get_cp_table_num = gfx_v8_0_cp_jump_table_num, 5524 .resume = gfx_v8_0_rlc_resume, 5525 .stop = gfx_v8_0_rlc_stop, 5526 .reset = gfx_v8_0_rlc_reset, 5527 .start = gfx_v8_0_rlc_start, 5528 .update_spm_vmid = gfx_v8_0_update_spm_vmid 5529 }; 5530 5531 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5532 bool enable) 5533 { 5534 uint32_t temp, data; 5535 5536 /* It is disabled by HW by default */ 5537 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 5538 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5539 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) 5540 /* 1 - RLC memory Light sleep */ 5541 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1); 5542 5543 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) 5544 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1); 5545 } 5546 5547 /* 3 - RLC_CGTT_MGCG_OVERRIDE */ 5548 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5549 if (adev->flags & AMD_IS_APU) 5550 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5551 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5552 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK); 5553 else 5554 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5555 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5556 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5557 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5558 5559 if (temp != data) 5560 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5561 5562 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5563 gfx_v8_0_wait_for_rlc_serdes(adev); 5564 5565 /* 5 - clear mgcg override */ 5566 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5567 5568 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) { 5569 /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */ 5570 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5571 data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK); 5572 data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT); 5573 data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK; 5574 data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK; 5575 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) && 5576 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS)) 5577 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK; 5578 data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK; 5579 data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT); 5580 if (temp != data) 5581 WREG32(mmCGTS_SM_CTRL_REG, data); 5582 } 5583 udelay(50); 5584 5585 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5586 gfx_v8_0_wait_for_rlc_serdes(adev); 5587 } else { 5588 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */ 5589 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5590 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5591 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5592 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5593 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5594 if (temp != data) 5595 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5596 5597 /* 2 - disable MGLS in RLC */ 5598 data = RREG32(mmRLC_MEM_SLP_CNTL); 5599 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 5600 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 5601 WREG32(mmRLC_MEM_SLP_CNTL, data); 5602 } 5603 5604 /* 3 - disable MGLS in CP */ 5605 data = RREG32(mmCP_MEM_SLP_CNTL); 5606 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 5607 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 5608 WREG32(mmCP_MEM_SLP_CNTL, data); 5609 } 5610 5611 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */ 5612 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5613 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK | 5614 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK); 5615 if (temp != data) 5616 WREG32(mmCGTS_SM_CTRL_REG, data); 5617 5618 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5619 gfx_v8_0_wait_for_rlc_serdes(adev); 5620 5621 /* 6 - set mgcg override */ 5622 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5623 5624 udelay(50); 5625 5626 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5627 gfx_v8_0_wait_for_rlc_serdes(adev); 5628 } 5629 } 5630 5631 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5632 bool enable) 5633 { 5634 uint32_t temp, temp1, data, data1; 5635 5636 temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5637 5638 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5639 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5640 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK; 5641 if (temp1 != data1) 5642 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5643 5644 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5645 gfx_v8_0_wait_for_rlc_serdes(adev); 5646 5647 /* 2 - clear cgcg override */ 5648 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5649 5650 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5651 gfx_v8_0_wait_for_rlc_serdes(adev); 5652 5653 /* 3 - write cmd to set CGLS */ 5654 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD); 5655 5656 /* 4 - enable cgcg */ 5657 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5658 5659 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5660 /* enable cgls*/ 5661 data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5662 5663 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5664 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK; 5665 5666 if (temp1 != data1) 5667 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5668 } else { 5669 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5670 } 5671 5672 if (temp != data) 5673 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5674 5675 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/ 5676 * Cmp_busy/GFX_Idle interrupts 5677 */ 5678 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5679 } else { 5680 /* disable cntx_empty_int_enable & GFX Idle interrupt */ 5681 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 5682 5683 /* TEST CGCG */ 5684 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5685 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK | 5686 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK); 5687 if (temp1 != data1) 5688 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5689 5690 /* read gfx register to wake up cgcg */ 5691 RREG32(mmCB_CGTT_SCLK_CTRL); 5692 RREG32(mmCB_CGTT_SCLK_CTRL); 5693 RREG32(mmCB_CGTT_SCLK_CTRL); 5694 RREG32(mmCB_CGTT_SCLK_CTRL); 5695 5696 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5697 gfx_v8_0_wait_for_rlc_serdes(adev); 5698 5699 /* write cmd to Set CGCG Override */ 5700 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5701 5702 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5703 gfx_v8_0_wait_for_rlc_serdes(adev); 5704 5705 /* write cmd to Clear CGLS */ 5706 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD); 5707 5708 /* disable cgcg, cgls should be disabled too. */ 5709 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | 5710 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5711 if (temp != data) 5712 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5713 /* enable interrupts again for PG */ 5714 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5715 } 5716 5717 gfx_v8_0_wait_for_rlc_serdes(adev); 5718 } 5719 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5720 bool enable) 5721 { 5722 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5723 5724 if (enable) { 5725 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS) 5726 * === MGCG + MGLS + TS(CG/LS) === 5727 */ 5728 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5729 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5730 } else { 5731 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS) 5732 * === CGCG + CGLS === 5733 */ 5734 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5735 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5736 } 5737 5738 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5739 return 0; 5740 } 5741 5742 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev, 5743 enum amd_clockgating_state state) 5744 { 5745 uint32_t msg_id, pp_state = 0; 5746 uint32_t pp_support_state = 0; 5747 5748 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5749 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5750 pp_support_state = PP_STATE_SUPPORT_LS; 5751 pp_state = PP_STATE_LS; 5752 } 5753 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5754 pp_support_state |= PP_STATE_SUPPORT_CG; 5755 pp_state |= PP_STATE_CG; 5756 } 5757 if (state == AMD_CG_STATE_UNGATE) 5758 pp_state = 0; 5759 5760 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5761 PP_BLOCK_GFX_CG, 5762 pp_support_state, 5763 pp_state); 5764 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5765 } 5766 5767 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5768 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5769 pp_support_state = PP_STATE_SUPPORT_LS; 5770 pp_state = PP_STATE_LS; 5771 } 5772 5773 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5774 pp_support_state |= PP_STATE_SUPPORT_CG; 5775 pp_state |= PP_STATE_CG; 5776 } 5777 5778 if (state == AMD_CG_STATE_UNGATE) 5779 pp_state = 0; 5780 5781 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5782 PP_BLOCK_GFX_MG, 5783 pp_support_state, 5784 pp_state); 5785 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5786 } 5787 5788 return 0; 5789 } 5790 5791 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, 5792 enum amd_clockgating_state state) 5793 { 5794 5795 uint32_t msg_id, pp_state = 0; 5796 uint32_t pp_support_state = 0; 5797 5798 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5799 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5800 pp_support_state = PP_STATE_SUPPORT_LS; 5801 pp_state = PP_STATE_LS; 5802 } 5803 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5804 pp_support_state |= PP_STATE_SUPPORT_CG; 5805 pp_state |= PP_STATE_CG; 5806 } 5807 if (state == AMD_CG_STATE_UNGATE) 5808 pp_state = 0; 5809 5810 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5811 PP_BLOCK_GFX_CG, 5812 pp_support_state, 5813 pp_state); 5814 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5815 } 5816 5817 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) { 5818 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5819 pp_support_state = PP_STATE_SUPPORT_LS; 5820 pp_state = PP_STATE_LS; 5821 } 5822 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5823 pp_support_state |= PP_STATE_SUPPORT_CG; 5824 pp_state |= PP_STATE_CG; 5825 } 5826 if (state == AMD_CG_STATE_UNGATE) 5827 pp_state = 0; 5828 5829 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5830 PP_BLOCK_GFX_3D, 5831 pp_support_state, 5832 pp_state); 5833 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5834 } 5835 5836 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5837 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5838 pp_support_state = PP_STATE_SUPPORT_LS; 5839 pp_state = PP_STATE_LS; 5840 } 5841 5842 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5843 pp_support_state |= PP_STATE_SUPPORT_CG; 5844 pp_state |= PP_STATE_CG; 5845 } 5846 5847 if (state == AMD_CG_STATE_UNGATE) 5848 pp_state = 0; 5849 5850 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5851 PP_BLOCK_GFX_MG, 5852 pp_support_state, 5853 pp_state); 5854 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5855 } 5856 5857 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 5858 pp_support_state = PP_STATE_SUPPORT_LS; 5859 5860 if (state == AMD_CG_STATE_UNGATE) 5861 pp_state = 0; 5862 else 5863 pp_state = PP_STATE_LS; 5864 5865 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5866 PP_BLOCK_GFX_RLC, 5867 pp_support_state, 5868 pp_state); 5869 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5870 } 5871 5872 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 5873 pp_support_state = PP_STATE_SUPPORT_LS; 5874 5875 if (state == AMD_CG_STATE_UNGATE) 5876 pp_state = 0; 5877 else 5878 pp_state = PP_STATE_LS; 5879 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5880 PP_BLOCK_GFX_CP, 5881 pp_support_state, 5882 pp_state); 5883 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5884 } 5885 5886 return 0; 5887 } 5888 5889 static int gfx_v8_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5890 enum amd_clockgating_state state) 5891 { 5892 struct amdgpu_device *adev = ip_block->adev; 5893 5894 if (amdgpu_sriov_vf(adev)) 5895 return 0; 5896 5897 switch (adev->asic_type) { 5898 case CHIP_FIJI: 5899 case CHIP_CARRIZO: 5900 case CHIP_STONEY: 5901 gfx_v8_0_update_gfx_clock_gating(adev, 5902 state == AMD_CG_STATE_GATE); 5903 break; 5904 case CHIP_TONGA: 5905 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state); 5906 break; 5907 case CHIP_POLARIS10: 5908 case CHIP_POLARIS11: 5909 case CHIP_POLARIS12: 5910 case CHIP_VEGAM: 5911 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state); 5912 break; 5913 default: 5914 break; 5915 } 5916 return 0; 5917 } 5918 5919 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring) 5920 { 5921 return *ring->rptr_cpu_addr; 5922 } 5923 5924 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5925 { 5926 struct amdgpu_device *adev = ring->adev; 5927 5928 if (ring->use_doorbell) 5929 /* XXX check if swapping is necessary on BE */ 5930 return *ring->wptr_cpu_addr; 5931 else 5932 return RREG32(mmCP_RB0_WPTR); 5933 } 5934 5935 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5936 { 5937 struct amdgpu_device *adev = ring->adev; 5938 5939 if (ring->use_doorbell) { 5940 /* XXX check if swapping is necessary on BE */ 5941 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 5942 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 5943 } else { 5944 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5945 (void)RREG32(mmCP_RB0_WPTR); 5946 } 5947 } 5948 5949 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5950 { 5951 u32 ref_and_mask, reg_mem_engine; 5952 struct amdgpu_device *adev = ring->adev; 5953 5954 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5955 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5956 return; 5957 } 5958 5959 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 5960 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 5961 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 5962 WAIT_REG_MEM_FUNCTION(3) | /* == */ 5963 reg_mem_engine)); 5964 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ); 5965 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE); 5966 amdgpu_ring_write(ring, ref_and_mask); 5967 amdgpu_ring_write(ring, ref_and_mask); 5968 amdgpu_ring_write(ring, 0x20); /* poll interval */ 5969 } 5970 5971 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) 5972 { 5973 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 5974 amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) | 5975 EVENT_INDEX(4)); 5976 5977 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 5978 amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) | 5979 EVENT_INDEX(0)); 5980 } 5981 5982 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5983 struct amdgpu_job *job, 5984 struct amdgpu_ib *ib, 5985 uint32_t flags) 5986 { 5987 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5988 u32 header, control = 0; 5989 5990 if (ib->flags & AMDGPU_IB_FLAG_CE) 5991 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5992 else 5993 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5994 5995 control |= ib->length_dw | (vmid << 24); 5996 5997 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5998 control |= INDIRECT_BUFFER_PRE_ENB(1); 5999 6000 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 6001 gfx_v8_0_ring_emit_de_meta(ring); 6002 } 6003 6004 amdgpu_ring_write(ring, header); 6005 amdgpu_ring_write(ring, 6006 #ifdef __BIG_ENDIAN 6007 (2 << 0) | 6008 #endif 6009 (ib->gpu_addr & 0xFFFFFFFC)); 6010 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6011 amdgpu_ring_write(ring, control); 6012 } 6013 6014 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 6015 struct amdgpu_job *job, 6016 struct amdgpu_ib *ib, 6017 uint32_t flags) 6018 { 6019 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6020 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 6021 6022 /* Currently, there is a high possibility to get wave ID mismatch 6023 * between ME and GDS, leading to a hw deadlock, because ME generates 6024 * different wave IDs than the GDS expects. This situation happens 6025 * randomly when at least 5 compute pipes use GDS ordered append. 6026 * The wave IDs generated by ME are also wrong after suspend/resume. 6027 * Those are probably bugs somewhere else in the kernel driver. 6028 * 6029 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6030 * GDS to 0 for this ring (me/pipe). 6031 */ 6032 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6033 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6034 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); 6035 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6036 } 6037 6038 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6039 amdgpu_ring_write(ring, 6040 #ifdef __BIG_ENDIAN 6041 (2 << 0) | 6042 #endif 6043 (ib->gpu_addr & 0xFFFFFFFC)); 6044 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6045 amdgpu_ring_write(ring, control); 6046 } 6047 6048 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr, 6049 u64 seq, unsigned flags) 6050 { 6051 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6052 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6053 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC; 6054 6055 /* Workaround for cache flush problems. First send a dummy EOP 6056 * event down the pipe with seq one below. 6057 */ 6058 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6059 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6060 EOP_TC_ACTION_EN | 6061 EOP_TC_WB_ACTION_EN | 6062 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6063 EVENT_INDEX(5))); 6064 amdgpu_ring_write(ring, addr & 0xfffffffc); 6065 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6066 DATA_SEL(1) | INT_SEL(0)); 6067 amdgpu_ring_write(ring, lower_32_bits(seq - 1)); 6068 amdgpu_ring_write(ring, upper_32_bits(seq - 1)); 6069 6070 /* Then send the real EOP event down the pipe: 6071 * EVENT_WRITE_EOP - flush caches, send int */ 6072 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6073 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6074 EOP_TC_ACTION_EN | 6075 EOP_TC_WB_ACTION_EN | 6076 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6077 EVENT_INDEX(5) | 6078 (exec ? EOP_EXEC : 0))); 6079 amdgpu_ring_write(ring, addr & 0xfffffffc); 6080 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6081 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6082 amdgpu_ring_write(ring, lower_32_bits(seq)); 6083 amdgpu_ring_write(ring, upper_32_bits(seq)); 6084 6085 } 6086 6087 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6088 { 6089 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6090 uint32_t seq = ring->fence_drv.sync_seq; 6091 uint64_t addr = ring->fence_drv.gpu_addr; 6092 6093 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6094 amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */ 6095 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 6096 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */ 6097 amdgpu_ring_write(ring, addr & 0xfffffffc); 6098 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); 6099 amdgpu_ring_write(ring, seq); 6100 amdgpu_ring_write(ring, 0xffffffff); 6101 amdgpu_ring_write(ring, 4); /* poll interval */ 6102 } 6103 6104 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6105 unsigned vmid, uint64_t pd_addr) 6106 { 6107 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6108 6109 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6110 6111 /* wait for the invalidate to complete */ 6112 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6113 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 6114 WAIT_REG_MEM_FUNCTION(0) | /* always */ 6115 WAIT_REG_MEM_ENGINE(0))); /* me */ 6116 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); 6117 amdgpu_ring_write(ring, 0); 6118 amdgpu_ring_write(ring, 0); /* ref */ 6119 amdgpu_ring_write(ring, 0); /* mask */ 6120 amdgpu_ring_write(ring, 0x20); /* poll interval */ 6121 6122 /* compute doesn't have PFP */ 6123 if (usepfp) { 6124 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6125 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6126 amdgpu_ring_write(ring, 0x0); 6127 } 6128 } 6129 6130 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 6131 { 6132 return *ring->wptr_cpu_addr; 6133 } 6134 6135 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 6136 { 6137 struct amdgpu_device *adev = ring->adev; 6138 6139 /* XXX check if swapping is necessary on BE */ 6140 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 6141 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 6142 } 6143 6144 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, 6145 u64 addr, u64 seq, 6146 unsigned flags) 6147 { 6148 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6149 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6150 6151 /* RELEASE_MEM - flush caches, send int */ 6152 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 6153 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6154 EOP_TC_ACTION_EN | 6155 EOP_TC_WB_ACTION_EN | 6156 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6157 EVENT_INDEX(5))); 6158 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6159 amdgpu_ring_write(ring, addr & 0xfffffffc); 6160 amdgpu_ring_write(ring, upper_32_bits(addr)); 6161 amdgpu_ring_write(ring, lower_32_bits(seq)); 6162 amdgpu_ring_write(ring, upper_32_bits(seq)); 6163 } 6164 6165 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6166 u64 seq, unsigned int flags) 6167 { 6168 /* write fence seq to the "addr" */ 6169 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6170 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6171 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6172 amdgpu_ring_write(ring, lower_32_bits(addr)); 6173 amdgpu_ring_write(ring, upper_32_bits(addr)); 6174 amdgpu_ring_write(ring, lower_32_bits(seq)); 6175 6176 if (flags & AMDGPU_FENCE_FLAG_INT) { 6177 /* set register to trigger INT */ 6178 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6179 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6180 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6181 amdgpu_ring_write(ring, mmCPC_INT_STATUS); 6182 amdgpu_ring_write(ring, 0); 6183 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6184 } 6185 } 6186 6187 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring) 6188 { 6189 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 6190 amdgpu_ring_write(ring, 0); 6191 } 6192 6193 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 6194 { 6195 uint32_t dw2 = 0; 6196 6197 if (amdgpu_sriov_vf(ring->adev)) 6198 gfx_v8_0_ring_emit_ce_meta(ring); 6199 6200 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6201 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6202 gfx_v8_0_ring_emit_vgt_flush(ring); 6203 /* set load_global_config & load_global_uconfig */ 6204 dw2 |= 0x8001; 6205 /* set load_cs_sh_regs */ 6206 dw2 |= 0x01000000; 6207 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6208 dw2 |= 0x10002; 6209 6210 /* set load_ce_ram if preamble presented */ 6211 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 6212 dw2 |= 0x10000000; 6213 } else { 6214 /* still load_ce_ram if this is the first time preamble presented 6215 * although there is no context switch happens. 6216 */ 6217 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 6218 dw2 |= 0x10000000; 6219 } 6220 6221 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6222 amdgpu_ring_write(ring, dw2); 6223 amdgpu_ring_write(ring, 0); 6224 } 6225 6226 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 6227 uint64_t addr) 6228 { 6229 unsigned ret; 6230 6231 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6232 amdgpu_ring_write(ring, lower_32_bits(addr)); 6233 amdgpu_ring_write(ring, upper_32_bits(addr)); 6234 /* discard following DWs if *cond_exec_gpu_addr==0 */ 6235 amdgpu_ring_write(ring, 0); 6236 ret = ring->wptr & ring->buf_mask; 6237 /* patch dummy value later */ 6238 amdgpu_ring_write(ring, 0); 6239 return ret; 6240 } 6241 6242 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6243 uint32_t reg_val_offs) 6244 { 6245 struct amdgpu_device *adev = ring->adev; 6246 6247 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6248 amdgpu_ring_write(ring, 0 | /* src: register*/ 6249 (5 << 8) | /* dst: memory */ 6250 (1 << 20)); /* write confirm */ 6251 amdgpu_ring_write(ring, reg); 6252 amdgpu_ring_write(ring, 0); 6253 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6254 reg_val_offs * 4)); 6255 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6256 reg_val_offs * 4)); 6257 } 6258 6259 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6260 uint32_t val) 6261 { 6262 uint32_t cmd; 6263 6264 switch (ring->funcs->type) { 6265 case AMDGPU_RING_TYPE_GFX: 6266 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6267 break; 6268 case AMDGPU_RING_TYPE_KIQ: 6269 cmd = 1 << 16; /* no inc addr */ 6270 break; 6271 default: 6272 cmd = WR_CONFIRM; 6273 break; 6274 } 6275 6276 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6277 amdgpu_ring_write(ring, cmd); 6278 amdgpu_ring_write(ring, reg); 6279 amdgpu_ring_write(ring, 0); 6280 amdgpu_ring_write(ring, val); 6281 } 6282 6283 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 6284 { 6285 struct amdgpu_device *adev = ring->adev; 6286 uint32_t value = 0; 6287 6288 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 6289 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 6290 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 6291 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 6292 WREG32(mmSQ_CMD, value); 6293 } 6294 6295 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6296 enum amdgpu_interrupt_state state) 6297 { 6298 WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE, 6299 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6300 } 6301 6302 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6303 int me, int pipe, 6304 enum amdgpu_interrupt_state state) 6305 { 6306 u32 mec_int_cntl, mec_int_cntl_reg; 6307 6308 /* 6309 * amdgpu controls only the first MEC. That's why this function only 6310 * handles the setting of interrupts for this specific MEC. All other 6311 * pipes' interrupts are set by amdkfd. 6312 */ 6313 6314 if (me == 1) { 6315 switch (pipe) { 6316 case 0: 6317 mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL; 6318 break; 6319 case 1: 6320 mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL; 6321 break; 6322 case 2: 6323 mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL; 6324 break; 6325 case 3: 6326 mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL; 6327 break; 6328 default: 6329 DRM_DEBUG("invalid pipe %d\n", pipe); 6330 return; 6331 } 6332 } else { 6333 DRM_DEBUG("invalid me %d\n", me); 6334 return; 6335 } 6336 6337 switch (state) { 6338 case AMDGPU_IRQ_STATE_DISABLE: 6339 mec_int_cntl = RREG32(mec_int_cntl_reg); 6340 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6341 WREG32(mec_int_cntl_reg, mec_int_cntl); 6342 break; 6343 case AMDGPU_IRQ_STATE_ENABLE: 6344 mec_int_cntl = RREG32(mec_int_cntl_reg); 6345 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6346 WREG32(mec_int_cntl_reg, mec_int_cntl); 6347 break; 6348 default: 6349 break; 6350 } 6351 } 6352 6353 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6354 struct amdgpu_irq_src *source, 6355 unsigned type, 6356 enum amdgpu_interrupt_state state) 6357 { 6358 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE, 6359 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6360 6361 return 0; 6362 } 6363 6364 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6365 struct amdgpu_irq_src *source, 6366 unsigned type, 6367 enum amdgpu_interrupt_state state) 6368 { 6369 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE, 6370 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6371 6372 return 0; 6373 } 6374 6375 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6376 struct amdgpu_irq_src *src, 6377 unsigned type, 6378 enum amdgpu_interrupt_state state) 6379 { 6380 switch (type) { 6381 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6382 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state); 6383 break; 6384 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6385 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6386 break; 6387 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6388 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6389 break; 6390 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6391 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6392 break; 6393 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6394 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6395 break; 6396 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 6397 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 6398 break; 6399 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 6400 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 6401 break; 6402 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 6403 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 6404 break; 6405 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 6406 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 6407 break; 6408 default: 6409 break; 6410 } 6411 return 0; 6412 } 6413 6414 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev, 6415 struct amdgpu_irq_src *source, 6416 unsigned int type, 6417 enum amdgpu_interrupt_state state) 6418 { 6419 int enable_flag; 6420 6421 switch (state) { 6422 case AMDGPU_IRQ_STATE_DISABLE: 6423 enable_flag = 0; 6424 break; 6425 6426 case AMDGPU_IRQ_STATE_ENABLE: 6427 enable_flag = 1; 6428 break; 6429 6430 default: 6431 return -EINVAL; 6432 } 6433 6434 WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6435 WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6436 WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6437 WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6438 WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6439 WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6440 enable_flag); 6441 WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6442 enable_flag); 6443 WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6444 enable_flag); 6445 WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6446 enable_flag); 6447 WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6448 enable_flag); 6449 WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6450 enable_flag); 6451 WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6452 enable_flag); 6453 WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6454 enable_flag); 6455 6456 return 0; 6457 } 6458 6459 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev, 6460 struct amdgpu_irq_src *source, 6461 unsigned int type, 6462 enum amdgpu_interrupt_state state) 6463 { 6464 int enable_flag; 6465 6466 switch (state) { 6467 case AMDGPU_IRQ_STATE_DISABLE: 6468 enable_flag = 1; 6469 break; 6470 6471 case AMDGPU_IRQ_STATE_ENABLE: 6472 enable_flag = 0; 6473 break; 6474 6475 default: 6476 return -EINVAL; 6477 } 6478 6479 WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL, 6480 enable_flag); 6481 6482 return 0; 6483 } 6484 6485 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev, 6486 struct amdgpu_irq_src *source, 6487 struct amdgpu_iv_entry *entry) 6488 { 6489 int i; 6490 u8 me_id, pipe_id, queue_id; 6491 struct amdgpu_ring *ring; 6492 6493 DRM_DEBUG("IH: CP EOP\n"); 6494 me_id = (entry->ring_id & 0x0c) >> 2; 6495 pipe_id = (entry->ring_id & 0x03) >> 0; 6496 queue_id = (entry->ring_id & 0x70) >> 4; 6497 6498 switch (me_id) { 6499 case 0: 6500 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 6501 break; 6502 case 1: 6503 case 2: 6504 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6505 ring = &adev->gfx.compute_ring[i]; 6506 /* Per-queue interrupt is supported for MEC starting from VI. 6507 * The interrupt can only be enabled/disabled per pipe instead of per queue. 6508 */ 6509 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 6510 amdgpu_fence_process(ring); 6511 } 6512 break; 6513 } 6514 return 0; 6515 } 6516 6517 static void gfx_v8_0_fault(struct amdgpu_device *adev, 6518 struct amdgpu_iv_entry *entry) 6519 { 6520 u8 me_id, pipe_id, queue_id; 6521 struct amdgpu_ring *ring; 6522 int i; 6523 6524 me_id = (entry->ring_id & 0x0c) >> 2; 6525 pipe_id = (entry->ring_id & 0x03) >> 0; 6526 queue_id = (entry->ring_id & 0x70) >> 4; 6527 6528 switch (me_id) { 6529 case 0: 6530 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 6531 break; 6532 case 1: 6533 case 2: 6534 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6535 ring = &adev->gfx.compute_ring[i]; 6536 if (ring->me == me_id && ring->pipe == pipe_id && 6537 ring->queue == queue_id) 6538 drm_sched_fault(&ring->sched); 6539 } 6540 break; 6541 } 6542 } 6543 6544 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev, 6545 struct amdgpu_irq_src *source, 6546 struct amdgpu_iv_entry *entry) 6547 { 6548 DRM_ERROR("Illegal register access in command stream\n"); 6549 gfx_v8_0_fault(adev, entry); 6550 return 0; 6551 } 6552 6553 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev, 6554 struct amdgpu_irq_src *source, 6555 struct amdgpu_iv_entry *entry) 6556 { 6557 DRM_ERROR("Illegal instruction in command stream\n"); 6558 gfx_v8_0_fault(adev, entry); 6559 return 0; 6560 } 6561 6562 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev, 6563 struct amdgpu_irq_src *source, 6564 struct amdgpu_iv_entry *entry) 6565 { 6566 DRM_ERROR("CP EDC/ECC error detected."); 6567 return 0; 6568 } 6569 6570 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data, 6571 bool from_wq) 6572 { 6573 u32 enc, se_id, sh_id, cu_id; 6574 char type[20]; 6575 int sq_edc_source = -1; 6576 6577 enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING); 6578 se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID); 6579 6580 switch (enc) { 6581 case 0: 6582 drm_info(adev_to_drm(adev), "SQ general purpose intr detected:" 6583 "se_id %d, immed_overflow %d, host_reg_overflow %d," 6584 "host_cmd_overflow %d, cmd_timestamp %d," 6585 "reg_timestamp %d, thread_trace_buff_full %d," 6586 "wlt %d, thread_trace %d.\n", 6587 se_id, 6588 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW), 6589 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW), 6590 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW), 6591 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP), 6592 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP), 6593 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL), 6594 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT), 6595 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE) 6596 ); 6597 break; 6598 case 1: 6599 case 2: 6600 6601 cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID); 6602 sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID); 6603 6604 /* 6605 * This function can be called either directly from ISR 6606 * or from BH in which case we can access SQ_EDC_INFO 6607 * instance 6608 */ 6609 if (from_wq) { 6610 mutex_lock(&adev->grbm_idx_mutex); 6611 gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id, 0); 6612 6613 sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE); 6614 6615 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 6616 mutex_unlock(&adev->grbm_idx_mutex); 6617 } 6618 6619 if (enc == 1) 6620 sprintf(type, "instruction intr"); 6621 else 6622 sprintf(type, "EDC/ECC error"); 6623 6624 drm_info(adev_to_drm(adev), 6625 "SQ %s detected: " 6626 "se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d " 6627 "trap %s, sq_ed_info.source %s.\n", 6628 type, se_id, sh_id, cu_id, 6629 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID), 6630 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID), 6631 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID), 6632 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false", 6633 (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable" 6634 ); 6635 break; 6636 default: 6637 DRM_ERROR("SQ invalid encoding type\n."); 6638 } 6639 } 6640 6641 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work) 6642 { 6643 6644 struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work); 6645 struct sq_work *sq_work = container_of(work, struct sq_work, work); 6646 6647 gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true); 6648 } 6649 6650 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev, 6651 struct amdgpu_irq_src *source, 6652 struct amdgpu_iv_entry *entry) 6653 { 6654 unsigned ih_data = entry->src_data[0]; 6655 6656 /* 6657 * Try to submit work so SQ_EDC_INFO can be accessed from 6658 * BH. If previous work submission hasn't finished yet 6659 * just print whatever info is possible directly from the ISR. 6660 */ 6661 if (work_pending(&adev->gfx.sq_work.work)) { 6662 gfx_v8_0_parse_sq_irq(adev, ih_data, false); 6663 } else { 6664 adev->gfx.sq_work.ih_data = ih_data; 6665 schedule_work(&adev->gfx.sq_work.work); 6666 } 6667 6668 return 0; 6669 } 6670 6671 static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring) 6672 { 6673 amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 6674 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6675 PACKET3_TC_ACTION_ENA | 6676 PACKET3_SH_KCACHE_ACTION_ENA | 6677 PACKET3_SH_ICACHE_ACTION_ENA | 6678 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6679 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6680 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6681 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6682 } 6683 6684 static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring) 6685 { 6686 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6687 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6688 PACKET3_TC_ACTION_ENA | 6689 PACKET3_SH_KCACHE_ACTION_ENA | 6690 PACKET3_SH_ICACHE_ACTION_ENA | 6691 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6692 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6693 amdgpu_ring_write(ring, 0xff); /* CP_COHER_SIZE_HI */ 6694 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6695 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6696 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6697 } 6698 6699 6700 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6701 #define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT 0x0000007f 6702 static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6703 uint32_t pipe, bool enable) 6704 { 6705 uint32_t val; 6706 uint32_t wcl_cs_reg; 6707 6708 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT; 6709 6710 switch (pipe) { 6711 case 0: 6712 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0; 6713 break; 6714 case 1: 6715 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1; 6716 break; 6717 case 2: 6718 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2; 6719 break; 6720 case 3: 6721 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3; 6722 break; 6723 default: 6724 DRM_DEBUG("invalid pipe %d\n", pipe); 6725 return; 6726 } 6727 6728 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6729 6730 } 6731 6732 #define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT 0x07ffffff 6733 static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6734 { 6735 struct amdgpu_device *adev = ring->adev; 6736 uint32_t val; 6737 int i; 6738 6739 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6740 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6741 * around 25% of gpu resources. 6742 */ 6743 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6744 amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val); 6745 6746 /* Restrict waves for normal/low priority compute queues as well 6747 * to get best QoS for high priority compute jobs. 6748 * 6749 * amdgpu controls only 1st ME(0-3 CS pipes). 6750 */ 6751 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6752 if (i != ring->pipe) 6753 gfx_v8_0_emit_wave_limit_cs(ring, i, enable); 6754 6755 } 6756 6757 } 6758 6759 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { 6760 .name = "gfx_v8_0", 6761 .early_init = gfx_v8_0_early_init, 6762 .late_init = gfx_v8_0_late_init, 6763 .sw_init = gfx_v8_0_sw_init, 6764 .sw_fini = gfx_v8_0_sw_fini, 6765 .hw_init = gfx_v8_0_hw_init, 6766 .hw_fini = gfx_v8_0_hw_fini, 6767 .suspend = gfx_v8_0_suspend, 6768 .resume = gfx_v8_0_resume, 6769 .wait_for_idle = gfx_v8_0_wait_for_idle, 6770 .soft_reset = gfx_v8_0_soft_reset, 6771 .set_clockgating_state = gfx_v8_0_set_clockgating_state, 6772 .set_powergating_state = gfx_v8_0_set_powergating_state, 6773 .get_clockgating_state = gfx_v8_0_get_clockgating_state, 6774 }; 6775 6776 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { 6777 .type = AMDGPU_RING_TYPE_GFX, 6778 .align_mask = 0xff, 6779 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6780 .support_64bit_ptrs = false, 6781 .get_rptr = gfx_v8_0_ring_get_rptr, 6782 .get_wptr = gfx_v8_0_ring_get_wptr_gfx, 6783 .set_wptr = gfx_v8_0_ring_set_wptr_gfx, 6784 .emit_frame_size = /* maximum 215dw if count 16 IBs in */ 6785 5 + /* COND_EXEC */ 6786 7 + /* PIPELINE_SYNC */ 6787 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */ 6788 12 + /* FENCE for VM_FLUSH */ 6789 20 + /* GDS switch */ 6790 4 + /* double SWITCH_BUFFER, 6791 the first COND_EXEC jump to the place just 6792 prior to this double SWITCH_BUFFER */ 6793 5 + /* COND_EXEC */ 6794 7 + /* HDP_flush */ 6795 4 + /* VGT_flush */ 6796 14 + /* CE_META */ 6797 31 + /* DE_META */ 6798 3 + /* CNTX_CTRL */ 6799 5 + /* HDP_INVL */ 6800 12 + 12 + /* FENCE x2 */ 6801 2 + /* SWITCH_BUFFER */ 6802 5, /* SURFACE_SYNC */ 6803 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */ 6804 .emit_ib = gfx_v8_0_ring_emit_ib_gfx, 6805 .emit_fence = gfx_v8_0_ring_emit_fence_gfx, 6806 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6807 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6808 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6809 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6810 .test_ring = gfx_v8_0_ring_test_ring, 6811 .test_ib = gfx_v8_0_ring_test_ib, 6812 .insert_nop = amdgpu_ring_insert_nop, 6813 .pad_ib = amdgpu_ring_generic_pad_ib, 6814 .emit_switch_buffer = gfx_v8_ring_emit_sb, 6815 .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl, 6816 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, 6817 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6818 .soft_recovery = gfx_v8_0_ring_soft_recovery, 6819 .emit_mem_sync = gfx_v8_0_emit_mem_sync, 6820 }; 6821 6822 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { 6823 .type = AMDGPU_RING_TYPE_COMPUTE, 6824 .align_mask = 0xff, 6825 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6826 .support_64bit_ptrs = false, 6827 .get_rptr = gfx_v8_0_ring_get_rptr, 6828 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6829 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6830 .emit_frame_size = 6831 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_ib_schedule) */ 6832 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6833 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6834 5 + /* hdp_invalidate */ 6835 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6836 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_vm_flush) */ 6837 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ 6838 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ 6839 7 + /* gfx_v8_0_emit_mem_sync_compute */ 6840 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6841 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6842 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6843 .emit_ib = gfx_v8_0_ring_emit_ib_compute, 6844 .emit_fence = gfx_v8_0_ring_emit_fence_compute, 6845 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6846 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6847 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6848 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6849 .test_ring = gfx_v8_0_ring_test_ring, 6850 .test_ib = gfx_v8_0_ring_test_ib, 6851 .insert_nop = amdgpu_ring_insert_nop, 6852 .pad_ib = amdgpu_ring_generic_pad_ib, 6853 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6854 .soft_recovery = gfx_v8_0_ring_soft_recovery, 6855 .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute, 6856 .emit_wave_limit = gfx_v8_0_emit_wave_limit, 6857 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, 6858 }; 6859 6860 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { 6861 .type = AMDGPU_RING_TYPE_KIQ, 6862 .align_mask = 0xff, 6863 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6864 .support_64bit_ptrs = false, 6865 .get_rptr = gfx_v8_0_ring_get_rptr, 6866 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6867 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6868 .emit_frame_size = 6869 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6870 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6871 5 + /* hdp_invalidate */ 6872 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6873 17 + /* gfx_v8_0_ring_emit_vm_flush */ 6874 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6875 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6876 .emit_fence = gfx_v8_0_ring_emit_fence_kiq, 6877 .test_ring = gfx_v8_0_ring_test_ring, 6878 .insert_nop = amdgpu_ring_insert_nop, 6879 .pad_ib = amdgpu_ring_generic_pad_ib, 6880 .emit_rreg = gfx_v8_0_ring_emit_rreg, 6881 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6882 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6883 }; 6884 6885 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev) 6886 { 6887 int i; 6888 6889 adev->gfx.kiq[0].ring.funcs = &gfx_v8_0_ring_funcs_kiq; 6890 6891 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6892 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx; 6893 6894 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6895 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute; 6896 } 6897 6898 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = { 6899 .set = gfx_v8_0_set_eop_interrupt_state, 6900 .process = gfx_v8_0_eop_irq, 6901 }; 6902 6903 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = { 6904 .set = gfx_v8_0_set_priv_reg_fault_state, 6905 .process = gfx_v8_0_priv_reg_irq, 6906 }; 6907 6908 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = { 6909 .set = gfx_v8_0_set_priv_inst_fault_state, 6910 .process = gfx_v8_0_priv_inst_irq, 6911 }; 6912 6913 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = { 6914 .set = gfx_v8_0_set_cp_ecc_int_state, 6915 .process = gfx_v8_0_cp_ecc_error_irq, 6916 }; 6917 6918 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = { 6919 .set = gfx_v8_0_set_sq_int_state, 6920 .process = gfx_v8_0_sq_irq, 6921 }; 6922 6923 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev) 6924 { 6925 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6926 adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs; 6927 6928 adev->gfx.priv_reg_irq.num_types = 1; 6929 adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs; 6930 6931 adev->gfx.priv_inst_irq.num_types = 1; 6932 adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs; 6933 6934 adev->gfx.cp_ecc_error_irq.num_types = 1; 6935 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs; 6936 6937 adev->gfx.sq_irq.num_types = 1; 6938 adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs; 6939 } 6940 6941 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev) 6942 { 6943 adev->gfx.rlc.funcs = &iceland_rlc_funcs; 6944 } 6945 6946 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev) 6947 { 6948 /* init asci gds info */ 6949 adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE); 6950 adev->gds.gws_size = 64; 6951 adev->gds.oa_size = 16; 6952 adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); 6953 } 6954 6955 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6956 u32 bitmap) 6957 { 6958 u32 data; 6959 6960 if (!bitmap) 6961 return; 6962 6963 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6964 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6965 6966 WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data); 6967 } 6968 6969 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6970 { 6971 u32 data, mask; 6972 6973 data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) | 6974 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG); 6975 6976 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6977 6978 return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask; 6979 } 6980 6981 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev) 6982 { 6983 int i, j, k, counter, active_cu_number = 0; 6984 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6985 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 6986 unsigned disable_masks[4 * 2]; 6987 u32 ao_cu_num; 6988 6989 memset(cu_info, 0, sizeof(*cu_info)); 6990 6991 if (adev->flags & AMD_IS_APU) 6992 ao_cu_num = 2; 6993 else 6994 ao_cu_num = adev->gfx.config.max_cu_per_sh; 6995 6996 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 4, 2); 6997 6998 mutex_lock(&adev->grbm_idx_mutex); 6999 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7000 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7001 mask = 1; 7002 ao_bitmap = 0; 7003 counter = 0; 7004 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7005 if (i < 4 && j < 2) 7006 gfx_v8_0_set_user_cu_inactive_bitmap( 7007 adev, disable_masks[i * 2 + j]); 7008 bitmap = gfx_v8_0_get_cu_active_bitmap(adev); 7009 cu_info->bitmap[0][i][j] = bitmap; 7010 7011 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7012 if (bitmap & mask) { 7013 if (counter < ao_cu_num) 7014 ao_bitmap |= mask; 7015 counter ++; 7016 } 7017 mask <<= 1; 7018 } 7019 active_cu_number += counter; 7020 if (i < 2 && j < 2) 7021 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7022 cu_info->ao_cu_bitmap[i][j] = ao_bitmap; 7023 } 7024 } 7025 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7026 mutex_unlock(&adev->grbm_idx_mutex); 7027 7028 cu_info->number = active_cu_number; 7029 cu_info->ao_cu_mask = ao_cu_mask; 7030 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7031 cu_info->max_waves_per_simd = 10; 7032 cu_info->max_scratch_slots_per_cu = 32; 7033 cu_info->wave_front_size = 64; 7034 cu_info->lds_size = 64; 7035 } 7036 7037 const struct amdgpu_ip_block_version gfx_v8_0_ip_block = 7038 { 7039 .type = AMD_IP_BLOCK_TYPE_GFX, 7040 .major = 8, 7041 .minor = 0, 7042 .rev = 0, 7043 .funcs = &gfx_v8_0_ip_funcs, 7044 }; 7045 7046 const struct amdgpu_ip_block_version gfx_v8_1_ip_block = 7047 { 7048 .type = AMD_IP_BLOCK_TYPE_GFX, 7049 .major = 8, 7050 .minor = 1, 7051 .rev = 0, 7052 .funcs = &gfx_v8_0_ip_funcs, 7053 }; 7054 7055 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 7056 { 7057 uint64_t ce_payload_addr; 7058 int cnt_ce; 7059 union { 7060 struct vi_ce_ib_state regular; 7061 struct vi_ce_ib_state_chained_ib chained; 7062 } ce_payload = {}; 7063 7064 if (ring->adev->virt.chained_ib_support) { 7065 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7066 offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload); 7067 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2; 7068 } else { 7069 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7070 offsetof(struct vi_gfx_meta_data, ce_payload); 7071 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2; 7072 } 7073 7074 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce)); 7075 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 7076 WRITE_DATA_DST_SEL(8) | 7077 WR_CONFIRM) | 7078 WRITE_DATA_CACHE_POLICY(0)); 7079 amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr)); 7080 amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr)); 7081 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2); 7082 } 7083 7084 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring) 7085 { 7086 uint64_t de_payload_addr, gds_addr, csa_addr; 7087 int cnt_de; 7088 union { 7089 struct vi_de_ib_state regular; 7090 struct vi_de_ib_state_chained_ib chained; 7091 } de_payload = {}; 7092 7093 csa_addr = amdgpu_csa_vaddr(ring->adev); 7094 gds_addr = csa_addr + 4096; 7095 if (ring->adev->virt.chained_ib_support) { 7096 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr); 7097 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr); 7098 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload); 7099 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2; 7100 } else { 7101 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr); 7102 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr); 7103 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload); 7104 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2; 7105 } 7106 7107 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de)); 7108 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 7109 WRITE_DATA_DST_SEL(8) | 7110 WR_CONFIRM) | 7111 WRITE_DATA_CACHE_POLICY(0)); 7112 amdgpu_ring_write(ring, lower_32_bits(de_payload_addr)); 7113 amdgpu_ring_write(ring, upper_32_bits(de_payload_addr)); 7114 amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2); 7115 } 7116