1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "amdgpu_ring.h" 33 #include "vi.h" 34 #include "vi_structs.h" 35 #include "vid.h" 36 #include "amdgpu_ucode.h" 37 #include "amdgpu_atombios.h" 38 #include "atombios_i2c.h" 39 #include "clearstate_vi.h" 40 41 #include "gmc/gmc_8_2_d.h" 42 #include "gmc/gmc_8_2_sh_mask.h" 43 44 #include "oss/oss_3_0_d.h" 45 #include "oss/oss_3_0_sh_mask.h" 46 47 #include "bif/bif_5_0_d.h" 48 #include "bif/bif_5_0_sh_mask.h" 49 #include "gca/gfx_8_0_d.h" 50 #include "gca/gfx_8_0_enum.h" 51 #include "gca/gfx_8_0_sh_mask.h" 52 53 #include "dce/dce_10_0_d.h" 54 #include "dce/dce_10_0_sh_mask.h" 55 56 #include "smu/smu_7_1_3_d.h" 57 58 #include "ivsrcid/ivsrcid_vislands30.h" 59 60 #define GFX8_NUM_GFX_RINGS 1 61 #define GFX8_MEC_HPD_SIZE 4096 62 63 #define TOPAZ_GB_ADDR_CONFIG_GOLDEN 0x22010001 64 #define CARRIZO_GB_ADDR_CONFIG_GOLDEN 0x22010001 65 #define POLARIS11_GB_ADDR_CONFIG_GOLDEN 0x22011002 66 #define TONGA_GB_ADDR_CONFIG_GOLDEN 0x22011003 67 68 #define ARRAY_MODE(x) ((x) << GB_TILE_MODE0__ARRAY_MODE__SHIFT) 69 #define PIPE_CONFIG(x) ((x) << GB_TILE_MODE0__PIPE_CONFIG__SHIFT) 70 #define TILE_SPLIT(x) ((x) << GB_TILE_MODE0__TILE_SPLIT__SHIFT) 71 #define MICRO_TILE_MODE_NEW(x) ((x) << GB_TILE_MODE0__MICRO_TILE_MODE_NEW__SHIFT) 72 #define SAMPLE_SPLIT(x) ((x) << GB_TILE_MODE0__SAMPLE_SPLIT__SHIFT) 73 #define BANK_WIDTH(x) ((x) << GB_MACROTILE_MODE0__BANK_WIDTH__SHIFT) 74 #define BANK_HEIGHT(x) ((x) << GB_MACROTILE_MODE0__BANK_HEIGHT__SHIFT) 75 #define MACRO_TILE_ASPECT(x) ((x) << GB_MACROTILE_MODE0__MACRO_TILE_ASPECT__SHIFT) 76 #define NUM_BANKS(x) ((x) << GB_MACROTILE_MODE0__NUM_BANKS__SHIFT) 77 78 #define RLC_CGTT_MGCG_OVERRIDE__CPF_MASK 0x00000001L 79 #define RLC_CGTT_MGCG_OVERRIDE__RLC_MASK 0x00000002L 80 #define RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK 0x00000004L 81 #define RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK 0x00000008L 82 #define RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK 0x00000010L 83 #define RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK 0x00000020L 84 85 /* BPM SERDES CMD */ 86 #define SET_BPM_SERDES_CMD 1 87 #define CLE_BPM_SERDES_CMD 0 88 89 /* BPM Register Address*/ 90 enum { 91 BPM_REG_CGLS_EN = 0, /* Enable/Disable CGLS */ 92 BPM_REG_CGLS_ON, /* ON/OFF CGLS: shall be controlled by RLC FW */ 93 BPM_REG_CGCG_OVERRIDE, /* Set/Clear CGCG Override */ 94 BPM_REG_MGCG_OVERRIDE, /* Set/Clear MGCG Override */ 95 BPM_REG_FGCG_OVERRIDE, /* Set/Clear FGCG Override */ 96 BPM_REG_FGCG_MAX 97 }; 98 99 #define RLC_FormatDirectRegListLength 14 100 101 MODULE_FIRMWARE("amdgpu/carrizo_ce.bin"); 102 MODULE_FIRMWARE("amdgpu/carrizo_pfp.bin"); 103 MODULE_FIRMWARE("amdgpu/carrizo_me.bin"); 104 MODULE_FIRMWARE("amdgpu/carrizo_mec.bin"); 105 MODULE_FIRMWARE("amdgpu/carrizo_mec2.bin"); 106 MODULE_FIRMWARE("amdgpu/carrizo_rlc.bin"); 107 108 MODULE_FIRMWARE("amdgpu/stoney_ce.bin"); 109 MODULE_FIRMWARE("amdgpu/stoney_pfp.bin"); 110 MODULE_FIRMWARE("amdgpu/stoney_me.bin"); 111 MODULE_FIRMWARE("amdgpu/stoney_mec.bin"); 112 MODULE_FIRMWARE("amdgpu/stoney_rlc.bin"); 113 114 MODULE_FIRMWARE("amdgpu/tonga_ce.bin"); 115 MODULE_FIRMWARE("amdgpu/tonga_pfp.bin"); 116 MODULE_FIRMWARE("amdgpu/tonga_me.bin"); 117 MODULE_FIRMWARE("amdgpu/tonga_mec.bin"); 118 MODULE_FIRMWARE("amdgpu/tonga_mec2.bin"); 119 MODULE_FIRMWARE("amdgpu/tonga_rlc.bin"); 120 121 MODULE_FIRMWARE("amdgpu/topaz_ce.bin"); 122 MODULE_FIRMWARE("amdgpu/topaz_pfp.bin"); 123 MODULE_FIRMWARE("amdgpu/topaz_me.bin"); 124 MODULE_FIRMWARE("amdgpu/topaz_mec.bin"); 125 MODULE_FIRMWARE("amdgpu/topaz_rlc.bin"); 126 127 MODULE_FIRMWARE("amdgpu/fiji_ce.bin"); 128 MODULE_FIRMWARE("amdgpu/fiji_pfp.bin"); 129 MODULE_FIRMWARE("amdgpu/fiji_me.bin"); 130 MODULE_FIRMWARE("amdgpu/fiji_mec.bin"); 131 MODULE_FIRMWARE("amdgpu/fiji_mec2.bin"); 132 MODULE_FIRMWARE("amdgpu/fiji_rlc.bin"); 133 134 MODULE_FIRMWARE("amdgpu/polaris10_ce.bin"); 135 MODULE_FIRMWARE("amdgpu/polaris10_ce_2.bin"); 136 MODULE_FIRMWARE("amdgpu/polaris10_pfp.bin"); 137 MODULE_FIRMWARE("amdgpu/polaris10_pfp_2.bin"); 138 MODULE_FIRMWARE("amdgpu/polaris10_me.bin"); 139 MODULE_FIRMWARE("amdgpu/polaris10_me_2.bin"); 140 MODULE_FIRMWARE("amdgpu/polaris10_mec.bin"); 141 MODULE_FIRMWARE("amdgpu/polaris10_mec_2.bin"); 142 MODULE_FIRMWARE("amdgpu/polaris10_mec2.bin"); 143 MODULE_FIRMWARE("amdgpu/polaris10_mec2_2.bin"); 144 MODULE_FIRMWARE("amdgpu/polaris10_rlc.bin"); 145 146 MODULE_FIRMWARE("amdgpu/polaris11_ce.bin"); 147 MODULE_FIRMWARE("amdgpu/polaris11_ce_2.bin"); 148 MODULE_FIRMWARE("amdgpu/polaris11_pfp.bin"); 149 MODULE_FIRMWARE("amdgpu/polaris11_pfp_2.bin"); 150 MODULE_FIRMWARE("amdgpu/polaris11_me.bin"); 151 MODULE_FIRMWARE("amdgpu/polaris11_me_2.bin"); 152 MODULE_FIRMWARE("amdgpu/polaris11_mec.bin"); 153 MODULE_FIRMWARE("amdgpu/polaris11_mec_2.bin"); 154 MODULE_FIRMWARE("amdgpu/polaris11_mec2.bin"); 155 MODULE_FIRMWARE("amdgpu/polaris11_mec2_2.bin"); 156 MODULE_FIRMWARE("amdgpu/polaris11_rlc.bin"); 157 158 MODULE_FIRMWARE("amdgpu/polaris12_ce.bin"); 159 MODULE_FIRMWARE("amdgpu/polaris12_ce_2.bin"); 160 MODULE_FIRMWARE("amdgpu/polaris12_pfp.bin"); 161 MODULE_FIRMWARE("amdgpu/polaris12_pfp_2.bin"); 162 MODULE_FIRMWARE("amdgpu/polaris12_me.bin"); 163 MODULE_FIRMWARE("amdgpu/polaris12_me_2.bin"); 164 MODULE_FIRMWARE("amdgpu/polaris12_mec.bin"); 165 MODULE_FIRMWARE("amdgpu/polaris12_mec_2.bin"); 166 MODULE_FIRMWARE("amdgpu/polaris12_mec2.bin"); 167 MODULE_FIRMWARE("amdgpu/polaris12_mec2_2.bin"); 168 MODULE_FIRMWARE("amdgpu/polaris12_rlc.bin"); 169 170 MODULE_FIRMWARE("amdgpu/vegam_ce.bin"); 171 MODULE_FIRMWARE("amdgpu/vegam_pfp.bin"); 172 MODULE_FIRMWARE("amdgpu/vegam_me.bin"); 173 MODULE_FIRMWARE("amdgpu/vegam_mec.bin"); 174 MODULE_FIRMWARE("amdgpu/vegam_mec2.bin"); 175 MODULE_FIRMWARE("amdgpu/vegam_rlc.bin"); 176 177 static const struct amdgpu_gds_reg_offset amdgpu_gds_reg_offset[] = 178 { 179 {mmGDS_VMID0_BASE, mmGDS_VMID0_SIZE, mmGDS_GWS_VMID0, mmGDS_OA_VMID0}, 180 {mmGDS_VMID1_BASE, mmGDS_VMID1_SIZE, mmGDS_GWS_VMID1, mmGDS_OA_VMID1}, 181 {mmGDS_VMID2_BASE, mmGDS_VMID2_SIZE, mmGDS_GWS_VMID2, mmGDS_OA_VMID2}, 182 {mmGDS_VMID3_BASE, mmGDS_VMID3_SIZE, mmGDS_GWS_VMID3, mmGDS_OA_VMID3}, 183 {mmGDS_VMID4_BASE, mmGDS_VMID4_SIZE, mmGDS_GWS_VMID4, mmGDS_OA_VMID4}, 184 {mmGDS_VMID5_BASE, mmGDS_VMID5_SIZE, mmGDS_GWS_VMID5, mmGDS_OA_VMID5}, 185 {mmGDS_VMID6_BASE, mmGDS_VMID6_SIZE, mmGDS_GWS_VMID6, mmGDS_OA_VMID6}, 186 {mmGDS_VMID7_BASE, mmGDS_VMID7_SIZE, mmGDS_GWS_VMID7, mmGDS_OA_VMID7}, 187 {mmGDS_VMID8_BASE, mmGDS_VMID8_SIZE, mmGDS_GWS_VMID8, mmGDS_OA_VMID8}, 188 {mmGDS_VMID9_BASE, mmGDS_VMID9_SIZE, mmGDS_GWS_VMID9, mmGDS_OA_VMID9}, 189 {mmGDS_VMID10_BASE, mmGDS_VMID10_SIZE, mmGDS_GWS_VMID10, mmGDS_OA_VMID10}, 190 {mmGDS_VMID11_BASE, mmGDS_VMID11_SIZE, mmGDS_GWS_VMID11, mmGDS_OA_VMID11}, 191 {mmGDS_VMID12_BASE, mmGDS_VMID12_SIZE, mmGDS_GWS_VMID12, mmGDS_OA_VMID12}, 192 {mmGDS_VMID13_BASE, mmGDS_VMID13_SIZE, mmGDS_GWS_VMID13, mmGDS_OA_VMID13}, 193 {mmGDS_VMID14_BASE, mmGDS_VMID14_SIZE, mmGDS_GWS_VMID14, mmGDS_OA_VMID14}, 194 {mmGDS_VMID15_BASE, mmGDS_VMID15_SIZE, mmGDS_GWS_VMID15, mmGDS_OA_VMID15} 195 }; 196 197 static const u32 golden_settings_tonga_a11[] = 198 { 199 mmCB_HW_CONTROL, 0xfffdf3cf, 0x00007208, 200 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 201 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 202 mmGB_GPU_ID, 0x0000000f, 0x00000000, 203 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 204 mmPA_SC_FIFO_DEPTH_CNTL, 0x000003ff, 0x000000fc, 205 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 206 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 207 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 208 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 209 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 210 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 211 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000002fb, 212 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x0000543b, 213 mmTCP_CHAN_STEER_LO, 0xffffffff, 0xa9210876, 214 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 215 }; 216 217 static const u32 tonga_golden_common_all[] = 218 { 219 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 220 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 221 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 222 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 223 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 224 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 225 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 226 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 227 }; 228 229 static const u32 tonga_mgcg_cgcg_init[] = 230 { 231 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 232 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 233 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 234 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 235 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 236 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 237 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 238 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 239 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 240 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 241 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 242 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 243 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 244 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 245 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 246 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 247 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 248 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 249 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 250 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 251 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 252 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 253 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 254 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 255 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 256 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 257 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 258 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 259 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 260 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 261 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 262 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 263 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 264 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 265 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 266 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 267 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 268 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 269 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 270 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 271 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 272 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 273 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 274 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 275 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 276 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 277 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 278 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 279 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 280 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 281 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 282 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 283 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 284 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 285 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 286 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 287 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 288 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 289 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 290 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 291 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 292 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 293 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 294 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 295 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 296 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 297 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 298 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 299 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 300 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 301 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 302 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 303 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 304 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 305 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 306 }; 307 308 static const u32 golden_settings_vegam_a11[] = 309 { 310 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 311 mmCB_HW_CONTROL_2, 0x0f000000, 0x0d000000, 312 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 313 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 314 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 315 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 316 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x3a00161a, 317 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002e, 318 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 319 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 320 mmSQ_CONFIG, 0x07f80000, 0x01180000, 321 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 322 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 323 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 324 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 325 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x32761054, 326 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 327 }; 328 329 static const u32 vegam_golden_common_all[] = 330 { 331 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 332 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 333 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 334 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 335 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 336 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 337 }; 338 339 static const u32 golden_settings_polaris11_a11[] = 340 { 341 mmCB_HW_CONTROL, 0x0000f3cf, 0x00007208, 342 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 343 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 344 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 345 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 346 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 347 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 348 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 349 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 350 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 351 mmSQ_CONFIG, 0x07f80000, 0x01180000, 352 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 353 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 354 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f3, 355 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 356 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003210, 357 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 358 }; 359 360 static const u32 polaris11_golden_common_all[] = 361 { 362 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 363 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011002, 364 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 365 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 366 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 367 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 368 }; 369 370 static const u32 golden_settings_polaris10_a11[] = 371 { 372 mmATC_MISC_CG, 0x000c0fc0, 0x000c0200, 373 mmCB_HW_CONTROL, 0x0001f3cf, 0x00007208, 374 mmCB_HW_CONTROL_2, 0x0f000000, 0x0f000000, 375 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 376 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 377 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 378 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 379 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x16000012, 380 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x0000002a, 381 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 382 mmRLC_CGCG_CGLS_CTRL_3D, 0xffffffff, 0x0001003c, 383 mmSQ_CONFIG, 0x07f80000, 0x07180000, 384 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 385 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 386 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f7, 387 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 388 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 389 }; 390 391 static const u32 polaris10_golden_common_all[] = 392 { 393 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 394 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x16000012, 395 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002A, 396 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 397 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 398 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 399 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 400 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 401 }; 402 403 static const u32 fiji_golden_common_all[] = 404 { 405 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 406 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x3a00161a, 407 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x0000002e, 408 mmGB_ADDR_CONFIG, 0xffffffff, 0x22011003, 409 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 410 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 411 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 412 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 413 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 414 mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x00000009, 415 }; 416 417 static const u32 golden_settings_fiji_a10[] = 418 { 419 mmCB_HW_CONTROL_3, 0x000001ff, 0x00000040, 420 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 421 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 422 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 423 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 424 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 425 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 426 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 427 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 428 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000ff, 429 mmVGT_RESET_DEBUG, 0x00000004, 0x00000004, 430 }; 431 432 static const u32 fiji_mgcg_cgcg_init[] = 433 { 434 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 435 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 436 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 437 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 438 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 439 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 440 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x40000100, 441 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 442 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 443 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 444 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 445 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 446 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 447 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 448 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 449 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 450 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 451 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 452 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 453 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 454 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 455 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 456 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 457 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 458 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 459 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 460 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 461 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 462 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 463 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 464 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 465 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 466 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 467 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 468 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 469 }; 470 471 static const u32 golden_settings_iceland_a11[] = 472 { 473 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 474 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 475 mmDB_DEBUG3, 0xc0000000, 0xc0000000, 476 mmGB_GPU_ID, 0x0000000f, 0x00000000, 477 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 478 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 479 mmPA_SC_RASTER_CONFIG, 0x3f3fffff, 0x00000002, 480 mmPA_SC_RASTER_CONFIG_1, 0x0000003f, 0x00000000, 481 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 482 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 483 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 484 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 485 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 486 mmTCP_ADDR_CONFIG, 0x000003ff, 0x000000f1, 487 mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000, 488 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010, 489 }; 490 491 static const u32 iceland_golden_common_all[] = 492 { 493 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 494 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 495 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 496 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 497 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 498 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 499 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 500 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 501 }; 502 503 static const u32 iceland_mgcg_cgcg_init[] = 504 { 505 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 506 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 507 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 508 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 509 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0xc0000100, 510 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0xc0000100, 511 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0xc0000100, 512 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 513 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 514 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 515 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 516 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 517 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 518 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 519 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 520 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 521 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 522 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 523 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 524 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 525 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 526 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 527 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0xff000100, 528 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 529 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 530 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 531 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 532 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 533 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 534 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 535 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 536 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 537 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 538 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 539 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 540 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 541 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 542 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 543 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 544 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 545 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 546 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 547 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 548 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 549 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 550 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 551 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 552 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 553 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 554 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 555 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 556 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 557 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 558 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x0f840f87, 559 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 560 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 561 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 562 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 563 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 564 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 565 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 566 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 567 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 568 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003c, 569 }; 570 571 static const u32 cz_golden_settings_a11[] = 572 { 573 mmCB_HW_CONTROL_3, 0x00000040, 0x00000040, 574 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 575 mmGB_GPU_ID, 0x0000000f, 0x00000000, 576 mmPA_SC_ENHANCE, 0xffffffff, 0x00000001, 577 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 578 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0000003c, 579 mmSQ_RANDOM_WAVE_PRI, 0x001fffff, 0x000006fd, 580 mmTA_CNTL_AUX, 0x000f000f, 0x00010000, 581 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 582 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 583 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f3, 584 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00001302 585 }; 586 587 static const u32 cz_golden_common_all[] = 588 { 589 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 590 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000002, 591 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 592 mmGB_ADDR_CONFIG, 0xffffffff, 0x22010001, 593 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 594 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 595 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 596 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF 597 }; 598 599 static const u32 cz_mgcg_cgcg_init[] = 600 { 601 mmRLC_CGTT_MGCG_OVERRIDE, 0xffffffff, 0xffffffff, 602 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 603 mmCB_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 604 mmCGTT_BCI_CLK_CTRL, 0xffffffff, 0x00000100, 605 mmCGTT_CP_CLK_CTRL, 0xffffffff, 0x00000100, 606 mmCGTT_CPC_CLK_CTRL, 0xffffffff, 0x00000100, 607 mmCGTT_CPF_CLK_CTRL, 0xffffffff, 0x00000100, 608 mmCGTT_GDS_CLK_CTRL, 0xffffffff, 0x00000100, 609 mmCGTT_IA_CLK_CTRL, 0xffffffff, 0x06000100, 610 mmCGTT_PA_CLK_CTRL, 0xffffffff, 0x00000100, 611 mmCGTT_WD_CLK_CTRL, 0xffffffff, 0x06000100, 612 mmCGTT_PC_CLK_CTRL, 0xffffffff, 0x00000100, 613 mmCGTT_RLC_CLK_CTRL, 0xffffffff, 0x00000100, 614 mmCGTT_SC_CLK_CTRL, 0xffffffff, 0x00000100, 615 mmCGTT_SPI_CLK_CTRL, 0xffffffff, 0x00000100, 616 mmCGTT_SQ_CLK_CTRL, 0xffffffff, 0x00000100, 617 mmCGTT_SQG_CLK_CTRL, 0xffffffff, 0x00000100, 618 mmCGTT_SX_CLK_CTRL0, 0xffffffff, 0x00000100, 619 mmCGTT_SX_CLK_CTRL1, 0xffffffff, 0x00000100, 620 mmCGTT_SX_CLK_CTRL2, 0xffffffff, 0x00000100, 621 mmCGTT_SX_CLK_CTRL3, 0xffffffff, 0x00000100, 622 mmCGTT_SX_CLK_CTRL4, 0xffffffff, 0x00000100, 623 mmCGTT_TCI_CLK_CTRL, 0xffffffff, 0x00000100, 624 mmCGTT_TCP_CLK_CTRL, 0xffffffff, 0x00000100, 625 mmCGTT_VGT_CLK_CTRL, 0xffffffff, 0x06000100, 626 mmDB_CGTT_CLK_CTRL_0, 0xffffffff, 0x00000100, 627 mmTA_CGTT_CTRL, 0xffffffff, 0x00000100, 628 mmTCA_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 629 mmTCC_CGTT_SCLK_CTRL, 0xffffffff, 0x00000100, 630 mmTD_CGTT_CTRL, 0xffffffff, 0x00000100, 631 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 632 mmCGTS_CU0_SP0_CTRL_REG, 0xffffffff, 0x00010000, 633 mmCGTS_CU0_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 634 mmCGTS_CU0_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 635 mmCGTS_CU0_SP1_CTRL_REG, 0xffffffff, 0x00060005, 636 mmCGTS_CU0_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 637 mmCGTS_CU1_SP0_CTRL_REG, 0xffffffff, 0x00010000, 638 mmCGTS_CU1_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 639 mmCGTS_CU1_TA_CTRL_REG, 0xffffffff, 0x00040007, 640 mmCGTS_CU1_SP1_CTRL_REG, 0xffffffff, 0x00060005, 641 mmCGTS_CU1_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 642 mmCGTS_CU2_SP0_CTRL_REG, 0xffffffff, 0x00010000, 643 mmCGTS_CU2_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 644 mmCGTS_CU2_TA_CTRL_REG, 0xffffffff, 0x00040007, 645 mmCGTS_CU2_SP1_CTRL_REG, 0xffffffff, 0x00060005, 646 mmCGTS_CU2_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 647 mmCGTS_CU3_SP0_CTRL_REG, 0xffffffff, 0x00010000, 648 mmCGTS_CU3_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 649 mmCGTS_CU3_TA_CTRL_REG, 0xffffffff, 0x00040007, 650 mmCGTS_CU3_SP1_CTRL_REG, 0xffffffff, 0x00060005, 651 mmCGTS_CU3_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 652 mmCGTS_CU4_SP0_CTRL_REG, 0xffffffff, 0x00010000, 653 mmCGTS_CU4_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 654 mmCGTS_CU4_TA_SQC_CTRL_REG, 0xffffffff, 0x00040007, 655 mmCGTS_CU4_SP1_CTRL_REG, 0xffffffff, 0x00060005, 656 mmCGTS_CU4_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 657 mmCGTS_CU5_SP0_CTRL_REG, 0xffffffff, 0x00010000, 658 mmCGTS_CU5_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 659 mmCGTS_CU5_TA_CTRL_REG, 0xffffffff, 0x00040007, 660 mmCGTS_CU5_SP1_CTRL_REG, 0xffffffff, 0x00060005, 661 mmCGTS_CU5_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 662 mmCGTS_CU6_SP0_CTRL_REG, 0xffffffff, 0x00010000, 663 mmCGTS_CU6_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 664 mmCGTS_CU6_TA_CTRL_REG, 0xffffffff, 0x00040007, 665 mmCGTS_CU6_SP1_CTRL_REG, 0xffffffff, 0x00060005, 666 mmCGTS_CU6_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 667 mmCGTS_CU7_SP0_CTRL_REG, 0xffffffff, 0x00010000, 668 mmCGTS_CU7_LDS_SQ_CTRL_REG, 0xffffffff, 0x00030002, 669 mmCGTS_CU7_TA_CTRL_REG, 0xffffffff, 0x00040007, 670 mmCGTS_CU7_SP1_CTRL_REG, 0xffffffff, 0x00060005, 671 mmCGTS_CU7_TD_TCP_CTRL_REG, 0xffffffff, 0x00090008, 672 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96e00200, 673 mmCP_RB_WPTR_POLL_CNTL, 0xffffffff, 0x00900100, 674 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 675 mmCP_MEM_SLP_CNTL, 0x00000001, 0x00000001, 676 }; 677 678 static const u32 stoney_golden_settings_a11[] = 679 { 680 mmDB_DEBUG2, 0xf00fffff, 0x00000400, 681 mmGB_GPU_ID, 0x0000000f, 0x00000000, 682 mmPA_SC_ENHANCE, 0xffffffff, 0x20000001, 683 mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000, 684 mmRLC_CGCG_CGLS_CTRL, 0x00000003, 0x0001003c, 685 mmTA_CNTL_AUX, 0x000f000f, 0x000b0000, 686 mmTCC_CTRL, 0x00100000, 0xf31fff7f, 687 mmTCC_EXE_DISABLE, 0x00000002, 0x00000002, 688 mmTCP_ADDR_CONFIG, 0x0000000f, 0x000000f1, 689 mmTCP_CHAN_STEER_LO, 0xffffffff, 0x10101010, 690 }; 691 692 static const u32 stoney_golden_common_all[] = 693 { 694 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 695 mmPA_SC_RASTER_CONFIG, 0xffffffff, 0x00000000, 696 mmPA_SC_RASTER_CONFIG_1, 0xffffffff, 0x00000000, 697 mmGB_ADDR_CONFIG, 0xffffffff, 0x12010001, 698 mmSPI_RESOURCE_RESERVE_CU_0, 0xffffffff, 0x00000800, 699 mmSPI_RESOURCE_RESERVE_CU_1, 0xffffffff, 0x00000800, 700 mmSPI_RESOURCE_RESERVE_EN_CU_0, 0xffffffff, 0x00FF7FBF, 701 mmSPI_RESOURCE_RESERVE_EN_CU_1, 0xffffffff, 0x00FF7FAF, 702 }; 703 704 static const u32 stoney_mgcg_cgcg_init[] = 705 { 706 mmGRBM_GFX_INDEX, 0xffffffff, 0xe0000000, 707 mmRLC_CGCG_CGLS_CTRL, 0xffffffff, 0x0020003f, 708 mmCP_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 709 mmRLC_MEM_SLP_CNTL, 0xffffffff, 0x00020201, 710 mmCGTS_SM_CTRL_REG, 0xffffffff, 0x96940200, 711 }; 712 713 714 static const char * const sq_edc_source_names[] = { 715 "SQ_EDC_INFO_SOURCE_INVALID: No EDC error has occurred", 716 "SQ_EDC_INFO_SOURCE_INST: EDC source is Instruction Fetch", 717 "SQ_EDC_INFO_SOURCE_SGPR: EDC source is SGPR or SQC data return", 718 "SQ_EDC_INFO_SOURCE_VGPR: EDC source is VGPR", 719 "SQ_EDC_INFO_SOURCE_LDS: EDC source is LDS", 720 "SQ_EDC_INFO_SOURCE_GDS: EDC source is GDS", 721 "SQ_EDC_INFO_SOURCE_TA: EDC source is TA", 722 }; 723 724 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev); 725 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev); 726 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev); 727 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev); 728 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev); 729 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev); 730 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring); 731 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring); 732 733 #define CG_ACLK_CNTL__ACLK_DIVIDER_MASK 0x0000007fL 734 #define CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT 0x00000000L 735 736 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev) 737 { 738 uint32_t data; 739 740 switch (adev->asic_type) { 741 case CHIP_TOPAZ: 742 amdgpu_device_program_register_sequence(adev, 743 iceland_mgcg_cgcg_init, 744 ARRAY_SIZE(iceland_mgcg_cgcg_init)); 745 amdgpu_device_program_register_sequence(adev, 746 golden_settings_iceland_a11, 747 ARRAY_SIZE(golden_settings_iceland_a11)); 748 amdgpu_device_program_register_sequence(adev, 749 iceland_golden_common_all, 750 ARRAY_SIZE(iceland_golden_common_all)); 751 break; 752 case CHIP_FIJI: 753 amdgpu_device_program_register_sequence(adev, 754 fiji_mgcg_cgcg_init, 755 ARRAY_SIZE(fiji_mgcg_cgcg_init)); 756 amdgpu_device_program_register_sequence(adev, 757 golden_settings_fiji_a10, 758 ARRAY_SIZE(golden_settings_fiji_a10)); 759 amdgpu_device_program_register_sequence(adev, 760 fiji_golden_common_all, 761 ARRAY_SIZE(fiji_golden_common_all)); 762 break; 763 764 case CHIP_TONGA: 765 amdgpu_device_program_register_sequence(adev, 766 tonga_mgcg_cgcg_init, 767 ARRAY_SIZE(tonga_mgcg_cgcg_init)); 768 amdgpu_device_program_register_sequence(adev, 769 golden_settings_tonga_a11, 770 ARRAY_SIZE(golden_settings_tonga_a11)); 771 amdgpu_device_program_register_sequence(adev, 772 tonga_golden_common_all, 773 ARRAY_SIZE(tonga_golden_common_all)); 774 break; 775 case CHIP_VEGAM: 776 amdgpu_device_program_register_sequence(adev, 777 golden_settings_vegam_a11, 778 ARRAY_SIZE(golden_settings_vegam_a11)); 779 amdgpu_device_program_register_sequence(adev, 780 vegam_golden_common_all, 781 ARRAY_SIZE(vegam_golden_common_all)); 782 break; 783 case CHIP_POLARIS11: 784 case CHIP_POLARIS12: 785 amdgpu_device_program_register_sequence(adev, 786 golden_settings_polaris11_a11, 787 ARRAY_SIZE(golden_settings_polaris11_a11)); 788 amdgpu_device_program_register_sequence(adev, 789 polaris11_golden_common_all, 790 ARRAY_SIZE(polaris11_golden_common_all)); 791 break; 792 case CHIP_POLARIS10: 793 amdgpu_device_program_register_sequence(adev, 794 golden_settings_polaris10_a11, 795 ARRAY_SIZE(golden_settings_polaris10_a11)); 796 amdgpu_device_program_register_sequence(adev, 797 polaris10_golden_common_all, 798 ARRAY_SIZE(polaris10_golden_common_all)); 799 data = RREG32_SMC(ixCG_ACLK_CNTL); 800 data &= ~CG_ACLK_CNTL__ACLK_DIVIDER_MASK; 801 data |= 0x18 << CG_ACLK_CNTL__ACLK_DIVIDER__SHIFT; 802 WREG32_SMC(ixCG_ACLK_CNTL, data); 803 if ((adev->pdev->device == 0x67DF) && (adev->pdev->revision == 0xc7) && 804 ((adev->pdev->subsystem_device == 0xb37 && adev->pdev->subsystem_vendor == 0x1002) || 805 (adev->pdev->subsystem_device == 0x4a8 && adev->pdev->subsystem_vendor == 0x1043) || 806 (adev->pdev->subsystem_device == 0x9480 && adev->pdev->subsystem_vendor == 0x1680))) { 807 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1E, 0xDD); 808 amdgpu_atombios_i2c_channel_trans(adev, 0x10, 0x96, 0x1F, 0xD0); 809 } 810 break; 811 case CHIP_CARRIZO: 812 amdgpu_device_program_register_sequence(adev, 813 cz_mgcg_cgcg_init, 814 ARRAY_SIZE(cz_mgcg_cgcg_init)); 815 amdgpu_device_program_register_sequence(adev, 816 cz_golden_settings_a11, 817 ARRAY_SIZE(cz_golden_settings_a11)); 818 amdgpu_device_program_register_sequence(adev, 819 cz_golden_common_all, 820 ARRAY_SIZE(cz_golden_common_all)); 821 break; 822 case CHIP_STONEY: 823 amdgpu_device_program_register_sequence(adev, 824 stoney_mgcg_cgcg_init, 825 ARRAY_SIZE(stoney_mgcg_cgcg_init)); 826 amdgpu_device_program_register_sequence(adev, 827 stoney_golden_settings_a11, 828 ARRAY_SIZE(stoney_golden_settings_a11)); 829 amdgpu_device_program_register_sequence(adev, 830 stoney_golden_common_all, 831 ARRAY_SIZE(stoney_golden_common_all)); 832 break; 833 default: 834 break; 835 } 836 } 837 838 static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring) 839 { 840 struct amdgpu_device *adev = ring->adev; 841 uint32_t tmp = 0; 842 unsigned i; 843 int r; 844 845 WREG32(mmSCRATCH_REG0, 0xCAFEDEAD); 846 r = amdgpu_ring_alloc(ring, 3); 847 if (r) 848 return r; 849 850 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 851 amdgpu_ring_write(ring, mmSCRATCH_REG0 - PACKET3_SET_UCONFIG_REG_START); 852 amdgpu_ring_write(ring, 0xDEADBEEF); 853 amdgpu_ring_commit(ring); 854 855 for (i = 0; i < adev->usec_timeout; i++) { 856 tmp = RREG32(mmSCRATCH_REG0); 857 if (tmp == 0xDEADBEEF) 858 break; 859 udelay(1); 860 } 861 862 if (i >= adev->usec_timeout) 863 r = -ETIMEDOUT; 864 865 return r; 866 } 867 868 static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 869 { 870 struct amdgpu_device *adev = ring->adev; 871 struct amdgpu_ib ib; 872 struct dma_fence *f = NULL; 873 874 unsigned int index; 875 uint64_t gpu_addr; 876 uint32_t tmp; 877 long r; 878 879 r = amdgpu_wb_get(adev, &index); 880 if (r) 881 return r; 882 883 gpu_addr = adev->wb.gpu_addr + (index * 4); 884 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 885 memset(&ib, 0, sizeof(ib)); 886 887 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 888 if (r) 889 goto err1; 890 891 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 892 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 893 ib.ptr[2] = lower_32_bits(gpu_addr); 894 ib.ptr[3] = upper_32_bits(gpu_addr); 895 ib.ptr[4] = 0xDEADBEEF; 896 ib.length_dw = 5; 897 898 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 899 if (r) 900 goto err2; 901 902 r = dma_fence_wait_timeout(f, false, timeout); 903 if (r == 0) { 904 r = -ETIMEDOUT; 905 goto err2; 906 } else if (r < 0) { 907 goto err2; 908 } 909 910 tmp = adev->wb.wb[index]; 911 if (tmp == 0xDEADBEEF) 912 r = 0; 913 else 914 r = -EINVAL; 915 916 err2: 917 amdgpu_ib_free(&ib, NULL); 918 dma_fence_put(f); 919 err1: 920 amdgpu_wb_free(adev, index); 921 return r; 922 } 923 924 925 static void gfx_v8_0_free_microcode(struct amdgpu_device *adev) 926 { 927 amdgpu_ucode_release(&adev->gfx.pfp_fw); 928 amdgpu_ucode_release(&adev->gfx.me_fw); 929 amdgpu_ucode_release(&adev->gfx.ce_fw); 930 amdgpu_ucode_release(&adev->gfx.rlc_fw); 931 amdgpu_ucode_release(&adev->gfx.mec_fw); 932 if ((adev->asic_type != CHIP_STONEY) && 933 (adev->asic_type != CHIP_TOPAZ)) 934 amdgpu_ucode_release(&adev->gfx.mec2_fw); 935 936 kfree(adev->gfx.rlc.register_list_format); 937 } 938 939 static int gfx_v8_0_init_microcode(struct amdgpu_device *adev) 940 { 941 const char *chip_name; 942 int err; 943 struct amdgpu_firmware_info *info = NULL; 944 const struct common_firmware_header *header = NULL; 945 const struct gfx_firmware_header_v1_0 *cp_hdr; 946 const struct rlc_firmware_header_v2_0 *rlc_hdr; 947 unsigned int *tmp = NULL, i; 948 949 DRM_DEBUG("\n"); 950 951 switch (adev->asic_type) { 952 case CHIP_TOPAZ: 953 chip_name = "topaz"; 954 break; 955 case CHIP_TONGA: 956 chip_name = "tonga"; 957 break; 958 case CHIP_CARRIZO: 959 chip_name = "carrizo"; 960 break; 961 case CHIP_FIJI: 962 chip_name = "fiji"; 963 break; 964 case CHIP_STONEY: 965 chip_name = "stoney"; 966 break; 967 case CHIP_POLARIS10: 968 chip_name = "polaris10"; 969 break; 970 case CHIP_POLARIS11: 971 chip_name = "polaris11"; 972 break; 973 case CHIP_POLARIS12: 974 chip_name = "polaris12"; 975 break; 976 case CHIP_VEGAM: 977 chip_name = "vegam"; 978 break; 979 default: 980 BUG(); 981 } 982 983 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 984 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 985 AMDGPU_UCODE_OPTIONAL, 986 "amdgpu/%s_pfp_2.bin", chip_name); 987 if (err == -ENODEV) { 988 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 989 AMDGPU_UCODE_REQUIRED, 990 "amdgpu/%s_pfp.bin", chip_name); 991 } 992 } else { 993 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 994 AMDGPU_UCODE_REQUIRED, 995 "amdgpu/%s_pfp.bin", chip_name); 996 } 997 if (err) 998 goto out; 999 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1000 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1001 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1002 1003 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1004 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1005 AMDGPU_UCODE_OPTIONAL, 1006 "amdgpu/%s_me_2.bin", chip_name); 1007 if (err == -ENODEV) { 1008 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1009 AMDGPU_UCODE_REQUIRED, 1010 "amdgpu/%s_me.bin", chip_name); 1011 } 1012 } else { 1013 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1014 AMDGPU_UCODE_REQUIRED, 1015 "amdgpu/%s_me.bin", chip_name); 1016 } 1017 if (err) 1018 goto out; 1019 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1020 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1021 1022 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1023 1024 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1025 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1026 AMDGPU_UCODE_OPTIONAL, 1027 "amdgpu/%s_ce_2.bin", chip_name); 1028 if (err == -ENODEV) { 1029 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1030 AMDGPU_UCODE_REQUIRED, 1031 "amdgpu/%s_ce.bin", chip_name); 1032 } 1033 } else { 1034 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1035 AMDGPU_UCODE_REQUIRED, 1036 "amdgpu/%s_ce.bin", chip_name); 1037 } 1038 if (err) 1039 goto out; 1040 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1041 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1042 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1043 1044 /* 1045 * Support for MCBP/Virtualization in combination with chained IBs is 1046 * formal released on feature version #46 1047 */ 1048 if (adev->gfx.ce_feature_version >= 46 && 1049 adev->gfx.pfp_feature_version >= 46) { 1050 adev->virt.chained_ib_support = true; 1051 drm_info(adev_to_drm(adev), "Chained IB support enabled!\n"); 1052 } else 1053 adev->virt.chained_ib_support = false; 1054 1055 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1056 AMDGPU_UCODE_REQUIRED, 1057 "amdgpu/%s_rlc.bin", chip_name); 1058 if (err) 1059 goto out; 1060 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1061 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1062 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1063 1064 adev->gfx.rlc.save_and_restore_offset = 1065 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1066 adev->gfx.rlc.clear_state_descriptor_offset = 1067 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1068 adev->gfx.rlc.avail_scratch_ram_locations = 1069 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1070 adev->gfx.rlc.reg_restore_list_size = 1071 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1072 adev->gfx.rlc.reg_list_format_start = 1073 le32_to_cpu(rlc_hdr->reg_list_format_start); 1074 adev->gfx.rlc.reg_list_format_separate_start = 1075 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1076 adev->gfx.rlc.starting_offsets_start = 1077 le32_to_cpu(rlc_hdr->starting_offsets_start); 1078 adev->gfx.rlc.reg_list_format_size_bytes = 1079 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1080 adev->gfx.rlc.reg_list_size_bytes = 1081 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1082 1083 adev->gfx.rlc.register_list_format = 1084 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1085 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1086 1087 if (!adev->gfx.rlc.register_list_format) { 1088 err = -ENOMEM; 1089 goto out; 1090 } 1091 1092 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1093 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1094 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1095 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1096 1097 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1098 1099 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1100 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1101 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1102 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1103 1104 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1105 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1106 AMDGPU_UCODE_OPTIONAL, 1107 "amdgpu/%s_mec_2.bin", chip_name); 1108 if (err == -ENODEV) { 1109 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1110 AMDGPU_UCODE_REQUIRED, 1111 "amdgpu/%s_mec.bin", chip_name); 1112 } 1113 } else { 1114 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1115 AMDGPU_UCODE_REQUIRED, 1116 "amdgpu/%s_mec.bin", chip_name); 1117 } 1118 if (err) 1119 goto out; 1120 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1121 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1122 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1123 1124 if ((adev->asic_type != CHIP_STONEY) && 1125 (adev->asic_type != CHIP_TOPAZ)) { 1126 if (adev->asic_type >= CHIP_POLARIS10 && adev->asic_type <= CHIP_POLARIS12) { 1127 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1128 AMDGPU_UCODE_OPTIONAL, 1129 "amdgpu/%s_mec2_2.bin", chip_name); 1130 if (err == -ENODEV) { 1131 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1132 AMDGPU_UCODE_REQUIRED, 1133 "amdgpu/%s_mec2.bin", chip_name); 1134 } 1135 } else { 1136 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1137 AMDGPU_UCODE_REQUIRED, 1138 "amdgpu/%s_mec2.bin", chip_name); 1139 } 1140 if (!err) { 1141 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1142 adev->gfx.mec2_fw->data; 1143 adev->gfx.mec2_fw_version = 1144 le32_to_cpu(cp_hdr->header.ucode_version); 1145 adev->gfx.mec2_feature_version = 1146 le32_to_cpu(cp_hdr->ucode_feature_version); 1147 } else { 1148 err = 0; 1149 adev->gfx.mec2_fw = NULL; 1150 } 1151 } 1152 1153 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1154 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1155 info->fw = adev->gfx.pfp_fw; 1156 header = (const struct common_firmware_header *)info->fw->data; 1157 adev->firmware.fw_size += 1158 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1159 1160 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1161 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1162 info->fw = adev->gfx.me_fw; 1163 header = (const struct common_firmware_header *)info->fw->data; 1164 adev->firmware.fw_size += 1165 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1166 1167 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1168 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1169 info->fw = adev->gfx.ce_fw; 1170 header = (const struct common_firmware_header *)info->fw->data; 1171 adev->firmware.fw_size += 1172 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1173 1174 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1175 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1176 info->fw = adev->gfx.rlc_fw; 1177 header = (const struct common_firmware_header *)info->fw->data; 1178 adev->firmware.fw_size += 1179 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1180 1181 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1182 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1183 info->fw = adev->gfx.mec_fw; 1184 header = (const struct common_firmware_header *)info->fw->data; 1185 adev->firmware.fw_size += 1186 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1187 1188 /* we need account JT in */ 1189 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1190 adev->firmware.fw_size += 1191 ALIGN(le32_to_cpu(cp_hdr->jt_size) << 2, PAGE_SIZE); 1192 1193 if (amdgpu_sriov_vf(adev)) { 1194 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_STORAGE]; 1195 info->ucode_id = AMDGPU_UCODE_ID_STORAGE; 1196 info->fw = adev->gfx.mec_fw; 1197 adev->firmware.fw_size += 1198 ALIGN(le32_to_cpu(64 * PAGE_SIZE), PAGE_SIZE); 1199 } 1200 1201 if (adev->gfx.mec2_fw) { 1202 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1203 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1204 info->fw = adev->gfx.mec2_fw; 1205 header = (const struct common_firmware_header *)info->fw->data; 1206 adev->firmware.fw_size += 1207 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1208 } 1209 1210 out: 1211 if (err) { 1212 dev_err(adev->dev, "gfx8: Failed to load firmware %s gfx firmware\n", chip_name); 1213 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1214 amdgpu_ucode_release(&adev->gfx.me_fw); 1215 amdgpu_ucode_release(&adev->gfx.ce_fw); 1216 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1217 amdgpu_ucode_release(&adev->gfx.mec_fw); 1218 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1219 } 1220 return err; 1221 } 1222 1223 static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 1224 { 1225 u32 count = 0; 1226 1227 if (adev->gfx.rlc.cs_data == NULL) 1228 return; 1229 if (buffer == NULL) 1230 return; 1231 1232 count = amdgpu_gfx_csb_preamble_start(buffer); 1233 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 1234 1235 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 1236 buffer[count++] = cpu_to_le32(mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 1237 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config); 1238 buffer[count++] = cpu_to_le32(adev->gfx.config.rb_config[0][0].raster_config_1); 1239 1240 amdgpu_gfx_csb_preamble_end(buffer, count); 1241 } 1242 1243 static int gfx_v8_0_cp_jump_table_num(struct amdgpu_device *adev) 1244 { 1245 if (adev->asic_type == CHIP_CARRIZO) 1246 return 5; 1247 else 1248 return 4; 1249 } 1250 1251 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev) 1252 { 1253 const struct cs_section_def *cs_data; 1254 int r; 1255 1256 adev->gfx.rlc.cs_data = vi_cs_data; 1257 1258 cs_data = adev->gfx.rlc.cs_data; 1259 1260 if (cs_data) { 1261 /* init clear state block */ 1262 r = amdgpu_gfx_rlc_init_csb(adev); 1263 if (r) 1264 return r; 1265 } 1266 1267 if ((adev->asic_type == CHIP_CARRIZO) || 1268 (adev->asic_type == CHIP_STONEY)) { 1269 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1270 r = amdgpu_gfx_rlc_init_cpt(adev); 1271 if (r) 1272 return r; 1273 } 1274 1275 /* init spm vmid with 0xf */ 1276 if (adev->gfx.rlc.funcs->update_spm_vmid) 1277 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf); 1278 1279 return 0; 1280 } 1281 1282 static void gfx_v8_0_mec_fini(struct amdgpu_device *adev) 1283 { 1284 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1285 } 1286 1287 static int gfx_v8_0_mec_init(struct amdgpu_device *adev) 1288 { 1289 int r; 1290 u32 *hpd; 1291 size_t mec_hpd_size; 1292 1293 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1294 1295 /* take ownership of the relevant compute queues */ 1296 amdgpu_gfx_compute_queue_acquire(adev); 1297 1298 mec_hpd_size = adev->gfx.num_compute_rings * GFX8_MEC_HPD_SIZE; 1299 if (mec_hpd_size) { 1300 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1301 AMDGPU_GEM_DOMAIN_VRAM | 1302 AMDGPU_GEM_DOMAIN_GTT, 1303 &adev->gfx.mec.hpd_eop_obj, 1304 &adev->gfx.mec.hpd_eop_gpu_addr, 1305 (void **)&hpd); 1306 if (r) { 1307 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1308 return r; 1309 } 1310 1311 memset(hpd, 0, mec_hpd_size); 1312 1313 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1314 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1315 } 1316 1317 return 0; 1318 } 1319 1320 static const u32 vgpr_init_compute_shader[] = 1321 { 1322 0x7e000209, 0x7e020208, 1323 0x7e040207, 0x7e060206, 1324 0x7e080205, 0x7e0a0204, 1325 0x7e0c0203, 0x7e0e0202, 1326 0x7e100201, 0x7e120200, 1327 0x7e140209, 0x7e160208, 1328 0x7e180207, 0x7e1a0206, 1329 0x7e1c0205, 0x7e1e0204, 1330 0x7e200203, 0x7e220202, 1331 0x7e240201, 0x7e260200, 1332 0x7e280209, 0x7e2a0208, 1333 0x7e2c0207, 0x7e2e0206, 1334 0x7e300205, 0x7e320204, 1335 0x7e340203, 0x7e360202, 1336 0x7e380201, 0x7e3a0200, 1337 0x7e3c0209, 0x7e3e0208, 1338 0x7e400207, 0x7e420206, 1339 0x7e440205, 0x7e460204, 1340 0x7e480203, 0x7e4a0202, 1341 0x7e4c0201, 0x7e4e0200, 1342 0x7e500209, 0x7e520208, 1343 0x7e540207, 0x7e560206, 1344 0x7e580205, 0x7e5a0204, 1345 0x7e5c0203, 0x7e5e0202, 1346 0x7e600201, 0x7e620200, 1347 0x7e640209, 0x7e660208, 1348 0x7e680207, 0x7e6a0206, 1349 0x7e6c0205, 0x7e6e0204, 1350 0x7e700203, 0x7e720202, 1351 0x7e740201, 0x7e760200, 1352 0x7e780209, 0x7e7a0208, 1353 0x7e7c0207, 0x7e7e0206, 1354 0xbf8a0000, 0xbf810000, 1355 }; 1356 1357 static const u32 sgpr_init_compute_shader[] = 1358 { 1359 0xbe8a0100, 0xbe8c0102, 1360 0xbe8e0104, 0xbe900106, 1361 0xbe920108, 0xbe940100, 1362 0xbe960102, 0xbe980104, 1363 0xbe9a0106, 0xbe9c0108, 1364 0xbe9e0100, 0xbea00102, 1365 0xbea20104, 0xbea40106, 1366 0xbea60108, 0xbea80100, 1367 0xbeaa0102, 0xbeac0104, 1368 0xbeae0106, 0xbeb00108, 1369 0xbeb20100, 0xbeb40102, 1370 0xbeb60104, 0xbeb80106, 1371 0xbeba0108, 0xbebc0100, 1372 0xbebe0102, 0xbec00104, 1373 0xbec20106, 0xbec40108, 1374 0xbec60100, 0xbec80102, 1375 0xbee60004, 0xbee70005, 1376 0xbeea0006, 0xbeeb0007, 1377 0xbee80008, 0xbee90009, 1378 0xbefc0000, 0xbf8a0000, 1379 0xbf810000, 0x00000000, 1380 }; 1381 1382 static const u32 vgpr_init_regs[] = 1383 { 1384 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xffffffff, 1385 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1386 mmCOMPUTE_NUM_THREAD_X, 256*4, 1387 mmCOMPUTE_NUM_THREAD_Y, 1, 1388 mmCOMPUTE_NUM_THREAD_Z, 1, 1389 mmCOMPUTE_PGM_RSRC1, 0x100004f, /* VGPRS=15 (64 logical VGPRs), SGPRS=1 (16 SGPRs), BULKY=1 */ 1390 mmCOMPUTE_PGM_RSRC2, 20, 1391 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1392 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1393 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1394 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1395 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1396 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1397 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1398 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1399 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1400 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1401 }; 1402 1403 static const u32 sgpr1_init_regs[] = 1404 { 1405 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0x0f, 1406 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, /* CU_GROUP_COUNT=1 */ 1407 mmCOMPUTE_NUM_THREAD_X, 256*5, 1408 mmCOMPUTE_NUM_THREAD_Y, 1, 1409 mmCOMPUTE_NUM_THREAD_Z, 1, 1410 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1411 mmCOMPUTE_PGM_RSRC2, 20, 1412 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1413 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1414 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1415 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1416 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1417 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1418 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1419 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1420 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1421 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1422 }; 1423 1424 static const u32 sgpr2_init_regs[] = 1425 { 1426 mmCOMPUTE_STATIC_THREAD_MGMT_SE0, 0xf0, 1427 mmCOMPUTE_RESOURCE_LIMITS, 0x1000000, 1428 mmCOMPUTE_NUM_THREAD_X, 256*5, 1429 mmCOMPUTE_NUM_THREAD_Y, 1, 1430 mmCOMPUTE_NUM_THREAD_Z, 1, 1431 mmCOMPUTE_PGM_RSRC1, 0x240, /* SGPRS=9 (80 GPRS) */ 1432 mmCOMPUTE_PGM_RSRC2, 20, 1433 mmCOMPUTE_USER_DATA_0, 0xedcedc00, 1434 mmCOMPUTE_USER_DATA_1, 0xedcedc01, 1435 mmCOMPUTE_USER_DATA_2, 0xedcedc02, 1436 mmCOMPUTE_USER_DATA_3, 0xedcedc03, 1437 mmCOMPUTE_USER_DATA_4, 0xedcedc04, 1438 mmCOMPUTE_USER_DATA_5, 0xedcedc05, 1439 mmCOMPUTE_USER_DATA_6, 0xedcedc06, 1440 mmCOMPUTE_USER_DATA_7, 0xedcedc07, 1441 mmCOMPUTE_USER_DATA_8, 0xedcedc08, 1442 mmCOMPUTE_USER_DATA_9, 0xedcedc09, 1443 }; 1444 1445 static const u32 sec_ded_counter_registers[] = 1446 { 1447 mmCPC_EDC_ATC_CNT, 1448 mmCPC_EDC_SCRATCH_CNT, 1449 mmCPC_EDC_UCODE_CNT, 1450 mmCPF_EDC_ATC_CNT, 1451 mmCPF_EDC_ROQ_CNT, 1452 mmCPF_EDC_TAG_CNT, 1453 mmCPG_EDC_ATC_CNT, 1454 mmCPG_EDC_DMA_CNT, 1455 mmCPG_EDC_TAG_CNT, 1456 mmDC_EDC_CSINVOC_CNT, 1457 mmDC_EDC_RESTORE_CNT, 1458 mmDC_EDC_STATE_CNT, 1459 mmGDS_EDC_CNT, 1460 mmGDS_EDC_GRBM_CNT, 1461 mmGDS_EDC_OA_DED, 1462 mmSPI_EDC_CNT, 1463 mmSQC_ATC_EDC_GATCL1_CNT, 1464 mmSQC_EDC_CNT, 1465 mmSQ_EDC_DED_CNT, 1466 mmSQ_EDC_INFO, 1467 mmSQ_EDC_SEC_CNT, 1468 mmTCC_EDC_CNT, 1469 mmTCP_ATC_EDC_GATCL1_CNT, 1470 mmTCP_EDC_CNT, 1471 mmTD_EDC_CNT 1472 }; 1473 1474 static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 1475 { 1476 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 1477 struct amdgpu_ib ib; 1478 struct dma_fence *f = NULL; 1479 int r, i; 1480 u32 tmp; 1481 unsigned total_size, vgpr_offset, sgpr_offset; 1482 u64 gpu_addr; 1483 1484 /* only supported on CZ */ 1485 if (adev->asic_type != CHIP_CARRIZO) 1486 return 0; 1487 1488 /* bail if the compute ring is not ready */ 1489 if (!ring->sched.ready) 1490 return -EBUSY; 1491 1492 if (amdgpu_in_reset(adev)) { 1493 /* Set preempt condition to execute IB */ 1494 amdgpu_ring_set_preempt_cond_exec(ring, true); 1495 /* Flush HDP cache so the GPU can see the updated COND_EXEC value */ 1496 amdgpu_device_flush_hdp(adev, NULL); 1497 } 1498 1499 tmp = RREG32(mmGB_EDC_MODE); 1500 WREG32(mmGB_EDC_MODE, 0); 1501 1502 total_size = 1503 (((ARRAY_SIZE(vgpr_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1504 total_size += 1505 (((ARRAY_SIZE(sgpr1_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1506 total_size += 1507 (((ARRAY_SIZE(sgpr2_init_regs) / 2) * 3) + 4 + 5 + 2) * 4; 1508 total_size = ALIGN(total_size, 256); 1509 vgpr_offset = total_size; 1510 total_size += ALIGN(sizeof(vgpr_init_compute_shader), 256); 1511 sgpr_offset = total_size; 1512 total_size += sizeof(sgpr_init_compute_shader); 1513 1514 /* allocate an indirect buffer to put the commands in */ 1515 memset(&ib, 0, sizeof(ib)); 1516 r = amdgpu_ib_get(adev, NULL, total_size, 1517 AMDGPU_IB_POOL_DIRECT, &ib); 1518 if (r) { 1519 drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r); 1520 return r; 1521 } 1522 1523 /* load the compute shaders */ 1524 for (i = 0; i < ARRAY_SIZE(vgpr_init_compute_shader); i++) 1525 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_compute_shader[i]; 1526 1527 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 1528 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 1529 1530 /* init the ib length to 0 */ 1531 ib.length_dw = 0; 1532 1533 /* VGPR */ 1534 /* write the register state for the compute dispatch */ 1535 for (i = 0; i < ARRAY_SIZE(vgpr_init_regs); i += 2) { 1536 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1537 ib.ptr[ib.length_dw++] = vgpr_init_regs[i] - PACKET3_SET_SH_REG_START; 1538 ib.ptr[ib.length_dw++] = vgpr_init_regs[i + 1]; 1539 } 1540 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1541 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 1542 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1543 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1544 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1545 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1546 1547 /* write dispatch packet */ 1548 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1549 ib.ptr[ib.length_dw++] = 8; /* x */ 1550 ib.ptr[ib.length_dw++] = 1; /* y */ 1551 ib.ptr[ib.length_dw++] = 1; /* z */ 1552 ib.ptr[ib.length_dw++] = 1553 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1554 1555 /* write CS partial flush packet */ 1556 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1557 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1558 1559 /* SGPR1 */ 1560 /* write the register state for the compute dispatch */ 1561 for (i = 0; i < ARRAY_SIZE(sgpr1_init_regs); i += 2) { 1562 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1563 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i] - PACKET3_SET_SH_REG_START; 1564 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i + 1]; 1565 } 1566 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1567 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 1568 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1569 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1570 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1571 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1572 1573 /* write dispatch packet */ 1574 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1575 ib.ptr[ib.length_dw++] = 8; /* x */ 1576 ib.ptr[ib.length_dw++] = 1; /* y */ 1577 ib.ptr[ib.length_dw++] = 1; /* z */ 1578 ib.ptr[ib.length_dw++] = 1579 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1580 1581 /* write CS partial flush packet */ 1582 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1583 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1584 1585 /* SGPR2 */ 1586 /* write the register state for the compute dispatch */ 1587 for (i = 0; i < ARRAY_SIZE(sgpr2_init_regs); i += 2) { 1588 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 1589 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i] - PACKET3_SET_SH_REG_START; 1590 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i + 1]; 1591 } 1592 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 1593 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 1594 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 1595 ib.ptr[ib.length_dw++] = mmCOMPUTE_PGM_LO - PACKET3_SET_SH_REG_START; 1596 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 1597 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 1598 1599 /* write dispatch packet */ 1600 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 1601 ib.ptr[ib.length_dw++] = 8; /* x */ 1602 ib.ptr[ib.length_dw++] = 1; /* y */ 1603 ib.ptr[ib.length_dw++] = 1; /* z */ 1604 ib.ptr[ib.length_dw++] = 1605 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 1606 1607 /* write CS partial flush packet */ 1608 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 1609 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 1610 1611 /* shedule the ib on the ring */ 1612 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1613 if (r) { 1614 drm_err(adev_to_drm(adev), "ib submit failed (%d).\n", r); 1615 goto fail; 1616 } 1617 1618 /* wait for the GPU to finish processing the IB */ 1619 r = dma_fence_wait(f, false); 1620 if (r) { 1621 drm_err(adev_to_drm(adev), "fence wait failed (%d).\n", r); 1622 goto fail; 1623 } 1624 1625 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, DED_MODE, 2); 1626 tmp = REG_SET_FIELD(tmp, GB_EDC_MODE, PROP_FED, 1); 1627 WREG32(mmGB_EDC_MODE, tmp); 1628 1629 tmp = RREG32(mmCC_GC_EDC_CONFIG); 1630 tmp = REG_SET_FIELD(tmp, CC_GC_EDC_CONFIG, DIS_EDC, 0) | 1; 1631 WREG32(mmCC_GC_EDC_CONFIG, tmp); 1632 1633 1634 /* read back registers to clear the counters */ 1635 for (i = 0; i < ARRAY_SIZE(sec_ded_counter_registers); i++) 1636 RREG32(sec_ded_counter_registers[i]); 1637 1638 fail: 1639 amdgpu_ib_free(&ib, NULL); 1640 dma_fence_put(f); 1641 1642 return r; 1643 } 1644 1645 static int gfx_v8_0_gpu_early_init(struct amdgpu_device *adev) 1646 { 1647 u32 gb_addr_config; 1648 u32 mc_arb_ramcfg; 1649 u32 dimm00_addr_map, dimm01_addr_map, dimm10_addr_map, dimm11_addr_map; 1650 u32 tmp; 1651 int ret; 1652 1653 switch (adev->asic_type) { 1654 case CHIP_TOPAZ: 1655 adev->gfx.config.max_shader_engines = 1; 1656 adev->gfx.config.max_tile_pipes = 2; 1657 adev->gfx.config.max_cu_per_sh = 6; 1658 adev->gfx.config.max_sh_per_se = 1; 1659 adev->gfx.config.max_backends_per_se = 2; 1660 adev->gfx.config.max_texture_channel_caches = 2; 1661 adev->gfx.config.max_gprs = 256; 1662 adev->gfx.config.max_gs_threads = 32; 1663 adev->gfx.config.max_hw_contexts = 8; 1664 1665 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1666 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1667 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1668 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1669 gb_addr_config = TOPAZ_GB_ADDR_CONFIG_GOLDEN; 1670 break; 1671 case CHIP_FIJI: 1672 adev->gfx.config.max_shader_engines = 4; 1673 adev->gfx.config.max_tile_pipes = 16; 1674 adev->gfx.config.max_cu_per_sh = 16; 1675 adev->gfx.config.max_sh_per_se = 1; 1676 adev->gfx.config.max_backends_per_se = 4; 1677 adev->gfx.config.max_texture_channel_caches = 16; 1678 adev->gfx.config.max_gprs = 256; 1679 adev->gfx.config.max_gs_threads = 32; 1680 adev->gfx.config.max_hw_contexts = 8; 1681 1682 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1683 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1684 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1685 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1686 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1687 break; 1688 case CHIP_POLARIS11: 1689 case CHIP_POLARIS12: 1690 ret = amdgpu_atombios_get_gfx_info(adev); 1691 if (ret) 1692 return ret; 1693 adev->gfx.config.max_gprs = 256; 1694 adev->gfx.config.max_gs_threads = 32; 1695 adev->gfx.config.max_hw_contexts = 8; 1696 1697 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1698 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1699 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1700 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1701 gb_addr_config = POLARIS11_GB_ADDR_CONFIG_GOLDEN; 1702 break; 1703 case CHIP_POLARIS10: 1704 case CHIP_VEGAM: 1705 ret = amdgpu_atombios_get_gfx_info(adev); 1706 if (ret) 1707 return ret; 1708 adev->gfx.config.max_gprs = 256; 1709 adev->gfx.config.max_gs_threads = 32; 1710 adev->gfx.config.max_hw_contexts = 8; 1711 1712 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1713 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1714 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1715 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1716 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1717 break; 1718 case CHIP_TONGA: 1719 adev->gfx.config.max_shader_engines = 4; 1720 adev->gfx.config.max_tile_pipes = 8; 1721 adev->gfx.config.max_cu_per_sh = 8; 1722 adev->gfx.config.max_sh_per_se = 1; 1723 adev->gfx.config.max_backends_per_se = 2; 1724 adev->gfx.config.max_texture_channel_caches = 8; 1725 adev->gfx.config.max_gprs = 256; 1726 adev->gfx.config.max_gs_threads = 32; 1727 adev->gfx.config.max_hw_contexts = 8; 1728 1729 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1730 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1731 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1732 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1733 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1734 break; 1735 case CHIP_CARRIZO: 1736 adev->gfx.config.max_shader_engines = 1; 1737 adev->gfx.config.max_tile_pipes = 2; 1738 adev->gfx.config.max_sh_per_se = 1; 1739 adev->gfx.config.max_backends_per_se = 2; 1740 adev->gfx.config.max_cu_per_sh = 8; 1741 adev->gfx.config.max_texture_channel_caches = 2; 1742 adev->gfx.config.max_gprs = 256; 1743 adev->gfx.config.max_gs_threads = 32; 1744 adev->gfx.config.max_hw_contexts = 8; 1745 1746 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1747 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1748 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1749 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1750 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1751 break; 1752 case CHIP_STONEY: 1753 adev->gfx.config.max_shader_engines = 1; 1754 adev->gfx.config.max_tile_pipes = 2; 1755 adev->gfx.config.max_sh_per_se = 1; 1756 adev->gfx.config.max_backends_per_se = 1; 1757 adev->gfx.config.max_cu_per_sh = 3; 1758 adev->gfx.config.max_texture_channel_caches = 2; 1759 adev->gfx.config.max_gprs = 256; 1760 adev->gfx.config.max_gs_threads = 16; 1761 adev->gfx.config.max_hw_contexts = 8; 1762 1763 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1764 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1765 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1766 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1767 gb_addr_config = CARRIZO_GB_ADDR_CONFIG_GOLDEN; 1768 break; 1769 default: 1770 adev->gfx.config.max_shader_engines = 2; 1771 adev->gfx.config.max_tile_pipes = 4; 1772 adev->gfx.config.max_cu_per_sh = 2; 1773 adev->gfx.config.max_sh_per_se = 1; 1774 adev->gfx.config.max_backends_per_se = 2; 1775 adev->gfx.config.max_texture_channel_caches = 4; 1776 adev->gfx.config.max_gprs = 256; 1777 adev->gfx.config.max_gs_threads = 32; 1778 adev->gfx.config.max_hw_contexts = 8; 1779 1780 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1781 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1782 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 1783 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x130; 1784 gb_addr_config = TONGA_GB_ADDR_CONFIG_GOLDEN; 1785 break; 1786 } 1787 1788 adev->gfx.config.mc_arb_ramcfg = RREG32(mmMC_ARB_RAMCFG); 1789 mc_arb_ramcfg = adev->gfx.config.mc_arb_ramcfg; 1790 1791 adev->gfx.config.num_banks = REG_GET_FIELD(mc_arb_ramcfg, 1792 MC_ARB_RAMCFG, NOOFBANK); 1793 adev->gfx.config.num_ranks = REG_GET_FIELD(mc_arb_ramcfg, 1794 MC_ARB_RAMCFG, NOOFRANKS); 1795 1796 adev->gfx.config.num_tile_pipes = adev->gfx.config.max_tile_pipes; 1797 adev->gfx.config.mem_max_burst_length_bytes = 256; 1798 if (adev->flags & AMD_IS_APU) { 1799 /* Get memory bank mapping mode. */ 1800 tmp = RREG32(mmMC_FUS_DRAM0_BANK_ADDR_MAPPING); 1801 dimm00_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1802 dimm01_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM0_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1803 1804 tmp = RREG32(mmMC_FUS_DRAM1_BANK_ADDR_MAPPING); 1805 dimm10_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM0ADDRMAP); 1806 dimm11_addr_map = REG_GET_FIELD(tmp, MC_FUS_DRAM1_BANK_ADDR_MAPPING, DIMM1ADDRMAP); 1807 1808 /* Validate settings in case only one DIMM installed. */ 1809 if ((dimm00_addr_map == 0) || (dimm00_addr_map == 3) || (dimm00_addr_map == 4) || (dimm00_addr_map > 12)) 1810 dimm00_addr_map = 0; 1811 if ((dimm01_addr_map == 0) || (dimm01_addr_map == 3) || (dimm01_addr_map == 4) || (dimm01_addr_map > 12)) 1812 dimm01_addr_map = 0; 1813 if ((dimm10_addr_map == 0) || (dimm10_addr_map == 3) || (dimm10_addr_map == 4) || (dimm10_addr_map > 12)) 1814 dimm10_addr_map = 0; 1815 if ((dimm11_addr_map == 0) || (dimm11_addr_map == 3) || (dimm11_addr_map == 4) || (dimm11_addr_map > 12)) 1816 dimm11_addr_map = 0; 1817 1818 /* If DIMM Addr map is 8GB, ROW size should be 2KB. Otherwise 1KB. */ 1819 /* If ROW size(DIMM1) != ROW size(DMIMM0), ROW size should be larger one. */ 1820 if ((dimm00_addr_map == 11) || (dimm01_addr_map == 11) || (dimm10_addr_map == 11) || (dimm11_addr_map == 11)) 1821 adev->gfx.config.mem_row_size_in_kb = 2; 1822 else 1823 adev->gfx.config.mem_row_size_in_kb = 1; 1824 } else { 1825 tmp = REG_GET_FIELD(mc_arb_ramcfg, MC_ARB_RAMCFG, NOOFCOLS); 1826 adev->gfx.config.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024; 1827 if (adev->gfx.config.mem_row_size_in_kb > 4) 1828 adev->gfx.config.mem_row_size_in_kb = 4; 1829 } 1830 1831 adev->gfx.config.shader_engine_tile_size = 32; 1832 adev->gfx.config.num_gpus = 1; 1833 adev->gfx.config.multi_gpu_tile_size = 64; 1834 1835 /* fix up row size */ 1836 switch (adev->gfx.config.mem_row_size_in_kb) { 1837 case 1: 1838 default: 1839 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 0); 1840 break; 1841 case 2: 1842 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 1); 1843 break; 1844 case 4: 1845 gb_addr_config = REG_SET_FIELD(gb_addr_config, GB_ADDR_CONFIG, ROW_SIZE, 2); 1846 break; 1847 } 1848 adev->gfx.config.gb_addr_config = gb_addr_config; 1849 1850 return 0; 1851 } 1852 1853 static int gfx_v8_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1854 int mec, int pipe, int queue) 1855 { 1856 int r; 1857 unsigned irq_type; 1858 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 1859 unsigned int hw_prio; 1860 1861 ring = &adev->gfx.compute_ring[ring_id]; 1862 1863 /* mec0 is me1 */ 1864 ring->me = mec + 1; 1865 ring->pipe = pipe; 1866 ring->queue = queue; 1867 1868 ring->ring_obj = NULL; 1869 ring->use_doorbell = true; 1870 ring->doorbell_index = adev->doorbell_index.mec_ring0 + ring_id; 1871 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1872 + (ring_id * GFX8_MEC_HPD_SIZE); 1873 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1874 1875 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1876 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1877 + ring->pipe; 1878 1879 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1880 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT; 1881 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1882 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1883 hw_prio, NULL); 1884 if (r) 1885 return r; 1886 1887 1888 return 0; 1889 } 1890 1891 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work); 1892 1893 static int gfx_v8_0_sw_init(struct amdgpu_ip_block *ip_block) 1894 { 1895 int i, j, k, r, ring_id; 1896 int xcc_id = 0; 1897 struct amdgpu_ring *ring; 1898 struct amdgpu_device *adev = ip_block->adev; 1899 1900 switch (adev->asic_type) { 1901 case CHIP_TONGA: 1902 case CHIP_CARRIZO: 1903 case CHIP_FIJI: 1904 case CHIP_POLARIS10: 1905 case CHIP_POLARIS11: 1906 case CHIP_POLARIS12: 1907 case CHIP_VEGAM: 1908 adev->gfx.mec.num_mec = 2; 1909 break; 1910 case CHIP_TOPAZ: 1911 case CHIP_STONEY: 1912 default: 1913 adev->gfx.mec.num_mec = 1; 1914 break; 1915 } 1916 1917 adev->gfx.mec.num_pipe_per_mec = 4; 1918 adev->gfx.mec.num_queue_per_pipe = 8; 1919 1920 /* EOP Event */ 1921 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_END_OF_PIPE, &adev->gfx.eop_irq); 1922 if (r) 1923 return r; 1924 1925 /* Privileged reg */ 1926 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_REG_FAULT, 1927 &adev->gfx.priv_reg_irq); 1928 if (r) 1929 return r; 1930 1931 /* Privileged inst */ 1932 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_PRIV_INSTR_FAULT, 1933 &adev->gfx.priv_inst_irq); 1934 if (r) 1935 return r; 1936 1937 /* Add CP EDC/ECC irq */ 1938 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_CP_ECC_ERROR, 1939 &adev->gfx.cp_ecc_error_irq); 1940 if (r) 1941 return r; 1942 1943 /* SQ interrupts. */ 1944 r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_SQ_INTERRUPT_MSG, 1945 &adev->gfx.sq_irq); 1946 if (r) { 1947 DRM_ERROR("amdgpu_irq_add() for SQ failed: %d\n", r); 1948 return r; 1949 } 1950 1951 INIT_WORK(&adev->gfx.sq_work.work, gfx_v8_0_sq_irq_work_func); 1952 1953 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1954 1955 r = gfx_v8_0_init_microcode(adev); 1956 if (r) { 1957 DRM_ERROR("Failed to load gfx firmware!\n"); 1958 return r; 1959 } 1960 1961 r = adev->gfx.rlc.funcs->init(adev); 1962 if (r) { 1963 DRM_ERROR("Failed to init rlc BOs!\n"); 1964 return r; 1965 } 1966 1967 r = gfx_v8_0_mec_init(adev); 1968 if (r) { 1969 DRM_ERROR("Failed to init MEC BOs!\n"); 1970 return r; 1971 } 1972 1973 /* set up the gfx ring */ 1974 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 1975 ring = &adev->gfx.gfx_ring[i]; 1976 ring->ring_obj = NULL; 1977 sprintf(ring->name, "gfx"); 1978 /* no gfx doorbells on iceland */ 1979 if (adev->asic_type != CHIP_TOPAZ) { 1980 ring->use_doorbell = true; 1981 ring->doorbell_index = adev->doorbell_index.gfx_ring0; 1982 } 1983 1984 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 1985 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 1986 AMDGPU_RING_PRIO_DEFAULT, NULL); 1987 if (r) 1988 return r; 1989 } 1990 1991 1992 /* set up the compute queues - allocate horizontally across pipes */ 1993 ring_id = 0; 1994 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1995 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1996 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1997 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 1998 k, j)) 1999 continue; 2000 2001 r = gfx_v8_0_compute_ring_init(adev, 2002 ring_id, 2003 i, k, j); 2004 if (r) 2005 return r; 2006 2007 ring_id++; 2008 } 2009 } 2010 } 2011 2012 r = amdgpu_gfx_kiq_init(adev, GFX8_MEC_HPD_SIZE, 0); 2013 if (r) { 2014 DRM_ERROR("Failed to init KIQ BOs!\n"); 2015 return r; 2016 } 2017 2018 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 2019 if (r) 2020 return r; 2021 2022 /* create MQD for all compute queues as well as KIQ for SRIOV case */ 2023 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct vi_mqd_allocation), 0); 2024 if (r) 2025 return r; 2026 2027 adev->gfx.ce_ram_size = 0x8000; 2028 2029 r = gfx_v8_0_gpu_early_init(adev); 2030 if (r) 2031 return r; 2032 2033 adev->gfx.gfx_supported_reset = 2034 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 2035 adev->gfx.compute_supported_reset = 2036 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 2037 2038 if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_ip_block_soft_reset) { 2039 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; 2040 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; 2041 } 2042 2043 return 0; 2044 } 2045 2046 static int gfx_v8_0_sw_fini(struct amdgpu_ip_block *ip_block) 2047 { 2048 struct amdgpu_device *adev = ip_block->adev; 2049 int i; 2050 2051 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2052 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2053 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2054 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2055 2056 amdgpu_gfx_mqd_sw_fini(adev, 0); 2057 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 2058 amdgpu_gfx_kiq_fini(adev, 0); 2059 2060 gfx_v8_0_mec_fini(adev); 2061 amdgpu_gfx_rlc_fini(adev); 2062 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2063 &adev->gfx.rlc.clear_state_gpu_addr, 2064 (void **)&adev->gfx.rlc.cs_ptr); 2065 if ((adev->asic_type == CHIP_CARRIZO) || 2066 (adev->asic_type == CHIP_STONEY)) { 2067 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2068 &adev->gfx.rlc.cp_table_gpu_addr, 2069 (void **)&adev->gfx.rlc.cp_table_ptr); 2070 } 2071 gfx_v8_0_free_microcode(adev); 2072 2073 return 0; 2074 } 2075 2076 static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev) 2077 { 2078 uint32_t *modearray, *mod2array; 2079 const u32 num_tile_mode_states = ARRAY_SIZE(adev->gfx.config.tile_mode_array); 2080 const u32 num_secondary_tile_mode_states = ARRAY_SIZE(adev->gfx.config.macrotile_mode_array); 2081 u32 reg_offset; 2082 2083 modearray = adev->gfx.config.tile_mode_array; 2084 mod2array = adev->gfx.config.macrotile_mode_array; 2085 2086 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2087 modearray[reg_offset] = 0; 2088 2089 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2090 mod2array[reg_offset] = 0; 2091 2092 switch (adev->asic_type) { 2093 case CHIP_TOPAZ: 2094 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2095 PIPE_CONFIG(ADDR_SURF_P2) | 2096 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2097 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2098 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2099 PIPE_CONFIG(ADDR_SURF_P2) | 2100 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2101 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2102 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2103 PIPE_CONFIG(ADDR_SURF_P2) | 2104 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2105 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2106 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2107 PIPE_CONFIG(ADDR_SURF_P2) | 2108 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2109 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2110 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2111 PIPE_CONFIG(ADDR_SURF_P2) | 2112 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2113 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2114 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2115 PIPE_CONFIG(ADDR_SURF_P2) | 2116 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2117 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2118 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2119 PIPE_CONFIG(ADDR_SURF_P2) | 2120 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2121 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2122 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2123 PIPE_CONFIG(ADDR_SURF_P2)); 2124 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2125 PIPE_CONFIG(ADDR_SURF_P2) | 2126 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2127 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2128 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2129 PIPE_CONFIG(ADDR_SURF_P2) | 2130 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2131 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2132 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2133 PIPE_CONFIG(ADDR_SURF_P2) | 2134 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2135 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2136 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2137 PIPE_CONFIG(ADDR_SURF_P2) | 2138 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2139 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2140 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2141 PIPE_CONFIG(ADDR_SURF_P2) | 2142 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2143 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2144 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2145 PIPE_CONFIG(ADDR_SURF_P2) | 2146 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2147 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2148 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2149 PIPE_CONFIG(ADDR_SURF_P2) | 2150 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2151 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2152 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2153 PIPE_CONFIG(ADDR_SURF_P2) | 2154 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2155 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2156 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2157 PIPE_CONFIG(ADDR_SURF_P2) | 2158 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2159 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2160 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2161 PIPE_CONFIG(ADDR_SURF_P2) | 2162 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2163 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2164 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2165 PIPE_CONFIG(ADDR_SURF_P2) | 2166 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2167 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2168 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2169 PIPE_CONFIG(ADDR_SURF_P2) | 2170 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2171 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2172 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2173 PIPE_CONFIG(ADDR_SURF_P2) | 2174 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2175 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2176 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2177 PIPE_CONFIG(ADDR_SURF_P2) | 2178 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2179 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2180 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2181 PIPE_CONFIG(ADDR_SURF_P2) | 2182 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2183 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2184 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2185 PIPE_CONFIG(ADDR_SURF_P2) | 2186 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2187 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2188 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2189 PIPE_CONFIG(ADDR_SURF_P2) | 2190 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2191 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2192 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2193 PIPE_CONFIG(ADDR_SURF_P2) | 2194 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2195 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2196 2197 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2198 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2199 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2200 NUM_BANKS(ADDR_SURF_8_BANK)); 2201 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2202 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2203 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2204 NUM_BANKS(ADDR_SURF_8_BANK)); 2205 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2206 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2207 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2208 NUM_BANKS(ADDR_SURF_8_BANK)); 2209 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2210 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2211 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2212 NUM_BANKS(ADDR_SURF_8_BANK)); 2213 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2214 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2215 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2216 NUM_BANKS(ADDR_SURF_8_BANK)); 2217 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2218 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2219 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2220 NUM_BANKS(ADDR_SURF_8_BANK)); 2221 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2222 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2223 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2224 NUM_BANKS(ADDR_SURF_8_BANK)); 2225 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2226 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2227 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2228 NUM_BANKS(ADDR_SURF_16_BANK)); 2229 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 2230 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2231 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2232 NUM_BANKS(ADDR_SURF_16_BANK)); 2233 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2234 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2235 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2236 NUM_BANKS(ADDR_SURF_16_BANK)); 2237 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2238 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2239 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2240 NUM_BANKS(ADDR_SURF_16_BANK)); 2241 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2242 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2243 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2244 NUM_BANKS(ADDR_SURF_16_BANK)); 2245 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2246 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2247 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2248 NUM_BANKS(ADDR_SURF_16_BANK)); 2249 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2250 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2251 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2252 NUM_BANKS(ADDR_SURF_8_BANK)); 2253 2254 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2255 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 2256 reg_offset != 23) 2257 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2258 2259 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2260 if (reg_offset != 7) 2261 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2262 2263 break; 2264 case CHIP_FIJI: 2265 case CHIP_VEGAM: 2266 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2267 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2268 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2269 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2270 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2271 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2272 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2273 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2274 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2275 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2276 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2277 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2278 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2279 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2280 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2281 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2282 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2283 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2284 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2285 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2286 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2287 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2288 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2289 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2290 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2291 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2292 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2293 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2294 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2295 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2296 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2297 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2298 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2299 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16)); 2300 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2301 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2302 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2304 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2305 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2306 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2307 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2308 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2309 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2310 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2311 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2312 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2313 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2314 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2315 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2316 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2317 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2318 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2320 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2321 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2322 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2323 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2324 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2325 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2326 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2327 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2328 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2329 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2330 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2331 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2332 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2333 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2334 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2335 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2336 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2337 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2338 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2339 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2340 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2341 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2342 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2343 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2344 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2345 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2346 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2347 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2348 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2349 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2350 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2351 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2352 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2353 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2354 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2355 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2356 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2357 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2358 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2359 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2360 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2361 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2362 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2363 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2364 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2365 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2366 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2367 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2368 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2369 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2370 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2371 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2372 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2373 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2374 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2375 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2376 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2377 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2378 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2379 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2380 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2381 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) | 2382 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2383 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2384 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2385 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2386 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2387 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2388 2389 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2390 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2391 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2392 NUM_BANKS(ADDR_SURF_8_BANK)); 2393 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2394 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2395 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2396 NUM_BANKS(ADDR_SURF_8_BANK)); 2397 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2398 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2399 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2400 NUM_BANKS(ADDR_SURF_8_BANK)); 2401 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2402 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2403 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2404 NUM_BANKS(ADDR_SURF_8_BANK)); 2405 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2406 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2407 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2408 NUM_BANKS(ADDR_SURF_8_BANK)); 2409 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2410 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2411 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2412 NUM_BANKS(ADDR_SURF_8_BANK)); 2413 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2414 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2415 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2416 NUM_BANKS(ADDR_SURF_8_BANK)); 2417 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2418 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2419 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2420 NUM_BANKS(ADDR_SURF_8_BANK)); 2421 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2422 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2423 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2424 NUM_BANKS(ADDR_SURF_8_BANK)); 2425 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2426 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2427 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2428 NUM_BANKS(ADDR_SURF_8_BANK)); 2429 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2430 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2431 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2432 NUM_BANKS(ADDR_SURF_8_BANK)); 2433 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2434 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2435 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2436 NUM_BANKS(ADDR_SURF_8_BANK)); 2437 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2438 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2439 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2440 NUM_BANKS(ADDR_SURF_8_BANK)); 2441 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2442 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2443 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2444 NUM_BANKS(ADDR_SURF_4_BANK)); 2445 2446 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2447 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2448 2449 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2450 if (reg_offset != 7) 2451 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2452 2453 break; 2454 case CHIP_TONGA: 2455 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2456 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2457 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2458 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2459 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2460 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2461 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2462 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2463 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2464 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2465 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2466 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2467 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2468 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2469 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2470 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2471 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2472 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2473 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2474 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2475 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2476 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2477 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2478 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2479 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2480 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2481 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2482 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2483 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2484 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2485 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2486 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2487 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2488 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2489 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2490 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2491 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2492 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2493 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2494 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2495 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2496 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2497 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2498 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2499 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2500 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2501 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2502 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2503 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2504 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2505 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2506 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2507 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2508 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2509 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2510 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2511 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2512 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2513 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2514 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2515 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2516 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2517 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2518 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2519 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2520 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2521 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2522 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2523 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2524 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2525 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2526 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2527 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2529 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2530 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2531 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2532 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2533 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2535 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2536 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2537 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2538 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2539 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2540 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2541 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2542 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2543 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2544 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2545 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2546 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2547 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2548 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2549 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2550 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2551 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2552 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2553 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2554 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2555 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2556 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2557 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2558 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2559 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2560 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2561 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2562 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2563 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2564 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2565 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2566 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2567 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2568 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2569 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2571 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2572 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2573 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2574 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2575 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2576 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2577 2578 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2579 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2580 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2581 NUM_BANKS(ADDR_SURF_16_BANK)); 2582 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2583 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2584 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2585 NUM_BANKS(ADDR_SURF_16_BANK)); 2586 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2587 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2588 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2589 NUM_BANKS(ADDR_SURF_16_BANK)); 2590 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2591 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2592 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2593 NUM_BANKS(ADDR_SURF_16_BANK)); 2594 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2595 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2596 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2597 NUM_BANKS(ADDR_SURF_16_BANK)); 2598 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2599 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2600 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2601 NUM_BANKS(ADDR_SURF_16_BANK)); 2602 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2603 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2604 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2605 NUM_BANKS(ADDR_SURF_16_BANK)); 2606 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2607 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2608 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2609 NUM_BANKS(ADDR_SURF_16_BANK)); 2610 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2611 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2612 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2613 NUM_BANKS(ADDR_SURF_16_BANK)); 2614 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2615 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2616 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2617 NUM_BANKS(ADDR_SURF_16_BANK)); 2618 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2621 NUM_BANKS(ADDR_SURF_16_BANK)); 2622 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2623 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2624 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2625 NUM_BANKS(ADDR_SURF_8_BANK)); 2626 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2627 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2628 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2629 NUM_BANKS(ADDR_SURF_4_BANK)); 2630 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2631 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2632 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2633 NUM_BANKS(ADDR_SURF_4_BANK)); 2634 2635 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2636 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2637 2638 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2639 if (reg_offset != 7) 2640 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2641 2642 break; 2643 case CHIP_POLARIS11: 2644 case CHIP_POLARIS12: 2645 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2646 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2647 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2648 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2649 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2650 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2651 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2652 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2653 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2654 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2655 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2656 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2657 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2658 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2659 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2660 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2661 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2662 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2663 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2664 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2665 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2666 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2667 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2668 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2669 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2670 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2671 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2672 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2673 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2674 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2675 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2676 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2677 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2678 PIPE_CONFIG(ADDR_SURF_P4_16x16)); 2679 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2680 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2681 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2682 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2683 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2684 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2685 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2686 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2687 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2688 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2689 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2690 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2691 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2692 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2693 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2694 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2695 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2696 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2697 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2698 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2699 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2700 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2701 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2702 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2703 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2704 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2705 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2706 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2707 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2708 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2709 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2710 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2711 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2712 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2713 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2714 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2715 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2716 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2717 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2718 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2719 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2720 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2721 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2722 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2723 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2724 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2725 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2726 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2727 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2728 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2729 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2730 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2731 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2732 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2733 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2734 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2735 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2736 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2737 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2738 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2739 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2740 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2741 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2742 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2743 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2744 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2745 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2746 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2747 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2748 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2749 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2750 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2751 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2752 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2753 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2754 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2755 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2756 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2757 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2758 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2759 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2760 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2761 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2762 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2763 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2764 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2765 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2766 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2767 2768 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2769 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2770 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2771 NUM_BANKS(ADDR_SURF_16_BANK)); 2772 2773 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2774 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2775 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2776 NUM_BANKS(ADDR_SURF_16_BANK)); 2777 2778 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2779 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2780 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2781 NUM_BANKS(ADDR_SURF_16_BANK)); 2782 2783 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2784 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2785 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2786 NUM_BANKS(ADDR_SURF_16_BANK)); 2787 2788 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2789 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2790 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2791 NUM_BANKS(ADDR_SURF_16_BANK)); 2792 2793 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2794 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2795 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2796 NUM_BANKS(ADDR_SURF_16_BANK)); 2797 2798 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2799 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2800 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2801 NUM_BANKS(ADDR_SURF_16_BANK)); 2802 2803 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2804 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 2805 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2806 NUM_BANKS(ADDR_SURF_16_BANK)); 2807 2808 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 2809 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2810 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2811 NUM_BANKS(ADDR_SURF_16_BANK)); 2812 2813 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2814 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2815 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2816 NUM_BANKS(ADDR_SURF_16_BANK)); 2817 2818 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2819 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2820 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2821 NUM_BANKS(ADDR_SURF_16_BANK)); 2822 2823 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2824 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2825 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2826 NUM_BANKS(ADDR_SURF_16_BANK)); 2827 2828 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2829 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2830 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2831 NUM_BANKS(ADDR_SURF_8_BANK)); 2832 2833 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2836 NUM_BANKS(ADDR_SURF_4_BANK)); 2837 2838 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 2839 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 2840 2841 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 2842 if (reg_offset != 7) 2843 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 2844 2845 break; 2846 case CHIP_POLARIS10: 2847 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2848 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 2850 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2851 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2852 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2853 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 2854 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2855 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2856 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2857 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 2858 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2859 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2860 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2861 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 2862 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2863 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2864 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2865 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2866 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2867 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2868 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2869 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2870 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2871 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2872 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2873 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2874 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2875 modearray[7] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2876 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2877 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 2878 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 2879 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 2880 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16)); 2881 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2882 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2883 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2884 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2885 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2886 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2887 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2888 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2889 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2890 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2891 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2892 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2893 modearray[12] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2894 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2895 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 2896 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2897 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2898 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2899 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2900 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2901 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2902 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2903 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2904 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2905 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 2906 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2907 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2908 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2909 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2910 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2911 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2912 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2913 modearray[17] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2914 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2915 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2916 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2917 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2918 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2919 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2920 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2921 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 2922 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2923 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2924 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2925 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2926 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2927 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2928 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2929 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 2930 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2931 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2932 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2933 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2934 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2935 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2936 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2937 modearray[23] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 2938 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2939 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2940 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2941 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 2942 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2943 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 2944 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2945 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 2946 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2947 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2948 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2949 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 2950 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2951 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 2952 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 2953 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 2954 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2955 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2956 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2957 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 2958 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2959 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2960 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 2961 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2962 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) | 2963 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2964 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2965 modearray[30] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 2966 PIPE_CONFIG(ADDR_SURF_P4_16x16) | 2967 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 2968 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 2969 2970 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2971 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2972 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2973 NUM_BANKS(ADDR_SURF_16_BANK)); 2974 2975 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2976 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2977 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2978 NUM_BANKS(ADDR_SURF_16_BANK)); 2979 2980 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2981 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2982 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2983 NUM_BANKS(ADDR_SURF_16_BANK)); 2984 2985 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2986 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 2987 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 2988 NUM_BANKS(ADDR_SURF_16_BANK)); 2989 2990 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2991 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 2992 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 2993 NUM_BANKS(ADDR_SURF_16_BANK)); 2994 2995 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 2996 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 2997 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 2998 NUM_BANKS(ADDR_SURF_16_BANK)); 2999 3000 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3001 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3002 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3003 NUM_BANKS(ADDR_SURF_16_BANK)); 3004 3005 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3006 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3007 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3008 NUM_BANKS(ADDR_SURF_16_BANK)); 3009 3010 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3011 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3012 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3013 NUM_BANKS(ADDR_SURF_16_BANK)); 3014 3015 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3016 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3017 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3018 NUM_BANKS(ADDR_SURF_16_BANK)); 3019 3020 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3021 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3022 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3023 NUM_BANKS(ADDR_SURF_16_BANK)); 3024 3025 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3026 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3027 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3028 NUM_BANKS(ADDR_SURF_8_BANK)); 3029 3030 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3031 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3032 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3033 NUM_BANKS(ADDR_SURF_4_BANK)); 3034 3035 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3036 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3037 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) | 3038 NUM_BANKS(ADDR_SURF_4_BANK)); 3039 3040 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3041 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3042 3043 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3044 if (reg_offset != 7) 3045 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3046 3047 break; 3048 case CHIP_STONEY: 3049 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3050 PIPE_CONFIG(ADDR_SURF_P2) | 3051 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3052 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3053 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3054 PIPE_CONFIG(ADDR_SURF_P2) | 3055 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3056 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3057 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3058 PIPE_CONFIG(ADDR_SURF_P2) | 3059 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3060 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3061 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3062 PIPE_CONFIG(ADDR_SURF_P2) | 3063 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3064 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3065 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3066 PIPE_CONFIG(ADDR_SURF_P2) | 3067 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3068 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3069 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3070 PIPE_CONFIG(ADDR_SURF_P2) | 3071 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3072 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3073 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3074 PIPE_CONFIG(ADDR_SURF_P2) | 3075 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3076 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3077 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3078 PIPE_CONFIG(ADDR_SURF_P2)); 3079 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3080 PIPE_CONFIG(ADDR_SURF_P2) | 3081 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3082 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3083 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3084 PIPE_CONFIG(ADDR_SURF_P2) | 3085 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3086 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3087 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3088 PIPE_CONFIG(ADDR_SURF_P2) | 3089 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3090 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3091 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3092 PIPE_CONFIG(ADDR_SURF_P2) | 3093 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3094 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3095 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3096 PIPE_CONFIG(ADDR_SURF_P2) | 3097 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3098 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3099 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3100 PIPE_CONFIG(ADDR_SURF_P2) | 3101 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3102 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3103 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3104 PIPE_CONFIG(ADDR_SURF_P2) | 3105 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3106 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3107 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3108 PIPE_CONFIG(ADDR_SURF_P2) | 3109 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3110 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3111 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3112 PIPE_CONFIG(ADDR_SURF_P2) | 3113 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3114 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3115 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3116 PIPE_CONFIG(ADDR_SURF_P2) | 3117 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3118 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3119 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3120 PIPE_CONFIG(ADDR_SURF_P2) | 3121 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3122 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3123 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3124 PIPE_CONFIG(ADDR_SURF_P2) | 3125 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3126 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3127 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3128 PIPE_CONFIG(ADDR_SURF_P2) | 3129 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3130 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3131 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3132 PIPE_CONFIG(ADDR_SURF_P2) | 3133 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3134 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3135 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3136 PIPE_CONFIG(ADDR_SURF_P2) | 3137 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3138 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3139 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3140 PIPE_CONFIG(ADDR_SURF_P2) | 3141 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3142 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3143 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3144 PIPE_CONFIG(ADDR_SURF_P2) | 3145 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3146 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3147 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3148 PIPE_CONFIG(ADDR_SURF_P2) | 3149 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3150 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3151 3152 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3153 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3154 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3155 NUM_BANKS(ADDR_SURF_8_BANK)); 3156 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3157 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3158 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3159 NUM_BANKS(ADDR_SURF_8_BANK)); 3160 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3161 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3162 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3163 NUM_BANKS(ADDR_SURF_8_BANK)); 3164 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3165 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3166 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3167 NUM_BANKS(ADDR_SURF_8_BANK)); 3168 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3169 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3170 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3171 NUM_BANKS(ADDR_SURF_8_BANK)); 3172 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3173 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3174 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3175 NUM_BANKS(ADDR_SURF_8_BANK)); 3176 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3177 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3178 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3179 NUM_BANKS(ADDR_SURF_8_BANK)); 3180 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3181 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3182 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3183 NUM_BANKS(ADDR_SURF_16_BANK)); 3184 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3185 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3186 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3187 NUM_BANKS(ADDR_SURF_16_BANK)); 3188 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3189 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3190 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3191 NUM_BANKS(ADDR_SURF_16_BANK)); 3192 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3193 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3194 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3195 NUM_BANKS(ADDR_SURF_16_BANK)); 3196 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3197 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3198 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3199 NUM_BANKS(ADDR_SURF_16_BANK)); 3200 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3201 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3202 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3203 NUM_BANKS(ADDR_SURF_16_BANK)); 3204 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3205 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3206 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3207 NUM_BANKS(ADDR_SURF_8_BANK)); 3208 3209 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3210 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3211 reg_offset != 23) 3212 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3213 3214 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3215 if (reg_offset != 7) 3216 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3217 3218 break; 3219 default: 3220 dev_warn(adev->dev, 3221 "Unknown chip type (%d) in function gfx_v8_0_tiling_mode_table_init() falling through to CHIP_CARRIZO\n", 3222 adev->asic_type); 3223 fallthrough; 3224 3225 case CHIP_CARRIZO: 3226 modearray[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3227 PIPE_CONFIG(ADDR_SURF_P2) | 3228 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) | 3229 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3230 modearray[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3231 PIPE_CONFIG(ADDR_SURF_P2) | 3232 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) | 3233 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3234 modearray[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3235 PIPE_CONFIG(ADDR_SURF_P2) | 3236 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) | 3237 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3238 modearray[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3239 PIPE_CONFIG(ADDR_SURF_P2) | 3240 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) | 3241 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3242 modearray[4] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3243 PIPE_CONFIG(ADDR_SURF_P2) | 3244 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3245 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3246 modearray[5] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3247 PIPE_CONFIG(ADDR_SURF_P2) | 3248 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3249 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3250 modearray[6] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3251 PIPE_CONFIG(ADDR_SURF_P2) | 3252 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_2KB) | 3253 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING)); 3254 modearray[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) | 3255 PIPE_CONFIG(ADDR_SURF_P2)); 3256 modearray[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3257 PIPE_CONFIG(ADDR_SURF_P2) | 3258 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3259 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3260 modearray[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3261 PIPE_CONFIG(ADDR_SURF_P2) | 3262 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3263 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3264 modearray[11] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3265 PIPE_CONFIG(ADDR_SURF_P2) | 3266 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) | 3267 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3268 modearray[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3269 PIPE_CONFIG(ADDR_SURF_P2) | 3270 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3271 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3272 modearray[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3273 PIPE_CONFIG(ADDR_SURF_P2) | 3274 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3275 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3276 modearray[15] = (ARRAY_MODE(ARRAY_3D_TILED_THIN1) | 3277 PIPE_CONFIG(ADDR_SURF_P2) | 3278 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3279 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3280 modearray[16] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3281 PIPE_CONFIG(ADDR_SURF_P2) | 3282 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3283 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3284 modearray[18] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3285 PIPE_CONFIG(ADDR_SURF_P2) | 3286 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3287 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3288 modearray[19] = (ARRAY_MODE(ARRAY_1D_TILED_THICK) | 3289 PIPE_CONFIG(ADDR_SURF_P2) | 3290 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3291 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3292 modearray[20] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3293 PIPE_CONFIG(ADDR_SURF_P2) | 3294 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3295 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3296 modearray[21] = (ARRAY_MODE(ARRAY_3D_TILED_THICK) | 3297 PIPE_CONFIG(ADDR_SURF_P2) | 3298 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3299 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3300 modearray[22] = (ARRAY_MODE(ARRAY_PRT_TILED_THICK) | 3301 PIPE_CONFIG(ADDR_SURF_P2) | 3302 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3303 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3304 modearray[24] = (ARRAY_MODE(ARRAY_2D_TILED_THICK) | 3305 PIPE_CONFIG(ADDR_SURF_P2) | 3306 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) | 3307 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3308 modearray[25] = (ARRAY_MODE(ARRAY_2D_TILED_XTHICK) | 3309 PIPE_CONFIG(ADDR_SURF_P2) | 3310 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3311 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3312 modearray[26] = (ARRAY_MODE(ARRAY_3D_TILED_XTHICK) | 3313 PIPE_CONFIG(ADDR_SURF_P2) | 3314 MICRO_TILE_MODE_NEW(ADDR_SURF_THICK_MICRO_TILING) | 3315 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_1)); 3316 modearray[27] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) | 3317 PIPE_CONFIG(ADDR_SURF_P2) | 3318 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3319 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3320 modearray[28] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) | 3321 PIPE_CONFIG(ADDR_SURF_P2) | 3322 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3323 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2)); 3324 modearray[29] = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) | 3325 PIPE_CONFIG(ADDR_SURF_P2) | 3326 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) | 3327 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_8)); 3328 3329 mod2array[0] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3330 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3331 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3332 NUM_BANKS(ADDR_SURF_8_BANK)); 3333 mod2array[1] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3334 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3335 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3336 NUM_BANKS(ADDR_SURF_8_BANK)); 3337 mod2array[2] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3338 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3339 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3340 NUM_BANKS(ADDR_SURF_8_BANK)); 3341 mod2array[3] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3342 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3343 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3344 NUM_BANKS(ADDR_SURF_8_BANK)); 3345 mod2array[4] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3346 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3347 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3348 NUM_BANKS(ADDR_SURF_8_BANK)); 3349 mod2array[5] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3350 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3351 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3352 NUM_BANKS(ADDR_SURF_8_BANK)); 3353 mod2array[6] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3354 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3355 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3356 NUM_BANKS(ADDR_SURF_8_BANK)); 3357 mod2array[8] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3358 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) | 3359 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3360 NUM_BANKS(ADDR_SURF_16_BANK)); 3361 mod2array[9] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) | 3362 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3363 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3364 NUM_BANKS(ADDR_SURF_16_BANK)); 3365 mod2array[10] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3366 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) | 3367 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3368 NUM_BANKS(ADDR_SURF_16_BANK)); 3369 mod2array[11] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) | 3370 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3371 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3372 NUM_BANKS(ADDR_SURF_16_BANK)); 3373 mod2array[12] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3374 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) | 3375 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3376 NUM_BANKS(ADDR_SURF_16_BANK)); 3377 mod2array[13] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3378 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3379 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) | 3380 NUM_BANKS(ADDR_SURF_16_BANK)); 3381 mod2array[14] = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) | 3382 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) | 3383 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) | 3384 NUM_BANKS(ADDR_SURF_8_BANK)); 3385 3386 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) 3387 if (reg_offset != 7 && reg_offset != 12 && reg_offset != 17 && 3388 reg_offset != 23) 3389 WREG32(mmGB_TILE_MODE0 + reg_offset, modearray[reg_offset]); 3390 3391 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) 3392 if (reg_offset != 7) 3393 WREG32(mmGB_MACROTILE_MODE0 + reg_offset, mod2array[reg_offset]); 3394 3395 break; 3396 } 3397 } 3398 3399 static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, 3400 u32 se_num, u32 sh_num, u32 instance, 3401 int xcc_id) 3402 { 3403 u32 data; 3404 3405 if (instance == 0xffffffff) 3406 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 3407 else 3408 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 3409 3410 if (se_num == 0xffffffff) 3411 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 3412 else 3413 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 3414 3415 if (sh_num == 0xffffffff) 3416 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 3417 else 3418 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 3419 3420 WREG32(mmGRBM_GFX_INDEX, data); 3421 } 3422 3423 static void gfx_v8_0_select_me_pipe_q(struct amdgpu_device *adev, 3424 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 3425 { 3426 vi_srbm_select(adev, me, pipe, q, vm); 3427 } 3428 3429 static u32 gfx_v8_0_get_rb_active_bitmap(struct amdgpu_device *adev) 3430 { 3431 u32 data, mask; 3432 3433 data = RREG32(mmCC_RB_BACKEND_DISABLE) | 3434 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3435 3436 data = REG_GET_FIELD(data, GC_USER_RB_BACKEND_DISABLE, BACKEND_DISABLE); 3437 3438 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 3439 adev->gfx.config.max_sh_per_se); 3440 3441 return (~data) & mask; 3442 } 3443 3444 static void 3445 gfx_v8_0_raster_config(struct amdgpu_device *adev, u32 *rconf, u32 *rconf1) 3446 { 3447 switch (adev->asic_type) { 3448 case CHIP_FIJI: 3449 case CHIP_VEGAM: 3450 *rconf |= RB_MAP_PKR0(2) | RB_MAP_PKR1(2) | 3451 RB_XSEL2(1) | PKR_MAP(2) | 3452 PKR_XSEL(1) | PKR_YSEL(1) | 3453 SE_MAP(2) | SE_XSEL(2) | SE_YSEL(3); 3454 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(3) | 3455 SE_PAIR_YSEL(2); 3456 break; 3457 case CHIP_TONGA: 3458 case CHIP_POLARIS10: 3459 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3460 SE_XSEL(1) | SE_YSEL(1); 3461 *rconf1 |= SE_PAIR_MAP(2) | SE_PAIR_XSEL(2) | 3462 SE_PAIR_YSEL(2); 3463 break; 3464 case CHIP_TOPAZ: 3465 case CHIP_CARRIZO: 3466 *rconf |= RB_MAP_PKR0(2); 3467 *rconf1 |= 0x0; 3468 break; 3469 case CHIP_POLARIS11: 3470 case CHIP_POLARIS12: 3471 *rconf |= RB_MAP_PKR0(2) | RB_XSEL2(1) | SE_MAP(2) | 3472 SE_XSEL(1) | SE_YSEL(1); 3473 *rconf1 |= 0x0; 3474 break; 3475 case CHIP_STONEY: 3476 *rconf |= 0x0; 3477 *rconf1 |= 0x0; 3478 break; 3479 default: 3480 DRM_ERROR("unknown asic: 0x%x\n", adev->asic_type); 3481 break; 3482 } 3483 } 3484 3485 static void 3486 gfx_v8_0_write_harvested_raster_configs(struct amdgpu_device *adev, 3487 u32 raster_config, u32 raster_config_1, 3488 unsigned rb_mask, unsigned num_rb) 3489 { 3490 unsigned sh_per_se = max_t(unsigned, adev->gfx.config.max_sh_per_se, 1); 3491 unsigned num_se = max_t(unsigned, adev->gfx.config.max_shader_engines, 1); 3492 unsigned rb_per_pkr = min_t(unsigned, num_rb / num_se / sh_per_se, 2); 3493 unsigned rb_per_se = num_rb / num_se; 3494 unsigned se_mask[4]; 3495 unsigned se; 3496 3497 se_mask[0] = ((1 << rb_per_se) - 1) & rb_mask; 3498 se_mask[1] = (se_mask[0] << rb_per_se) & rb_mask; 3499 se_mask[2] = (se_mask[1] << rb_per_se) & rb_mask; 3500 se_mask[3] = (se_mask[2] << rb_per_se) & rb_mask; 3501 3502 WARN_ON(!(num_se == 1 || num_se == 2 || num_se == 4)); 3503 WARN_ON(!(sh_per_se == 1 || sh_per_se == 2)); 3504 WARN_ON(!(rb_per_pkr == 1 || rb_per_pkr == 2)); 3505 3506 if ((num_se > 2) && ((!se_mask[0] && !se_mask[1]) || 3507 (!se_mask[2] && !se_mask[3]))) { 3508 raster_config_1 &= ~SE_PAIR_MAP_MASK; 3509 3510 if (!se_mask[0] && !se_mask[1]) { 3511 raster_config_1 |= 3512 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_3); 3513 } else { 3514 raster_config_1 |= 3515 SE_PAIR_MAP(RASTER_CONFIG_SE_PAIR_MAP_0); 3516 } 3517 } 3518 3519 for (se = 0; se < num_se; se++) { 3520 unsigned raster_config_se = raster_config; 3521 unsigned pkr0_mask = ((1 << rb_per_pkr) - 1) << (se * rb_per_se); 3522 unsigned pkr1_mask = pkr0_mask << rb_per_pkr; 3523 int idx = (se / 2) * 2; 3524 3525 if ((num_se > 1) && (!se_mask[idx] || !se_mask[idx + 1])) { 3526 raster_config_se &= ~SE_MAP_MASK; 3527 3528 if (!se_mask[idx]) { 3529 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_3); 3530 } else { 3531 raster_config_se |= SE_MAP(RASTER_CONFIG_SE_MAP_0); 3532 } 3533 } 3534 3535 pkr0_mask &= rb_mask; 3536 pkr1_mask &= rb_mask; 3537 if (rb_per_se > 2 && (!pkr0_mask || !pkr1_mask)) { 3538 raster_config_se &= ~PKR_MAP_MASK; 3539 3540 if (!pkr0_mask) { 3541 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_3); 3542 } else { 3543 raster_config_se |= PKR_MAP(RASTER_CONFIG_PKR_MAP_0); 3544 } 3545 } 3546 3547 if (rb_per_se >= 2) { 3548 unsigned rb0_mask = 1 << (se * rb_per_se); 3549 unsigned rb1_mask = rb0_mask << 1; 3550 3551 rb0_mask &= rb_mask; 3552 rb1_mask &= rb_mask; 3553 if (!rb0_mask || !rb1_mask) { 3554 raster_config_se &= ~RB_MAP_PKR0_MASK; 3555 3556 if (!rb0_mask) { 3557 raster_config_se |= 3558 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_3); 3559 } else { 3560 raster_config_se |= 3561 RB_MAP_PKR0(RASTER_CONFIG_RB_MAP_0); 3562 } 3563 } 3564 3565 if (rb_per_se > 2) { 3566 rb0_mask = 1 << (se * rb_per_se + rb_per_pkr); 3567 rb1_mask = rb0_mask << 1; 3568 rb0_mask &= rb_mask; 3569 rb1_mask &= rb_mask; 3570 if (!rb0_mask || !rb1_mask) { 3571 raster_config_se &= ~RB_MAP_PKR1_MASK; 3572 3573 if (!rb0_mask) { 3574 raster_config_se |= 3575 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_3); 3576 } else { 3577 raster_config_se |= 3578 RB_MAP_PKR1(RASTER_CONFIG_RB_MAP_0); 3579 } 3580 } 3581 } 3582 } 3583 3584 /* GRBM_GFX_INDEX has a different offset on VI */ 3585 gfx_v8_0_select_se_sh(adev, se, 0xffffffff, 0xffffffff, 0); 3586 WREG32(mmPA_SC_RASTER_CONFIG, raster_config_se); 3587 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3588 } 3589 3590 /* GRBM_GFX_INDEX has a different offset on VI */ 3591 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3592 } 3593 3594 static void gfx_v8_0_setup_rb(struct amdgpu_device *adev) 3595 { 3596 int i, j; 3597 u32 data; 3598 u32 raster_config = 0, raster_config_1 = 0; 3599 u32 active_rbs = 0; 3600 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 3601 adev->gfx.config.max_sh_per_se; 3602 unsigned num_rb_pipes; 3603 3604 mutex_lock(&adev->grbm_idx_mutex); 3605 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3606 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3607 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3608 data = gfx_v8_0_get_rb_active_bitmap(adev); 3609 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 3610 rb_bitmap_width_per_sh); 3611 } 3612 } 3613 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3614 3615 adev->gfx.config.backend_enable_mask = active_rbs; 3616 adev->gfx.config.num_rbs = hweight32(active_rbs); 3617 3618 num_rb_pipes = min_t(unsigned, adev->gfx.config.max_backends_per_se * 3619 adev->gfx.config.max_shader_engines, 16); 3620 3621 gfx_v8_0_raster_config(adev, &raster_config, &raster_config_1); 3622 3623 if (!adev->gfx.config.backend_enable_mask || 3624 adev->gfx.config.num_rbs >= num_rb_pipes) { 3625 WREG32(mmPA_SC_RASTER_CONFIG, raster_config); 3626 WREG32(mmPA_SC_RASTER_CONFIG_1, raster_config_1); 3627 } else { 3628 gfx_v8_0_write_harvested_raster_configs(adev, raster_config, raster_config_1, 3629 adev->gfx.config.backend_enable_mask, 3630 num_rb_pipes); 3631 } 3632 3633 /* cache the values for userspace */ 3634 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3635 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3636 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3637 adev->gfx.config.rb_config[i][j].rb_backend_disable = 3638 RREG32(mmCC_RB_BACKEND_DISABLE); 3639 adev->gfx.config.rb_config[i][j].user_rb_backend_disable = 3640 RREG32(mmGC_USER_RB_BACKEND_DISABLE); 3641 adev->gfx.config.rb_config[i][j].raster_config = 3642 RREG32(mmPA_SC_RASTER_CONFIG); 3643 adev->gfx.config.rb_config[i][j].raster_config_1 = 3644 RREG32(mmPA_SC_RASTER_CONFIG_1); 3645 } 3646 } 3647 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3648 mutex_unlock(&adev->grbm_idx_mutex); 3649 } 3650 3651 #define DEFAULT_SH_MEM_BASES (0x6000) 3652 /** 3653 * gfx_v8_0_init_compute_vmid - gart enable 3654 * 3655 * @adev: amdgpu_device pointer 3656 * 3657 * Initialize compute vmid sh_mem registers 3658 * 3659 */ 3660 static void gfx_v8_0_init_compute_vmid(struct amdgpu_device *adev) 3661 { 3662 int i; 3663 uint32_t sh_mem_config; 3664 uint32_t sh_mem_bases; 3665 3666 /* 3667 * Configure apertures: 3668 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 3669 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 3670 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 3671 */ 3672 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 3673 3674 sh_mem_config = SH_MEM_ADDRESS_MODE_HSA64 << 3675 SH_MEM_CONFIG__ADDRESS_MODE__SHIFT | 3676 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 3677 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT | 3678 MTYPE_CC << SH_MEM_CONFIG__DEFAULT_MTYPE__SHIFT | 3679 SH_MEM_CONFIG__PRIVATE_ATC_MASK; 3680 3681 mutex_lock(&adev->srbm_mutex); 3682 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3683 vi_srbm_select(adev, 0, 0, 0, i); 3684 /* CP and shaders */ 3685 WREG32(mmSH_MEM_CONFIG, sh_mem_config); 3686 WREG32(mmSH_MEM_APE1_BASE, 1); 3687 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3688 WREG32(mmSH_MEM_BASES, sh_mem_bases); 3689 } 3690 vi_srbm_select(adev, 0, 0, 0, 0); 3691 mutex_unlock(&adev->srbm_mutex); 3692 3693 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 3694 access. These should be enabled by FW for target VMIDs. */ 3695 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 3696 WREG32(amdgpu_gds_reg_offset[i].mem_base, 0); 3697 WREG32(amdgpu_gds_reg_offset[i].mem_size, 0); 3698 WREG32(amdgpu_gds_reg_offset[i].gws, 0); 3699 WREG32(amdgpu_gds_reg_offset[i].oa, 0); 3700 } 3701 } 3702 3703 static void gfx_v8_0_init_gds_vmid(struct amdgpu_device *adev) 3704 { 3705 int vmid; 3706 3707 /* 3708 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 3709 * access. Compute VMIDs should be enabled by FW for target VMIDs, 3710 * the driver can enable them for graphics. VMID0 should maintain 3711 * access so that HWS firmware can save/restore entries. 3712 */ 3713 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 3714 WREG32(amdgpu_gds_reg_offset[vmid].mem_base, 0); 3715 WREG32(amdgpu_gds_reg_offset[vmid].mem_size, 0); 3716 WREG32(amdgpu_gds_reg_offset[vmid].gws, 0); 3717 WREG32(amdgpu_gds_reg_offset[vmid].oa, 0); 3718 } 3719 } 3720 3721 static void gfx_v8_0_config_init(struct amdgpu_device *adev) 3722 { 3723 switch (adev->asic_type) { 3724 default: 3725 adev->gfx.config.double_offchip_lds_buf = 1; 3726 break; 3727 case CHIP_CARRIZO: 3728 case CHIP_STONEY: 3729 adev->gfx.config.double_offchip_lds_buf = 0; 3730 break; 3731 } 3732 } 3733 3734 static void gfx_v8_0_constants_init(struct amdgpu_device *adev) 3735 { 3736 u32 tmp, sh_static_mem_cfg; 3737 int i; 3738 3739 WREG32_FIELD(GRBM_CNTL, READ_TIMEOUT, 0xFF); 3740 WREG32(mmGB_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3741 WREG32(mmHDP_ADDR_CONFIG, adev->gfx.config.gb_addr_config); 3742 WREG32(mmDMIF_ADDR_CALC, adev->gfx.config.gb_addr_config); 3743 3744 gfx_v8_0_tiling_mode_table_init(adev); 3745 gfx_v8_0_setup_rb(adev); 3746 gfx_v8_0_get_cu_info(adev); 3747 gfx_v8_0_config_init(adev); 3748 3749 /* XXX SH_MEM regs */ 3750 /* where to put LDS, scratch, GPUVM in FSA64 space */ 3751 sh_static_mem_cfg = REG_SET_FIELD(0, SH_STATIC_MEM_CONFIG, 3752 SWIZZLE_ENABLE, 1); 3753 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3754 ELEMENT_SIZE, 1); 3755 sh_static_mem_cfg = REG_SET_FIELD(sh_static_mem_cfg, SH_STATIC_MEM_CONFIG, 3756 INDEX_STRIDE, 3); 3757 WREG32(mmSH_STATIC_MEM_CONFIG, sh_static_mem_cfg); 3758 3759 mutex_lock(&adev->srbm_mutex); 3760 for (i = 0; i < adev->vm_manager.id_mgr[0].num_ids; i++) { 3761 vi_srbm_select(adev, 0, 0, 0, i); 3762 /* CP and shaders */ 3763 if (i == 0) { 3764 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_UC); 3765 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3766 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3767 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3768 WREG32(mmSH_MEM_CONFIG, tmp); 3769 WREG32(mmSH_MEM_BASES, 0); 3770 } else { 3771 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, DEFAULT_MTYPE, MTYPE_NC); 3772 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, APE1_MTYPE, MTYPE_UC); 3773 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, ALIGNMENT_MODE, 3774 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 3775 WREG32(mmSH_MEM_CONFIG, tmp); 3776 tmp = adev->gmc.shared_aperture_start >> 48; 3777 WREG32(mmSH_MEM_BASES, tmp); 3778 } 3779 3780 WREG32(mmSH_MEM_APE1_BASE, 1); 3781 WREG32(mmSH_MEM_APE1_LIMIT, 0); 3782 } 3783 vi_srbm_select(adev, 0, 0, 0, 0); 3784 mutex_unlock(&adev->srbm_mutex); 3785 3786 gfx_v8_0_init_compute_vmid(adev); 3787 gfx_v8_0_init_gds_vmid(adev); 3788 3789 mutex_lock(&adev->grbm_idx_mutex); 3790 /* 3791 * making sure that the following register writes will be broadcasted 3792 * to all the shaders 3793 */ 3794 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3795 3796 WREG32(mmPA_SC_FIFO_SIZE, 3797 (adev->gfx.config.sc_prim_fifo_size_frontend << 3798 PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT) | 3799 (adev->gfx.config.sc_prim_fifo_size_backend << 3800 PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT) | 3801 (adev->gfx.config.sc_hiz_tile_fifo_size << 3802 PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT) | 3803 (adev->gfx.config.sc_earlyz_tile_fifo_size << 3804 PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT)); 3805 3806 tmp = RREG32(mmSPI_ARB_PRIORITY); 3807 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS0, 2); 3808 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS1, 2); 3809 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS2, 2); 3810 tmp = REG_SET_FIELD(tmp, SPI_ARB_PRIORITY, PIPE_ORDER_TS3, 2); 3811 WREG32(mmSPI_ARB_PRIORITY, tmp); 3812 3813 mutex_unlock(&adev->grbm_idx_mutex); 3814 3815 } 3816 3817 static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 3818 { 3819 u32 i, j, k; 3820 u32 mask; 3821 3822 mutex_lock(&adev->grbm_idx_mutex); 3823 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 3824 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 3825 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 3826 for (k = 0; k < adev->usec_timeout; k++) { 3827 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0) 3828 break; 3829 udelay(1); 3830 } 3831 if (k == adev->usec_timeout) { 3832 gfx_v8_0_select_se_sh(adev, 0xffffffff, 3833 0xffffffff, 0xffffffff, 0); 3834 mutex_unlock(&adev->grbm_idx_mutex); 3835 drm_info(adev_to_drm(adev), "Timeout wait for RLC serdes %u,%u\n", 3836 i, j); 3837 return; 3838 } 3839 } 3840 } 3841 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 3842 mutex_unlock(&adev->grbm_idx_mutex); 3843 3844 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 3845 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 3846 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 3847 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 3848 for (k = 0; k < adev->usec_timeout; k++) { 3849 if ((RREG32(mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 3850 break; 3851 udelay(1); 3852 } 3853 } 3854 3855 static void gfx_v8_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 3856 bool enable) 3857 { 3858 u32 tmp = RREG32(mmCP_INT_CNTL_RING0); 3859 3860 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 3861 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 3862 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 3863 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 3864 3865 WREG32(mmCP_INT_CNTL_RING0, tmp); 3866 } 3867 3868 static void gfx_v8_0_init_csb(struct amdgpu_device *adev) 3869 { 3870 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 3871 /* csib */ 3872 WREG32(mmRLC_CSIB_ADDR_HI, 3873 adev->gfx.rlc.clear_state_gpu_addr >> 32); 3874 WREG32(mmRLC_CSIB_ADDR_LO, 3875 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 3876 WREG32(mmRLC_CSIB_LENGTH, 3877 adev->gfx.rlc.clear_state_size); 3878 } 3879 3880 static void gfx_v8_0_parse_ind_reg_list(int *register_list_format, 3881 int ind_offset, 3882 int list_size, 3883 int *unique_indices, 3884 int *indices_count, 3885 int max_indices, 3886 int *ind_start_offsets, 3887 int *offset_count, 3888 int max_offset) 3889 { 3890 int indices; 3891 bool new_entry = true; 3892 3893 for (; ind_offset < list_size; ind_offset++) { 3894 3895 if (new_entry) { 3896 new_entry = false; 3897 ind_start_offsets[*offset_count] = ind_offset; 3898 *offset_count = *offset_count + 1; 3899 BUG_ON(*offset_count >= max_offset); 3900 } 3901 3902 if (register_list_format[ind_offset] == 0xFFFFFFFF) { 3903 new_entry = true; 3904 continue; 3905 } 3906 3907 ind_offset += 2; 3908 3909 /* look for the matching indice */ 3910 for (indices = 0; 3911 indices < *indices_count; 3912 indices++) { 3913 if (unique_indices[indices] == 3914 register_list_format[ind_offset]) 3915 break; 3916 } 3917 3918 if (indices >= *indices_count) { 3919 unique_indices[*indices_count] = 3920 register_list_format[ind_offset]; 3921 indices = *indices_count; 3922 *indices_count = *indices_count + 1; 3923 BUG_ON(*indices_count >= max_indices); 3924 } 3925 3926 register_list_format[ind_offset] = indices; 3927 } 3928 } 3929 3930 static int gfx_v8_0_init_save_restore_list(struct amdgpu_device *adev) 3931 { 3932 int i, temp, data; 3933 int unique_indices[] = {0, 0, 0, 0, 0, 0, 0, 0}; 3934 int indices_count = 0; 3935 int indirect_start_offsets[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 3936 int offset_count = 0; 3937 3938 int list_size; 3939 unsigned int *register_list_format = 3940 kmemdup(adev->gfx.rlc.register_list_format, 3941 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 3942 if (!register_list_format) 3943 return -ENOMEM; 3944 3945 gfx_v8_0_parse_ind_reg_list(register_list_format, 3946 RLC_FormatDirectRegListLength, 3947 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 3948 unique_indices, 3949 &indices_count, 3950 ARRAY_SIZE(unique_indices), 3951 indirect_start_offsets, 3952 &offset_count, 3953 ARRAY_SIZE(indirect_start_offsets)); 3954 3955 /* save and restore list */ 3956 WREG32_FIELD(RLC_SRM_CNTL, AUTO_INCR_ADDR, 1); 3957 3958 WREG32(mmRLC_SRM_ARAM_ADDR, 0); 3959 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 3960 WREG32(mmRLC_SRM_ARAM_DATA, adev->gfx.rlc.register_restore[i]); 3961 3962 /* indirect list */ 3963 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_list_format_start); 3964 for (i = 0; i < adev->gfx.rlc.reg_list_format_size_bytes >> 2; i++) 3965 WREG32(mmRLC_GPM_SCRATCH_DATA, register_list_format[i]); 3966 3967 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 3968 list_size = list_size >> 1; 3969 WREG32(mmRLC_GPM_SCRATCH_ADDR, adev->gfx.rlc.reg_restore_list_size); 3970 WREG32(mmRLC_GPM_SCRATCH_DATA, list_size); 3971 3972 /* starting offsets starts */ 3973 WREG32(mmRLC_GPM_SCRATCH_ADDR, 3974 adev->gfx.rlc.starting_offsets_start); 3975 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 3976 WREG32(mmRLC_GPM_SCRATCH_DATA, 3977 indirect_start_offsets[i]); 3978 3979 /* unique indices */ 3980 temp = mmRLC_SRM_INDEX_CNTL_ADDR_0; 3981 data = mmRLC_SRM_INDEX_CNTL_DATA_0; 3982 for (i = 0; i < ARRAY_SIZE(unique_indices); i++) { 3983 if (unique_indices[i] != 0) { 3984 WREG32(temp + i, unique_indices[i] & 0x3FFFF); 3985 WREG32(data + i, unique_indices[i] >> 20); 3986 } 3987 } 3988 kfree(register_list_format); 3989 3990 return 0; 3991 } 3992 3993 static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev) 3994 { 3995 WREG32_FIELD(RLC_SRM_CNTL, SRM_ENABLE, 1); 3996 } 3997 3998 static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev) 3999 { 4000 uint32_t data; 4001 4002 WREG32_FIELD(CP_RB_WPTR_POLL_CNTL, IDLE_POLL_COUNT, 0x60); 4003 4004 data = REG_SET_FIELD(0, RLC_PG_DELAY, POWER_UP_DELAY, 0x10); 4005 data = REG_SET_FIELD(data, RLC_PG_DELAY, POWER_DOWN_DELAY, 0x10); 4006 data = REG_SET_FIELD(data, RLC_PG_DELAY, CMD_PROPAGATE_DELAY, 0x10); 4007 data = REG_SET_FIELD(data, RLC_PG_DELAY, MEM_SLEEP_DELAY, 0x10); 4008 WREG32(mmRLC_PG_DELAY, data); 4009 4010 WREG32_FIELD(RLC_PG_DELAY_2, SERDES_CMD_DELAY, 0x3); 4011 WREG32_FIELD(RLC_AUTO_PG_CTRL, GRBM_REG_SAVE_GFX_IDLE_THRESHOLD, 0x55f0); 4012 4013 } 4014 4015 static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 4016 bool enable) 4017 { 4018 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PU_ENABLE, enable ? 1 : 0); 4019 } 4020 4021 static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 4022 bool enable) 4023 { 4024 WREG32_FIELD(RLC_PG_CNTL, SMU_CLK_SLOWDOWN_ON_PD_ENABLE, enable ? 1 : 0); 4025 } 4026 4027 static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable) 4028 { 4029 WREG32_FIELD(RLC_PG_CNTL, CP_PG_DISABLE, enable ? 0 : 1); 4030 } 4031 4032 static void gfx_v8_0_init_pg(struct amdgpu_device *adev) 4033 { 4034 if ((adev->asic_type == CHIP_CARRIZO) || 4035 (adev->asic_type == CHIP_STONEY)) { 4036 gfx_v8_0_init_csb(adev); 4037 gfx_v8_0_init_save_restore_list(adev); 4038 gfx_v8_0_enable_save_restore_machine(adev); 4039 WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8); 4040 gfx_v8_0_init_power_gating(adev); 4041 WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask); 4042 } else if ((adev->asic_type == CHIP_POLARIS11) || 4043 (adev->asic_type == CHIP_POLARIS12) || 4044 (adev->asic_type == CHIP_VEGAM)) { 4045 gfx_v8_0_init_csb(adev); 4046 gfx_v8_0_init_save_restore_list(adev); 4047 gfx_v8_0_enable_save_restore_machine(adev); 4048 gfx_v8_0_init_power_gating(adev); 4049 } 4050 4051 } 4052 4053 static void gfx_v8_0_rlc_stop(struct amdgpu_device *adev) 4054 { 4055 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 0); 4056 4057 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4058 gfx_v8_0_wait_for_rlc_serdes(adev); 4059 } 4060 4061 static void gfx_v8_0_rlc_reset(struct amdgpu_device *adev) 4062 { 4063 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4064 udelay(50); 4065 4066 WREG32_FIELD(GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 4067 udelay(50); 4068 } 4069 4070 static void gfx_v8_0_rlc_start(struct amdgpu_device *adev) 4071 { 4072 WREG32_FIELD(RLC_CNTL, RLC_ENABLE_F32, 1); 4073 4074 /* carrizo do enable cp interrupt after cp inited */ 4075 if (!(adev->flags & AMD_IS_APU)) 4076 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4077 4078 udelay(50); 4079 } 4080 4081 static int gfx_v8_0_rlc_resume(struct amdgpu_device *adev) 4082 { 4083 if (amdgpu_sriov_vf(adev)) { 4084 gfx_v8_0_init_csb(adev); 4085 return 0; 4086 } 4087 4088 adev->gfx.rlc.funcs->stop(adev); 4089 adev->gfx.rlc.funcs->reset(adev); 4090 gfx_v8_0_init_pg(adev); 4091 adev->gfx.rlc.funcs->start(adev); 4092 4093 return 0; 4094 } 4095 4096 static void gfx_v8_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 4097 { 4098 u32 tmp = RREG32(mmCP_ME_CNTL); 4099 4100 if (enable) { 4101 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 0); 4102 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 0); 4103 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 0); 4104 } else { 4105 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, 1); 4106 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, 1); 4107 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, 1); 4108 } 4109 WREG32(mmCP_ME_CNTL, tmp); 4110 udelay(50); 4111 } 4112 4113 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev) 4114 { 4115 u32 count = 0; 4116 const struct cs_section_def *sect = NULL; 4117 const struct cs_extent_def *ext = NULL; 4118 4119 /* begin clear state */ 4120 count += 2; 4121 /* context control state */ 4122 count += 3; 4123 4124 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4125 for (ext = sect->section; ext->extent != NULL; ++ext) { 4126 if (sect->id == SECT_CONTEXT) 4127 count += 2 + ext->reg_count; 4128 else 4129 return 0; 4130 } 4131 } 4132 /* pa_sc_raster_config/pa_sc_raster_config1 */ 4133 count += 4; 4134 /* end clear state */ 4135 count += 2; 4136 /* clear state */ 4137 count += 2; 4138 4139 return count; 4140 } 4141 4142 static int gfx_v8_0_cp_gfx_start(struct amdgpu_device *adev) 4143 { 4144 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 4145 const struct cs_section_def *sect = NULL; 4146 const struct cs_extent_def *ext = NULL; 4147 int r, i; 4148 4149 /* init the CP */ 4150 WREG32(mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 4151 WREG32(mmCP_ENDIAN_SWAP, 0); 4152 WREG32(mmCP_DEVICE_ID, 1); 4153 4154 gfx_v8_0_cp_gfx_enable(adev, true); 4155 4156 r = amdgpu_ring_alloc(ring, gfx_v8_0_get_csb_size(adev) + 4); 4157 if (r) { 4158 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 4159 return r; 4160 } 4161 4162 /* clear state buffer */ 4163 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4164 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 4165 4166 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 4167 amdgpu_ring_write(ring, 0x80000000); 4168 amdgpu_ring_write(ring, 0x80000000); 4169 4170 for (sect = vi_cs_data; sect->section != NULL; ++sect) { 4171 for (ext = sect->section; ext->extent != NULL; ++ext) { 4172 if (sect->id == SECT_CONTEXT) { 4173 amdgpu_ring_write(ring, 4174 PACKET3(PACKET3_SET_CONTEXT_REG, 4175 ext->reg_count)); 4176 amdgpu_ring_write(ring, 4177 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 4178 for (i = 0; i < ext->reg_count; i++) 4179 amdgpu_ring_write(ring, ext->extent[i]); 4180 } 4181 } 4182 } 4183 4184 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2)); 4185 amdgpu_ring_write(ring, mmPA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START); 4186 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config); 4187 amdgpu_ring_write(ring, adev->gfx.config.rb_config[0][0].raster_config_1); 4188 4189 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 4190 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 4191 4192 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 4193 amdgpu_ring_write(ring, 0); 4194 4195 /* init the CE partitions */ 4196 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 4197 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 4198 amdgpu_ring_write(ring, 0x8000); 4199 amdgpu_ring_write(ring, 0x8000); 4200 4201 amdgpu_ring_commit(ring); 4202 4203 return 0; 4204 } 4205 static void gfx_v8_0_set_cpg_door_bell(struct amdgpu_device *adev, struct amdgpu_ring *ring) 4206 { 4207 u32 tmp; 4208 /* no gfx doorbells on iceland */ 4209 if (adev->asic_type == CHIP_TOPAZ) 4210 return; 4211 4212 tmp = RREG32(mmCP_RB_DOORBELL_CONTROL); 4213 4214 if (ring->use_doorbell) { 4215 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4216 DOORBELL_OFFSET, ring->doorbell_index); 4217 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4218 DOORBELL_HIT, 0); 4219 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4220 DOORBELL_EN, 1); 4221 } else { 4222 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 4223 } 4224 4225 WREG32(mmCP_RB_DOORBELL_CONTROL, tmp); 4226 4227 if (adev->flags & AMD_IS_APU) 4228 return; 4229 4230 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 4231 DOORBELL_RANGE_LOWER, 4232 adev->doorbell_index.gfx_ring0); 4233 WREG32(mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 4234 4235 WREG32(mmCP_RB_DOORBELL_RANGE_UPPER, 4236 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 4237 } 4238 4239 static int gfx_v8_0_cp_gfx_resume(struct amdgpu_device *adev) 4240 { 4241 struct amdgpu_ring *ring; 4242 u32 tmp; 4243 u32 rb_bufsz; 4244 u64 rb_addr, rptr_addr, wptr_gpu_addr; 4245 4246 /* Set the write pointer delay */ 4247 WREG32(mmCP_RB_WPTR_DELAY, 0); 4248 4249 /* set the RB to use vmid 0 */ 4250 WREG32(mmCP_RB_VMID, 0); 4251 4252 /* Set ring buffer size */ 4253 ring = &adev->gfx.gfx_ring[0]; 4254 rb_bufsz = order_base_2(ring->ring_size / 8); 4255 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 4256 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 4257 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MTYPE, 3); 4258 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, MIN_IB_AVAILSZ, 1); 4259 #ifdef __BIG_ENDIAN 4260 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 4261 #endif 4262 WREG32(mmCP_RB0_CNTL, tmp); 4263 4264 /* Initialize the ring buffer's read and write pointers */ 4265 WREG32(mmCP_RB0_CNTL, tmp | CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK); 4266 ring->wptr = 0; 4267 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 4268 4269 /* set the wb address whether it's enabled or not */ 4270 rptr_addr = ring->rptr_gpu_addr; 4271 WREG32(mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 4272 WREG32(mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 0xFF); 4273 4274 wptr_gpu_addr = ring->wptr_gpu_addr; 4275 WREG32(mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 4276 WREG32(mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 4277 mdelay(1); 4278 WREG32(mmCP_RB0_CNTL, tmp); 4279 4280 rb_addr = ring->gpu_addr >> 8; 4281 WREG32(mmCP_RB0_BASE, rb_addr); 4282 WREG32(mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 4283 4284 gfx_v8_0_set_cpg_door_bell(adev, ring); 4285 /* start the ring */ 4286 amdgpu_ring_clear_ring(ring); 4287 gfx_v8_0_cp_gfx_start(adev); 4288 4289 return 0; 4290 } 4291 4292 static void gfx_v8_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 4293 { 4294 if (enable) { 4295 WREG32(mmCP_MEC_CNTL, 0); 4296 } else { 4297 WREG32(mmCP_MEC_CNTL, (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 4298 adev->gfx.kiq[0].ring.sched.ready = false; 4299 } 4300 udelay(50); 4301 } 4302 4303 /* KIQ functions */ 4304 static void gfx_v8_0_kiq_setting(struct amdgpu_ring *ring) 4305 { 4306 uint32_t tmp; 4307 struct amdgpu_device *adev = ring->adev; 4308 4309 /* tell RLC which is KIQ queue */ 4310 tmp = RREG32(mmRLC_CP_SCHEDULERS); 4311 tmp &= 0xffffff00; 4312 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4313 WREG32(mmRLC_CP_SCHEDULERS, tmp | 0x80); 4314 } 4315 4316 static int gfx_v8_0_kiq_kcq_enable(struct amdgpu_device *adev) 4317 { 4318 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 4319 uint64_t queue_mask = 0; 4320 int r, i; 4321 4322 for (i = 0; i < AMDGPU_MAX_COMPUTE_QUEUES; ++i) { 4323 if (!test_bit(i, adev->gfx.mec_bitmap[0].queue_bitmap)) 4324 continue; 4325 4326 /* This situation may be hit in the future if a new HW 4327 * generation exposes more than 64 queues. If so, the 4328 * definition of queue_mask needs updating */ 4329 if (WARN_ON(i >= (sizeof(queue_mask)*8))) { 4330 DRM_ERROR("Invalid KCQ enabled: %d\n", i); 4331 break; 4332 } 4333 4334 queue_mask |= (1ull << i); 4335 } 4336 4337 r = amdgpu_ring_alloc(kiq_ring, (8 * adev->gfx.num_compute_rings) + 8); 4338 if (r) { 4339 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4340 return r; 4341 } 4342 /* set resources */ 4343 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 4344 amdgpu_ring_write(kiq_ring, 0); /* vmid_mask:0 queue_type:0 (KIQ) */ 4345 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 4346 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 4347 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 4348 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 4349 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 4350 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 4351 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4352 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4353 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 4354 uint64_t wptr_addr = ring->wptr_gpu_addr; 4355 4356 /* map queues */ 4357 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 4358 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 4359 amdgpu_ring_write(kiq_ring, 4360 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 4361 amdgpu_ring_write(kiq_ring, 4362 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index) | 4363 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 4364 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 4365 PACKET3_MAP_QUEUES_ME(ring->me == 1 ? 0 : 1)); /* doorbell */ 4366 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 4367 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 4368 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 4369 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 4370 } 4371 4372 amdgpu_ring_commit(kiq_ring); 4373 4374 return 0; 4375 } 4376 4377 static int gfx_v8_0_deactivate_hqd(struct amdgpu_device *adev, u32 req) 4378 { 4379 int i, r = 0; 4380 4381 if (RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK) { 4382 WREG32_FIELD(CP_HQD_DEQUEUE_REQUEST, DEQUEUE_REQ, req); 4383 for (i = 0; i < adev->usec_timeout; i++) { 4384 if (!(RREG32(mmCP_HQD_ACTIVE) & CP_HQD_ACTIVE__ACTIVE_MASK)) 4385 break; 4386 udelay(1); 4387 } 4388 if (i == adev->usec_timeout) 4389 r = -ETIMEDOUT; 4390 } 4391 WREG32(mmCP_HQD_DEQUEUE_REQUEST, 0); 4392 WREG32(mmCP_HQD_PQ_RPTR, 0); 4393 WREG32(mmCP_HQD_PQ_WPTR, 0); 4394 4395 return r; 4396 } 4397 4398 static void gfx_v8_0_mqd_set_priority(struct amdgpu_ring *ring, struct vi_mqd *mqd) 4399 { 4400 struct amdgpu_device *adev = ring->adev; 4401 4402 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 4403 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 4404 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 4405 mqd->cp_hqd_queue_priority = 4406 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 4407 } 4408 } 4409 } 4410 4411 static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring) 4412 { 4413 struct amdgpu_device *adev = ring->adev; 4414 struct vi_mqd *mqd = ring->mqd_ptr; 4415 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4416 uint32_t tmp; 4417 4418 mqd->header = 0xC0310800; 4419 mqd->compute_pipelinestat_enable = 0x00000001; 4420 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4421 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4422 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4423 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4424 mqd->compute_misc_reserved = 0x00000003; 4425 mqd->dynamic_cu_mask_addr_lo = lower_32_bits(ring->mqd_gpu_addr 4426 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4427 mqd->dynamic_cu_mask_addr_hi = upper_32_bits(ring->mqd_gpu_addr 4428 + offsetof(struct vi_mqd_allocation, dynamic_cu_mask)); 4429 eop_base_addr = ring->eop_gpu_addr >> 8; 4430 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4431 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4432 4433 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4434 tmp = RREG32(mmCP_HQD_EOP_CONTROL); 4435 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4436 (order_base_2(GFX8_MEC_HPD_SIZE / 4) - 1)); 4437 4438 mqd->cp_hqd_eop_control = tmp; 4439 4440 /* enable doorbell? */ 4441 tmp = REG_SET_FIELD(RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL), 4442 CP_HQD_PQ_DOORBELL_CONTROL, 4443 DOORBELL_EN, 4444 ring->use_doorbell ? 1 : 0); 4445 4446 mqd->cp_hqd_pq_doorbell_control = tmp; 4447 4448 /* set the pointer to the MQD */ 4449 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 4450 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 4451 4452 /* set MQD vmid to 0 */ 4453 tmp = RREG32(mmCP_MQD_CONTROL); 4454 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4455 mqd->cp_mqd_control = tmp; 4456 4457 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4458 hqd_gpu_addr = ring->gpu_addr >> 8; 4459 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4460 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4461 4462 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4463 tmp = RREG32(mmCP_HQD_PQ_CONTROL); 4464 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4465 (order_base_2(ring->ring_size / 4) - 1)); 4466 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4467 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4468 #ifdef __BIG_ENDIAN 4469 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 4470 #endif 4471 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 4472 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 4473 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4474 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4475 mqd->cp_hqd_pq_control = tmp; 4476 4477 /* set the wb address whether it's enabled or not */ 4478 wb_gpu_addr = ring->rptr_gpu_addr; 4479 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4480 mqd->cp_hqd_pq_rptr_report_addr_hi = 4481 upper_32_bits(wb_gpu_addr) & 0xffff; 4482 4483 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4484 wb_gpu_addr = ring->wptr_gpu_addr; 4485 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4486 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4487 4488 tmp = 0; 4489 /* enable the doorbell if requested */ 4490 if (ring->use_doorbell) { 4491 tmp = RREG32(mmCP_HQD_PQ_DOORBELL_CONTROL); 4492 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4493 DOORBELL_OFFSET, ring->doorbell_index); 4494 4495 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4496 DOORBELL_EN, 1); 4497 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4498 DOORBELL_SOURCE, 0); 4499 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4500 DOORBELL_HIT, 0); 4501 } 4502 4503 mqd->cp_hqd_pq_doorbell_control = tmp; 4504 4505 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4506 ring->wptr = 0; 4507 mqd->cp_hqd_pq_wptr = ring->wptr; 4508 mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR); 4509 4510 /* set the vmid for the queue */ 4511 mqd->cp_hqd_vmid = 0; 4512 4513 tmp = RREG32(mmCP_HQD_PERSISTENT_STATE); 4514 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 4515 mqd->cp_hqd_persistent_state = tmp; 4516 4517 /* set MTYPE */ 4518 tmp = RREG32(mmCP_HQD_IB_CONTROL); 4519 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4520 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MTYPE, 3); 4521 mqd->cp_hqd_ib_control = tmp; 4522 4523 tmp = RREG32(mmCP_HQD_IQ_TIMER); 4524 tmp = REG_SET_FIELD(tmp, CP_HQD_IQ_TIMER, MTYPE, 3); 4525 mqd->cp_hqd_iq_timer = tmp; 4526 4527 tmp = RREG32(mmCP_HQD_CTX_SAVE_CONTROL); 4528 tmp = REG_SET_FIELD(tmp, CP_HQD_CTX_SAVE_CONTROL, MTYPE, 3); 4529 mqd->cp_hqd_ctx_save_control = tmp; 4530 4531 /* defaults */ 4532 mqd->cp_hqd_eop_rptr = RREG32(mmCP_HQD_EOP_RPTR); 4533 mqd->cp_hqd_eop_wptr = RREG32(mmCP_HQD_EOP_WPTR); 4534 mqd->cp_hqd_ctx_save_base_addr_lo = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_LO); 4535 mqd->cp_hqd_ctx_save_base_addr_hi = RREG32(mmCP_HQD_CTX_SAVE_BASE_ADDR_HI); 4536 mqd->cp_hqd_cntl_stack_offset = RREG32(mmCP_HQD_CNTL_STACK_OFFSET); 4537 mqd->cp_hqd_cntl_stack_size = RREG32(mmCP_HQD_CNTL_STACK_SIZE); 4538 mqd->cp_hqd_wg_state_offset = RREG32(mmCP_HQD_WG_STATE_OFFSET); 4539 mqd->cp_hqd_ctx_save_size = RREG32(mmCP_HQD_CTX_SAVE_SIZE); 4540 mqd->cp_hqd_eop_done_events = RREG32(mmCP_HQD_EOP_EVENTS); 4541 mqd->cp_hqd_error = RREG32(mmCP_HQD_ERROR); 4542 mqd->cp_hqd_eop_wptr_mem = RREG32(mmCP_HQD_EOP_WPTR_MEM); 4543 mqd->cp_hqd_eop_dones = RREG32(mmCP_HQD_EOP_DONES); 4544 4545 /* set static priority for a queue/ring */ 4546 gfx_v8_0_mqd_set_priority(ring, mqd); 4547 tmp = RREG32(mmCP_HQD_QUANTUM); 4548 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); 4549 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 4550 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10); 4551 mqd->cp_hqd_quantum = tmp; 4552 4553 /* map_queues packet doesn't need activate the queue, 4554 * so only kiq need set this field. 4555 */ 4556 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 4557 mqd->cp_hqd_active = 1; 4558 4559 return 0; 4560 } 4561 4562 static int gfx_v8_0_mqd_commit(struct amdgpu_device *adev, 4563 struct vi_mqd *mqd) 4564 { 4565 uint32_t mqd_reg; 4566 uint32_t *mqd_data; 4567 4568 /* HQD registers extend from mmCP_MQD_BASE_ADDR to mmCP_HQD_ERROR */ 4569 mqd_data = &mqd->cp_mqd_base_addr_lo; 4570 4571 /* disable wptr polling */ 4572 WREG32_FIELD(CP_PQ_WPTR_POLL_CNTL, EN, 0); 4573 4574 /* program all HQD registers */ 4575 for (mqd_reg = mmCP_HQD_VMID; mqd_reg <= mmCP_HQD_EOP_CONTROL; mqd_reg++) 4576 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4577 4578 /* Tonga errata: EOP RPTR/WPTR should be left unmodified. 4579 * This is safe since EOP RPTR==WPTR for any inactive HQD 4580 * on ASICs that do not support context-save. 4581 * EOP writes/reads can start anywhere in the ring. 4582 */ 4583 if (adev->asic_type != CHIP_TONGA) { 4584 WREG32(mmCP_HQD_EOP_RPTR, mqd->cp_hqd_eop_rptr); 4585 WREG32(mmCP_HQD_EOP_WPTR, mqd->cp_hqd_eop_wptr); 4586 WREG32(mmCP_HQD_EOP_WPTR_MEM, mqd->cp_hqd_eop_wptr_mem); 4587 } 4588 4589 for (mqd_reg = mmCP_HQD_EOP_EVENTS; mqd_reg <= mmCP_HQD_ERROR; mqd_reg++) 4590 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4591 4592 /* activate the HQD */ 4593 for (mqd_reg = mmCP_MQD_BASE_ADDR; mqd_reg <= mmCP_HQD_ACTIVE; mqd_reg++) 4594 WREG32(mqd_reg, mqd_data[mqd_reg - mmCP_MQD_BASE_ADDR]); 4595 4596 return 0; 4597 } 4598 4599 static int gfx_v8_0_kiq_init_queue(struct amdgpu_ring *ring) 4600 { 4601 struct amdgpu_device *adev = ring->adev; 4602 struct vi_mqd *mqd = ring->mqd_ptr; 4603 4604 gfx_v8_0_kiq_setting(ring); 4605 4606 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4607 /* reset MQD to a clean status */ 4608 if (adev->gfx.kiq[0].mqd_backup) 4609 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct vi_mqd_allocation)); 4610 4611 /* reset ring buffer */ 4612 ring->wptr = 0; 4613 amdgpu_ring_clear_ring(ring); 4614 mutex_lock(&adev->srbm_mutex); 4615 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4616 gfx_v8_0_mqd_commit(adev, mqd); 4617 vi_srbm_select(adev, 0, 0, 0, 0); 4618 mutex_unlock(&adev->srbm_mutex); 4619 } else { 4620 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4621 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4622 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4623 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4624 amdgpu_ring_clear_ring(ring); 4625 mutex_lock(&adev->srbm_mutex); 4626 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4627 gfx_v8_0_mqd_init(ring); 4628 gfx_v8_0_mqd_commit(adev, mqd); 4629 vi_srbm_select(adev, 0, 0, 0, 0); 4630 mutex_unlock(&adev->srbm_mutex); 4631 4632 if (adev->gfx.kiq[0].mqd_backup) 4633 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct vi_mqd_allocation)); 4634 } 4635 4636 return 0; 4637 } 4638 4639 static int gfx_v8_0_kcq_init_queue(struct amdgpu_ring *ring) 4640 { 4641 struct amdgpu_device *adev = ring->adev; 4642 struct vi_mqd *mqd = ring->mqd_ptr; 4643 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4644 4645 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4646 memset((void *)mqd, 0, sizeof(struct vi_mqd_allocation)); 4647 ((struct vi_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 4648 ((struct vi_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 4649 mutex_lock(&adev->srbm_mutex); 4650 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4651 gfx_v8_0_mqd_init(ring); 4652 vi_srbm_select(adev, 0, 0, 0, 0); 4653 mutex_unlock(&adev->srbm_mutex); 4654 4655 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4656 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct vi_mqd_allocation)); 4657 } else { 4658 /* restore MQD to a clean status */ 4659 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4660 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct vi_mqd_allocation)); 4661 /* reset ring buffer */ 4662 ring->wptr = 0; 4663 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4664 amdgpu_ring_clear_ring(ring); 4665 } 4666 return 0; 4667 } 4668 4669 static void gfx_v8_0_set_mec_doorbell_range(struct amdgpu_device *adev) 4670 { 4671 if (adev->asic_type > CHIP_TONGA) { 4672 WREG32(mmCP_MEC_DOORBELL_RANGE_LOWER, adev->doorbell_index.kiq << 2); 4673 WREG32(mmCP_MEC_DOORBELL_RANGE_UPPER, adev->doorbell_index.mec_ring7 << 2); 4674 } 4675 /* enable doorbells */ 4676 WREG32_FIELD(CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4677 } 4678 4679 static int gfx_v8_0_kiq_resume(struct amdgpu_device *adev) 4680 { 4681 gfx_v8_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 4682 return 0; 4683 } 4684 4685 static int gfx_v8_0_kcq_resume(struct amdgpu_device *adev) 4686 { 4687 int i, r; 4688 4689 gfx_v8_0_cp_compute_enable(adev, true); 4690 4691 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4692 r = gfx_v8_0_kcq_init_queue(&adev->gfx.compute_ring[i]); 4693 if (r) 4694 return r; 4695 } 4696 4697 gfx_v8_0_set_mec_doorbell_range(adev); 4698 4699 return gfx_v8_0_kiq_kcq_enable(adev); 4700 } 4701 4702 static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev) 4703 { 4704 int r, i; 4705 struct amdgpu_ring *ring; 4706 4707 /* collect all the ring_tests here, gfx, kiq, compute */ 4708 ring = &adev->gfx.gfx_ring[0]; 4709 r = amdgpu_ring_test_helper(ring); 4710 if (r) 4711 return r; 4712 4713 ring = &adev->gfx.kiq[0].ring; 4714 r = amdgpu_ring_test_helper(ring); 4715 if (r) 4716 return r; 4717 4718 r = 0; 4719 4720 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4721 ring = &adev->gfx.compute_ring[i]; 4722 r |= amdgpu_ring_test_helper(ring); 4723 } 4724 4725 return r; 4726 } 4727 4728 static int gfx_v8_0_cp_resume(struct amdgpu_device *adev) 4729 { 4730 int r; 4731 4732 if (!(adev->flags & AMD_IS_APU)) 4733 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 4734 4735 r = gfx_v8_0_kiq_resume(adev); 4736 if (r) 4737 return r; 4738 4739 r = gfx_v8_0_cp_gfx_resume(adev); 4740 if (r) 4741 return r; 4742 4743 r = gfx_v8_0_kcq_resume(adev); 4744 if (r) 4745 return r; 4746 4747 r = gfx_v8_0_cp_test_all_rings(adev); 4748 if (r) 4749 return r; 4750 4751 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 4752 4753 return 0; 4754 } 4755 4756 static void gfx_v8_0_cp_enable(struct amdgpu_device *adev, bool enable) 4757 { 4758 gfx_v8_0_cp_gfx_enable(adev, enable); 4759 gfx_v8_0_cp_compute_enable(adev, enable); 4760 } 4761 4762 static int gfx_v8_0_hw_init(struct amdgpu_ip_block *ip_block) 4763 { 4764 int r; 4765 struct amdgpu_device *adev = ip_block->adev; 4766 4767 gfx_v8_0_init_golden_registers(adev); 4768 gfx_v8_0_constants_init(adev); 4769 4770 r = adev->gfx.rlc.funcs->resume(adev); 4771 if (r) 4772 return r; 4773 4774 r = gfx_v8_0_cp_resume(adev); 4775 4776 return r; 4777 } 4778 4779 static int gfx_v8_0_kcq_disable(struct amdgpu_device *adev) 4780 { 4781 int r, i; 4782 struct amdgpu_ring *kiq_ring = &adev->gfx.kiq[0].ring; 4783 4784 r = amdgpu_ring_alloc(kiq_ring, 6 * adev->gfx.num_compute_rings); 4785 if (r) 4786 DRM_ERROR("Failed to lock KIQ (%d).\n", r); 4787 4788 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4789 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4790 4791 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 4792 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 4793 PACKET3_UNMAP_QUEUES_ACTION(1) | /* RESET_QUEUES */ 4794 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 4795 PACKET3_UNMAP_QUEUES_ENGINE_SEL(0) | 4796 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 4797 amdgpu_ring_write(kiq_ring, PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 4798 amdgpu_ring_write(kiq_ring, 0); 4799 amdgpu_ring_write(kiq_ring, 0); 4800 amdgpu_ring_write(kiq_ring, 0); 4801 } 4802 /* Submit unmap queue packet */ 4803 amdgpu_ring_commit(kiq_ring); 4804 /* 4805 * Ring test will do a basic scratch register change check. Just run 4806 * this to ensure that unmap queues that is submitted before got 4807 * processed successfully before returning. 4808 */ 4809 r = amdgpu_ring_test_helper(kiq_ring); 4810 if (r) 4811 DRM_ERROR("KCQ disable failed\n"); 4812 4813 return r; 4814 } 4815 4816 static bool gfx_v8_0_is_idle(struct amdgpu_ip_block *ip_block) 4817 { 4818 struct amdgpu_device *adev = ip_block->adev; 4819 4820 if (REG_GET_FIELD(RREG32(mmGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE) 4821 || RREG32(mmGRBM_STATUS2) != 0x8) 4822 return false; 4823 else 4824 return true; 4825 } 4826 4827 static bool gfx_v8_0_rlc_is_idle(void *handle) 4828 { 4829 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4830 4831 if (RREG32(mmGRBM_STATUS2) != 0x8) 4832 return false; 4833 else 4834 return true; 4835 } 4836 4837 static int gfx_v8_0_wait_for_rlc_idle(void *handle) 4838 { 4839 unsigned int i; 4840 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4841 4842 for (i = 0; i < adev->usec_timeout; i++) { 4843 if (gfx_v8_0_rlc_is_idle(handle)) 4844 return 0; 4845 4846 udelay(1); 4847 } 4848 return -ETIMEDOUT; 4849 } 4850 4851 static int gfx_v8_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 4852 { 4853 unsigned int i; 4854 struct amdgpu_device *adev = ip_block->adev; 4855 4856 for (i = 0; i < adev->usec_timeout; i++) { 4857 if (gfx_v8_0_is_idle(ip_block)) 4858 return 0; 4859 4860 udelay(1); 4861 } 4862 return -ETIMEDOUT; 4863 } 4864 4865 static int gfx_v8_0_hw_fini(struct amdgpu_ip_block *ip_block) 4866 { 4867 struct amdgpu_device *adev = ip_block->adev; 4868 4869 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4870 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4871 4872 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 4873 4874 amdgpu_irq_put(adev, &adev->gfx.sq_irq, 0); 4875 4876 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4877 gfx_v8_0_kcq_disable(adev); 4878 4879 if (amdgpu_sriov_vf(adev)) { 4880 pr_debug("For SRIOV client, shouldn't do anything.\n"); 4881 return 0; 4882 } 4883 4884 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4885 if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_idle(ip_block)) 4886 pr_err("cp is busy, skip halt cp\n"); 4887 if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_rlc_idle(adev)) 4888 pr_err("rlc is busy\n"); 4889 gfx_v8_0_cp_enable(adev, false); 4890 adev->gfx.rlc.funcs->stop(adev); 4891 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4892 4893 return 0; 4894 } 4895 4896 static int gfx_v8_0_suspend(struct amdgpu_ip_block *ip_block) 4897 { 4898 return gfx_v8_0_hw_fini(ip_block); 4899 } 4900 4901 static int gfx_v8_0_resume(struct amdgpu_ip_block *ip_block) 4902 { 4903 return gfx_v8_0_hw_init(ip_block); 4904 } 4905 4906 static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) 4907 { 4908 struct amdgpu_device *adev = ip_block->adev; 4909 u32 grbm_soft_reset = 0, srbm_soft_reset = 0; 4910 u32 tmp; 4911 int i; 4912 int r; 4913 4914 grbm_soft_reset = 4915 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1) | 4916 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_GFX, 1) | 4917 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CP, 1) | 4918 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPF, 1) | 4919 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPC, 1) | 4920 REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPG, 1); 4921 4922 srbm_soft_reset = 4923 REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1) | 4924 REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_SEM, 1); 4925 4926 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4927 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4928 4929 mutex_lock(&adev->srbm_mutex); 4930 vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4931 gfx_v8_0_deactivate_hqd(adev, 2); 4932 vi_srbm_select(adev, 0, 0, 0, 0); 4933 mutex_unlock(&adev->srbm_mutex); 4934 4935 udelay(50); 4936 } 4937 4938 ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE); 4939 ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_UNGATE); 4940 ip_block->version->funcs->suspend(ip_block); 4941 4942 if (grbm_soft_reset || srbm_soft_reset) { 4943 tmp = RREG32(mmGMCON_DEBUG); 4944 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1); 4945 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1); 4946 WREG32(mmGMCON_DEBUG, tmp); 4947 4948 udelay(100); 4949 } 4950 4951 if (grbm_soft_reset) { 4952 tmp = RREG32(mmGRBM_SOFT_RESET); 4953 tmp |= grbm_soft_reset; 4954 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4955 WREG32(mmGRBM_SOFT_RESET, tmp); 4956 tmp = RREG32(mmGRBM_SOFT_RESET); 4957 4958 udelay(100); 4959 4960 tmp &= ~grbm_soft_reset; 4961 WREG32(mmGRBM_SOFT_RESET, tmp); 4962 tmp = RREG32(mmGRBM_SOFT_RESET); 4963 4964 udelay(100); 4965 } 4966 4967 if (srbm_soft_reset) { 4968 tmp = RREG32(mmSRBM_SOFT_RESET); 4969 tmp |= srbm_soft_reset; 4970 dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); 4971 WREG32(mmSRBM_SOFT_RESET, tmp); 4972 tmp = RREG32(mmSRBM_SOFT_RESET); 4973 4974 udelay(100); 4975 4976 tmp &= ~srbm_soft_reset; 4977 WREG32(mmSRBM_SOFT_RESET, tmp); 4978 tmp = RREG32(mmSRBM_SOFT_RESET); 4979 4980 udelay(100); 4981 } 4982 4983 if (grbm_soft_reset || srbm_soft_reset) { 4984 tmp = RREG32(mmGMCON_DEBUG); 4985 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 0); 4986 tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 0); 4987 WREG32(mmGMCON_DEBUG, tmp); 4988 } 4989 4990 /* Wait a little for things to settle down */ 4991 udelay(100); 4992 4993 r = ip_block->version->funcs->resume(ip_block); 4994 r |= ip_block->version->funcs->late_init(ip_block); 4995 if (r) 4996 return r; 4997 4998 ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_GATE); 4999 ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_GATE); 5000 5001 return 0; 5002 } 5003 5004 /** 5005 * gfx_v8_0_get_gpu_clock_counter - return GPU clock counter snapshot 5006 * 5007 * @adev: amdgpu_device pointer 5008 * 5009 * Fetches a GPU clock counter snapshot. 5010 * Returns the 64 bit clock counter snapshot. 5011 */ 5012 static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5013 { 5014 uint64_t clock; 5015 5016 mutex_lock(&adev->gfx.gpu_clock_mutex); 5017 WREG32(mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 5018 clock = (uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_LSB) | 5019 ((uint64_t)RREG32(mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 5020 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5021 return clock; 5022 } 5023 5024 static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5025 uint32_t vmid, 5026 uint32_t gds_base, uint32_t gds_size, 5027 uint32_t gws_base, uint32_t gws_size, 5028 uint32_t oa_base, uint32_t oa_size) 5029 { 5030 /* GDS Base */ 5031 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5032 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5033 WRITE_DATA_DST_SEL(0))); 5034 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_base); 5035 amdgpu_ring_write(ring, 0); 5036 amdgpu_ring_write(ring, gds_base); 5037 5038 /* GDS Size */ 5039 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5040 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5041 WRITE_DATA_DST_SEL(0))); 5042 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].mem_size); 5043 amdgpu_ring_write(ring, 0); 5044 amdgpu_ring_write(ring, gds_size); 5045 5046 /* GWS */ 5047 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5048 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5049 WRITE_DATA_DST_SEL(0))); 5050 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].gws); 5051 amdgpu_ring_write(ring, 0); 5052 amdgpu_ring_write(ring, gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5053 5054 /* OA */ 5055 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5056 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5057 WRITE_DATA_DST_SEL(0))); 5058 amdgpu_ring_write(ring, amdgpu_gds_reg_offset[vmid].oa); 5059 amdgpu_ring_write(ring, 0); 5060 amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base)); 5061 } 5062 5063 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 5064 { 5065 WREG32(mmSQ_IND_INDEX, 5066 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5067 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5068 (address << SQ_IND_INDEX__INDEX__SHIFT) | 5069 (SQ_IND_INDEX__FORCE_READ_MASK)); 5070 return RREG32(mmSQ_IND_DATA); 5071 } 5072 5073 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 5074 uint32_t wave, uint32_t thread, 5075 uint32_t regno, uint32_t num, uint32_t *out) 5076 { 5077 WREG32(mmSQ_IND_INDEX, 5078 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 5079 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 5080 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 5081 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 5082 (SQ_IND_INDEX__FORCE_READ_MASK) | 5083 (SQ_IND_INDEX__AUTO_INCR_MASK)); 5084 while (num--) 5085 *(out++) = RREG32(mmSQ_IND_DATA); 5086 } 5087 5088 static void gfx_v8_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 5089 { 5090 /* type 0 wave data */ 5091 dst[(*no_fields)++] = 0; 5092 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 5093 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 5094 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 5095 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 5096 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 5097 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 5098 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 5099 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 5100 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 5101 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 5102 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 5103 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 5104 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_LO); 5105 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TBA_HI); 5106 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_LO); 5107 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TMA_HI); 5108 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 5109 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 5110 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 5111 } 5112 5113 static void gfx_v8_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 5114 uint32_t wave, uint32_t start, 5115 uint32_t size, uint32_t *dst) 5116 { 5117 wave_read_regs( 5118 adev, simd, wave, 0, 5119 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 5120 } 5121 5122 /** 5123 * gfx_v8_0_get_hdp_flush_mask - get the reference and mask for HDP flush 5124 * 5125 * @ring: amdgpu_ring structure holding ring information 5126 * @ref_and_mask: pointer to store the reference and mask 5127 * @reg_mem_engine: pointer to store the register memory engine 5128 * 5129 * Calculates the reference and mask for HDP flush based on the ring type and me. 5130 */ 5131 static void gfx_v8_0_get_hdp_flush_mask(struct amdgpu_ring *ring, 5132 uint32_t *ref_and_mask, uint32_t *reg_mem_engine) 5133 { 5134 if (!ring || !ref_and_mask || !reg_mem_engine) { 5135 DRM_INFO("%s:invalid params\n", __func__); 5136 return; 5137 } 5138 5139 if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) || 5140 (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)) { 5141 switch (ring->me) { 5142 case 1: 5143 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP2_MASK << ring->pipe; 5144 break; 5145 case 2: 5146 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP6_MASK << ring->pipe; 5147 break; 5148 default: 5149 return; 5150 } 5151 *reg_mem_engine = 0; 5152 } else { 5153 *ref_and_mask = GPU_HDP_FLUSH_DONE__CP0_MASK; 5154 *reg_mem_engine = WAIT_REG_MEM_ENGINE(1); /* pfp */ 5155 } 5156 } 5157 5158 static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = { 5159 .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter, 5160 .select_se_sh = &gfx_v8_0_select_se_sh, 5161 .read_wave_data = &gfx_v8_0_read_wave_data, 5162 .read_wave_sgprs = &gfx_v8_0_read_wave_sgprs, 5163 .select_me_pipe_q = &gfx_v8_0_select_me_pipe_q, 5164 .get_hdp_flush_mask = &gfx_v8_0_get_hdp_flush_mask, 5165 }; 5166 5167 static int gfx_v8_0_early_init(struct amdgpu_ip_block *ip_block) 5168 { 5169 struct amdgpu_device *adev = ip_block->adev; 5170 5171 adev->gfx.xcc_mask = 1; 5172 adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS; 5173 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5174 AMDGPU_MAX_COMPUTE_RINGS); 5175 adev->gfx.funcs = &gfx_v8_0_gfx_funcs; 5176 gfx_v8_0_set_ring_funcs(adev); 5177 gfx_v8_0_set_irq_funcs(adev); 5178 gfx_v8_0_set_gds_init(adev); 5179 gfx_v8_0_set_rlc_funcs(adev); 5180 5181 return 0; 5182 } 5183 5184 static int gfx_v8_0_late_init(struct amdgpu_ip_block *ip_block) 5185 { 5186 struct amdgpu_device *adev = ip_block->adev; 5187 int r; 5188 5189 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5190 if (r) 5191 return r; 5192 5193 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5194 if (r) 5195 return r; 5196 5197 /* requires IBs so do in late init after IB pool is initialized */ 5198 r = gfx_v8_0_do_edc_gpr_workarounds(adev); 5199 if (r) 5200 return r; 5201 5202 r = amdgpu_irq_get(adev, &adev->gfx.cp_ecc_error_irq, 0); 5203 if (r) { 5204 DRM_ERROR("amdgpu_irq_get() failed to get IRQ for EDC, r: %d.\n", r); 5205 return r; 5206 } 5207 5208 r = amdgpu_irq_get(adev, &adev->gfx.sq_irq, 0); 5209 if (r) { 5210 DRM_ERROR( 5211 "amdgpu_irq_get() failed to get IRQ for SQ, r: %d.\n", 5212 r); 5213 return r; 5214 } 5215 5216 return 0; 5217 } 5218 5219 static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 5220 bool enable) 5221 { 5222 if ((adev->asic_type == CHIP_POLARIS11) || 5223 (adev->asic_type == CHIP_POLARIS12) || 5224 (adev->asic_type == CHIP_VEGAM)) 5225 /* Send msg to SMU via Powerplay */ 5226 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, enable, 0); 5227 5228 WREG32_FIELD(RLC_PG_CNTL, STATIC_PER_CU_PG_ENABLE, enable ? 1 : 0); 5229 } 5230 5231 static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 5232 bool enable) 5233 { 5234 WREG32_FIELD(RLC_PG_CNTL, DYN_PER_CU_PG_ENABLE, enable ? 1 : 0); 5235 } 5236 5237 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev, 5238 bool enable) 5239 { 5240 WREG32_FIELD(RLC_PG_CNTL, QUICK_PG_ENABLE, enable ? 1 : 0); 5241 } 5242 5243 static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 5244 bool enable) 5245 { 5246 WREG32_FIELD(RLC_PG_CNTL, GFX_POWER_GATING_ENABLE, enable ? 1 : 0); 5247 } 5248 5249 static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev, 5250 bool enable) 5251 { 5252 WREG32_FIELD(RLC_PG_CNTL, GFX_PIPELINE_PG_ENABLE, enable ? 1 : 0); 5253 5254 /* Read any GFX register to wake up GFX. */ 5255 if (!enable) 5256 RREG32(mmDB_RENDER_CONTROL); 5257 } 5258 5259 static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev, 5260 bool enable) 5261 { 5262 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 5263 cz_enable_gfx_cg_power_gating(adev, true); 5264 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 5265 cz_enable_gfx_pipeline_power_gating(adev, true); 5266 } else { 5267 cz_enable_gfx_cg_power_gating(adev, false); 5268 cz_enable_gfx_pipeline_power_gating(adev, false); 5269 } 5270 } 5271 5272 static int gfx_v8_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5273 enum amd_powergating_state state) 5274 { 5275 struct amdgpu_device *adev = ip_block->adev; 5276 bool enable = (state == AMD_PG_STATE_GATE); 5277 5278 if (amdgpu_sriov_vf(adev)) 5279 return 0; 5280 5281 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5282 AMD_PG_SUPPORT_RLC_SMU_HS | 5283 AMD_PG_SUPPORT_CP | 5284 AMD_PG_SUPPORT_GFX_DMG)) 5285 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5286 switch (adev->asic_type) { 5287 case CHIP_CARRIZO: 5288 case CHIP_STONEY: 5289 5290 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5291 cz_enable_sck_slow_down_on_power_up(adev, true); 5292 cz_enable_sck_slow_down_on_power_down(adev, true); 5293 } else { 5294 cz_enable_sck_slow_down_on_power_up(adev, false); 5295 cz_enable_sck_slow_down_on_power_down(adev, false); 5296 } 5297 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5298 cz_enable_cp_power_gating(adev, true); 5299 else 5300 cz_enable_cp_power_gating(adev, false); 5301 5302 cz_update_gfx_cg_power_gating(adev, enable); 5303 5304 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5305 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5306 else 5307 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5308 5309 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5310 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5311 else 5312 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5313 break; 5314 case CHIP_POLARIS11: 5315 case CHIP_POLARIS12: 5316 case CHIP_VEGAM: 5317 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 5318 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true); 5319 else 5320 gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false); 5321 5322 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5323 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5324 else 5325 gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5326 5327 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable) 5328 polaris11_enable_gfx_quick_mg_power_gating(adev, true); 5329 else 5330 polaris11_enable_gfx_quick_mg_power_gating(adev, false); 5331 break; 5332 default: 5333 break; 5334 } 5335 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_SMG | 5336 AMD_PG_SUPPORT_RLC_SMU_HS | 5337 AMD_PG_SUPPORT_CP | 5338 AMD_PG_SUPPORT_GFX_DMG)) 5339 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5340 return 0; 5341 } 5342 5343 static void gfx_v8_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5344 { 5345 struct amdgpu_device *adev = ip_block->adev; 5346 int data; 5347 5348 if (amdgpu_sriov_vf(adev)) 5349 *flags = 0; 5350 5351 /* AMD_CG_SUPPORT_GFX_MGCG */ 5352 data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5353 if (!(data & RLC_CGTT_MGCG_OVERRIDE__CPF_MASK)) 5354 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5355 5356 /* AMD_CG_SUPPORT_GFX_CGLG */ 5357 data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5358 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5359 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5360 5361 /* AMD_CG_SUPPORT_GFX_CGLS */ 5362 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5363 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5364 5365 /* AMD_CG_SUPPORT_GFX_CGTS */ 5366 data = RREG32(mmCGTS_SM_CTRL_REG); 5367 if (!(data & CGTS_SM_CTRL_REG__OVERRIDE_MASK)) 5368 *flags |= AMD_CG_SUPPORT_GFX_CGTS; 5369 5370 /* AMD_CG_SUPPORT_GFX_CGTS_LS */ 5371 if (!(data & CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK)) 5372 *flags |= AMD_CG_SUPPORT_GFX_CGTS_LS; 5373 5374 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5375 data = RREG32(mmRLC_MEM_SLP_CNTL); 5376 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5377 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5378 5379 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5380 data = RREG32(mmCP_MEM_SLP_CNTL); 5381 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5382 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5383 } 5384 5385 static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev, 5386 uint32_t reg_addr, uint32_t cmd) 5387 { 5388 uint32_t data; 5389 5390 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 5391 5392 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff); 5393 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff); 5394 5395 data = RREG32(mmRLC_SERDES_WR_CTRL); 5396 if (adev->asic_type == CHIP_STONEY) 5397 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5398 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5399 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5400 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5401 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5402 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5403 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5404 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5405 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5406 else 5407 data &= ~(RLC_SERDES_WR_CTRL__WRITE_COMMAND_MASK | 5408 RLC_SERDES_WR_CTRL__READ_COMMAND_MASK | 5409 RLC_SERDES_WR_CTRL__P1_SELECT_MASK | 5410 RLC_SERDES_WR_CTRL__P2_SELECT_MASK | 5411 RLC_SERDES_WR_CTRL__RDDATA_RESET_MASK | 5412 RLC_SERDES_WR_CTRL__POWER_DOWN_MASK | 5413 RLC_SERDES_WR_CTRL__POWER_UP_MASK | 5414 RLC_SERDES_WR_CTRL__SHORT_FORMAT_MASK | 5415 RLC_SERDES_WR_CTRL__BPM_DATA_MASK | 5416 RLC_SERDES_WR_CTRL__REG_ADDR_MASK | 5417 RLC_SERDES_WR_CTRL__SRBM_OVERRIDE_MASK); 5418 data |= (RLC_SERDES_WR_CTRL__RSVD_BPM_ADDR_MASK | 5419 (cmd << RLC_SERDES_WR_CTRL__BPM_DATA__SHIFT) | 5420 (reg_addr << RLC_SERDES_WR_CTRL__REG_ADDR__SHIFT) | 5421 (0xff << RLC_SERDES_WR_CTRL__BPM_ADDR__SHIFT)); 5422 5423 WREG32(mmRLC_SERDES_WR_CTRL, data); 5424 } 5425 5426 #define MSG_ENTER_RLC_SAFE_MODE 1 5427 #define MSG_EXIT_RLC_SAFE_MODE 0 5428 #define RLC_GPR_REG2__REQ_MASK 0x00000001 5429 #define RLC_GPR_REG2__REQ__SHIFT 0 5430 #define RLC_GPR_REG2__MESSAGE__SHIFT 0x00000001 5431 #define RLC_GPR_REG2__MESSAGE_MASK 0x0000001e 5432 5433 static bool gfx_v8_0_is_rlc_enabled(struct amdgpu_device *adev) 5434 { 5435 uint32_t rlc_setting; 5436 5437 rlc_setting = RREG32(mmRLC_CNTL); 5438 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 5439 return false; 5440 5441 return true; 5442 } 5443 5444 static void gfx_v8_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5445 { 5446 uint32_t data; 5447 unsigned i; 5448 data = RREG32(mmRLC_CNTL); 5449 data |= RLC_SAFE_MODE__CMD_MASK; 5450 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5451 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5452 WREG32(mmRLC_SAFE_MODE, data); 5453 5454 /* wait for RLC_SAFE_MODE */ 5455 for (i = 0; i < adev->usec_timeout; i++) { 5456 if ((RREG32(mmRLC_GPM_STAT) & 5457 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5458 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) == 5459 (RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK | 5460 RLC_GPM_STAT__GFX_POWER_STATUS_MASK)) 5461 break; 5462 udelay(1); 5463 } 5464 for (i = 0; i < adev->usec_timeout; i++) { 5465 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5466 break; 5467 udelay(1); 5468 } 5469 } 5470 5471 static void gfx_v8_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5472 { 5473 uint32_t data; 5474 unsigned i; 5475 5476 data = RREG32(mmRLC_CNTL); 5477 data |= RLC_SAFE_MODE__CMD_MASK; 5478 data &= ~RLC_SAFE_MODE__MESSAGE_MASK; 5479 WREG32(mmRLC_SAFE_MODE, data); 5480 5481 for (i = 0; i < adev->usec_timeout; i++) { 5482 if (!REG_GET_FIELD(RREG32(mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 5483 break; 5484 udelay(1); 5485 } 5486 } 5487 5488 static void gfx_v8_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5489 struct amdgpu_ring *ring, unsigned vmid) 5490 { 5491 u32 data; 5492 5493 amdgpu_gfx_off_ctrl(adev, false); 5494 5495 if (amdgpu_sriov_is_pp_one_vf(adev)) 5496 data = RREG32_NO_KIQ(mmRLC_SPM_VMID); 5497 else 5498 data = RREG32(mmRLC_SPM_VMID); 5499 5500 data &= ~RLC_SPM_VMID__RLC_SPM_VMID_MASK; 5501 data |= (vmid & RLC_SPM_VMID__RLC_SPM_VMID_MASK) << RLC_SPM_VMID__RLC_SPM_VMID__SHIFT; 5502 5503 if (amdgpu_sriov_is_pp_one_vf(adev)) 5504 WREG32_NO_KIQ(mmRLC_SPM_VMID, data); 5505 else 5506 WREG32(mmRLC_SPM_VMID, data); 5507 5508 amdgpu_gfx_off_ctrl(adev, true); 5509 } 5510 5511 static const struct amdgpu_rlc_funcs iceland_rlc_funcs = { 5512 .is_rlc_enabled = gfx_v8_0_is_rlc_enabled, 5513 .set_safe_mode = gfx_v8_0_set_safe_mode, 5514 .unset_safe_mode = gfx_v8_0_unset_safe_mode, 5515 .init = gfx_v8_0_rlc_init, 5516 .get_csb_size = gfx_v8_0_get_csb_size, 5517 .get_csb_buffer = gfx_v8_0_get_csb_buffer, 5518 .get_cp_table_num = gfx_v8_0_cp_jump_table_num, 5519 .resume = gfx_v8_0_rlc_resume, 5520 .stop = gfx_v8_0_rlc_stop, 5521 .reset = gfx_v8_0_rlc_reset, 5522 .start = gfx_v8_0_rlc_start, 5523 .update_spm_vmid = gfx_v8_0_update_spm_vmid 5524 }; 5525 5526 static void gfx_v8_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5527 bool enable) 5528 { 5529 uint32_t temp, data; 5530 5531 /* It is disabled by HW by default */ 5532 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 5533 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5534 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) 5535 /* 1 - RLC memory Light sleep */ 5536 WREG32_FIELD(RLC_MEM_SLP_CNTL, RLC_MEM_LS_EN, 1); 5537 5538 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) 5539 WREG32_FIELD(CP_MEM_SLP_CNTL, CP_MEM_LS_EN, 1); 5540 } 5541 5542 /* 3 - RLC_CGTT_MGCG_OVERRIDE */ 5543 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5544 if (adev->flags & AMD_IS_APU) 5545 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5546 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5547 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK); 5548 else 5549 data &= ~(RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5550 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5551 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5552 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5553 5554 if (temp != data) 5555 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5556 5557 /* 4 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5558 gfx_v8_0_wait_for_rlc_serdes(adev); 5559 5560 /* 5 - clear mgcg override */ 5561 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5562 5563 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS) { 5564 /* 6 - Enable CGTS(Tree Shade) MGCG /MGLS */ 5565 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5566 data &= ~(CGTS_SM_CTRL_REG__SM_MODE_MASK); 5567 data |= (0x2 << CGTS_SM_CTRL_REG__SM_MODE__SHIFT); 5568 data |= CGTS_SM_CTRL_REG__SM_MODE_ENABLE_MASK; 5569 data &= ~CGTS_SM_CTRL_REG__OVERRIDE_MASK; 5570 if ((adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) && 5571 (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGTS_LS)) 5572 data &= ~CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK; 5573 data |= CGTS_SM_CTRL_REG__ON_MONITOR_ADD_EN_MASK; 5574 data |= (0x96 << CGTS_SM_CTRL_REG__ON_MONITOR_ADD__SHIFT); 5575 if (temp != data) 5576 WREG32(mmCGTS_SM_CTRL_REG, data); 5577 } 5578 udelay(50); 5579 5580 /* 7 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5581 gfx_v8_0_wait_for_rlc_serdes(adev); 5582 } else { 5583 /* 1 - MGCG_OVERRIDE[0] for CP and MGCG_OVERRIDE[1] for RLC */ 5584 temp = data = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5585 data |= (RLC_CGTT_MGCG_OVERRIDE__CPF_MASK | 5586 RLC_CGTT_MGCG_OVERRIDE__RLC_MASK | 5587 RLC_CGTT_MGCG_OVERRIDE__MGCG_MASK | 5588 RLC_CGTT_MGCG_OVERRIDE__GRBM_MASK); 5589 if (temp != data) 5590 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data); 5591 5592 /* 2 - disable MGLS in RLC */ 5593 data = RREG32(mmRLC_MEM_SLP_CNTL); 5594 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 5595 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 5596 WREG32(mmRLC_MEM_SLP_CNTL, data); 5597 } 5598 5599 /* 3 - disable MGLS in CP */ 5600 data = RREG32(mmCP_MEM_SLP_CNTL); 5601 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 5602 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 5603 WREG32(mmCP_MEM_SLP_CNTL, data); 5604 } 5605 5606 /* 4 - Disable CGTS(Tree Shade) MGCG and MGLS */ 5607 temp = data = RREG32(mmCGTS_SM_CTRL_REG); 5608 data |= (CGTS_SM_CTRL_REG__OVERRIDE_MASK | 5609 CGTS_SM_CTRL_REG__LS_OVERRIDE_MASK); 5610 if (temp != data) 5611 WREG32(mmCGTS_SM_CTRL_REG, data); 5612 5613 /* 5 - wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5614 gfx_v8_0_wait_for_rlc_serdes(adev); 5615 5616 /* 6 - set mgcg override */ 5617 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_MGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5618 5619 udelay(50); 5620 5621 /* 7- wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5622 gfx_v8_0_wait_for_rlc_serdes(adev); 5623 } 5624 } 5625 5626 static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5627 bool enable) 5628 { 5629 uint32_t temp, temp1, data, data1; 5630 5631 temp = data = RREG32(mmRLC_CGCG_CGLS_CTRL); 5632 5633 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5634 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5635 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK; 5636 if (temp1 != data1) 5637 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5638 5639 /* : wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5640 gfx_v8_0_wait_for_rlc_serdes(adev); 5641 5642 /* 2 - clear cgcg override */ 5643 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, CLE_BPM_SERDES_CMD); 5644 5645 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5646 gfx_v8_0_wait_for_rlc_serdes(adev); 5647 5648 /* 3 - write cmd to set CGLS */ 5649 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, SET_BPM_SERDES_CMD); 5650 5651 /* 4 - enable cgcg */ 5652 data |= RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5653 5654 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5655 /* enable cgls*/ 5656 data |= RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5657 5658 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5659 data1 &= ~RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK; 5660 5661 if (temp1 != data1) 5662 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5663 } else { 5664 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5665 } 5666 5667 if (temp != data) 5668 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5669 5670 /* 5 enable cntx_empty_int_enable/cntx_busy_int_enable/ 5671 * Cmp_busy/GFX_Idle interrupts 5672 */ 5673 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5674 } else { 5675 /* disable cntx_empty_int_enable & GFX Idle interrupt */ 5676 gfx_v8_0_enable_gui_idle_interrupt(adev, false); 5677 5678 /* TEST CGCG */ 5679 temp1 = data1 = RREG32(mmRLC_CGTT_MGCG_OVERRIDE); 5680 data1 |= (RLC_CGTT_MGCG_OVERRIDE__CGCG_MASK | 5681 RLC_CGTT_MGCG_OVERRIDE__CGLS_MASK); 5682 if (temp1 != data1) 5683 WREG32(mmRLC_CGTT_MGCG_OVERRIDE, data1); 5684 5685 /* read gfx register to wake up cgcg */ 5686 RREG32(mmCB_CGTT_SCLK_CTRL); 5687 RREG32(mmCB_CGTT_SCLK_CTRL); 5688 RREG32(mmCB_CGTT_SCLK_CTRL); 5689 RREG32(mmCB_CGTT_SCLK_CTRL); 5690 5691 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5692 gfx_v8_0_wait_for_rlc_serdes(adev); 5693 5694 /* write cmd to Set CGCG Override */ 5695 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGCG_OVERRIDE, SET_BPM_SERDES_CMD); 5696 5697 /* wait for RLC_SERDES_CU_MASTER & RLC_SERDES_NONCU_MASTER idle */ 5698 gfx_v8_0_wait_for_rlc_serdes(adev); 5699 5700 /* write cmd to Clear CGLS */ 5701 gfx_v8_0_send_serdes_cmd(adev, BPM_REG_CGLS_EN, CLE_BPM_SERDES_CMD); 5702 5703 /* disable cgcg, cgls should be disabled too. */ 5704 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | 5705 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5706 if (temp != data) 5707 WREG32(mmRLC_CGCG_CGLS_CTRL, data); 5708 /* enable interrupts again for PG */ 5709 gfx_v8_0_enable_gui_idle_interrupt(adev, true); 5710 } 5711 5712 gfx_v8_0_wait_for_rlc_serdes(adev); 5713 } 5714 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5715 bool enable) 5716 { 5717 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5718 5719 if (enable) { 5720 /* CGCG/CGLS should be enabled after MGCG/MGLS/TS(CG/LS) 5721 * === MGCG + MGLS + TS(CG/LS) === 5722 */ 5723 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5724 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5725 } else { 5726 /* CGCG/CGLS should be disabled before MGCG/MGLS/TS(CG/LS) 5727 * === CGCG + CGLS === 5728 */ 5729 gfx_v8_0_update_coarse_grain_clock_gating(adev, enable); 5730 gfx_v8_0_update_medium_grain_clock_gating(adev, enable); 5731 } 5732 5733 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5734 return 0; 5735 } 5736 5737 static int gfx_v8_0_tonga_update_gfx_clock_gating(struct amdgpu_device *adev, 5738 enum amd_clockgating_state state) 5739 { 5740 uint32_t msg_id, pp_state = 0; 5741 uint32_t pp_support_state = 0; 5742 5743 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5744 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5745 pp_support_state = PP_STATE_SUPPORT_LS; 5746 pp_state = PP_STATE_LS; 5747 } 5748 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5749 pp_support_state |= PP_STATE_SUPPORT_CG; 5750 pp_state |= PP_STATE_CG; 5751 } 5752 if (state == AMD_CG_STATE_UNGATE) 5753 pp_state = 0; 5754 5755 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5756 PP_BLOCK_GFX_CG, 5757 pp_support_state, 5758 pp_state); 5759 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5760 } 5761 5762 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5763 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5764 pp_support_state = PP_STATE_SUPPORT_LS; 5765 pp_state = PP_STATE_LS; 5766 } 5767 5768 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5769 pp_support_state |= PP_STATE_SUPPORT_CG; 5770 pp_state |= PP_STATE_CG; 5771 } 5772 5773 if (state == AMD_CG_STATE_UNGATE) 5774 pp_state = 0; 5775 5776 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5777 PP_BLOCK_GFX_MG, 5778 pp_support_state, 5779 pp_state); 5780 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5781 } 5782 5783 return 0; 5784 } 5785 5786 static int gfx_v8_0_polaris_update_gfx_clock_gating(struct amdgpu_device *adev, 5787 enum amd_clockgating_state state) 5788 { 5789 5790 uint32_t msg_id, pp_state = 0; 5791 uint32_t pp_support_state = 0; 5792 5793 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_CGCG | AMD_CG_SUPPORT_GFX_CGLS)) { 5794 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5795 pp_support_state = PP_STATE_SUPPORT_LS; 5796 pp_state = PP_STATE_LS; 5797 } 5798 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5799 pp_support_state |= PP_STATE_SUPPORT_CG; 5800 pp_state |= PP_STATE_CG; 5801 } 5802 if (state == AMD_CG_STATE_UNGATE) 5803 pp_state = 0; 5804 5805 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5806 PP_BLOCK_GFX_CG, 5807 pp_support_state, 5808 pp_state); 5809 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5810 } 5811 5812 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_3D_CGCG | AMD_CG_SUPPORT_GFX_3D_CGLS)) { 5813 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5814 pp_support_state = PP_STATE_SUPPORT_LS; 5815 pp_state = PP_STATE_LS; 5816 } 5817 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5818 pp_support_state |= PP_STATE_SUPPORT_CG; 5819 pp_state |= PP_STATE_CG; 5820 } 5821 if (state == AMD_CG_STATE_UNGATE) 5822 pp_state = 0; 5823 5824 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5825 PP_BLOCK_GFX_3D, 5826 pp_support_state, 5827 pp_state); 5828 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5829 } 5830 5831 if (adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS)) { 5832 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5833 pp_support_state = PP_STATE_SUPPORT_LS; 5834 pp_state = PP_STATE_LS; 5835 } 5836 5837 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5838 pp_support_state |= PP_STATE_SUPPORT_CG; 5839 pp_state |= PP_STATE_CG; 5840 } 5841 5842 if (state == AMD_CG_STATE_UNGATE) 5843 pp_state = 0; 5844 5845 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5846 PP_BLOCK_GFX_MG, 5847 pp_support_state, 5848 pp_state); 5849 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5850 } 5851 5852 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 5853 pp_support_state = PP_STATE_SUPPORT_LS; 5854 5855 if (state == AMD_CG_STATE_UNGATE) 5856 pp_state = 0; 5857 else 5858 pp_state = PP_STATE_LS; 5859 5860 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5861 PP_BLOCK_GFX_RLC, 5862 pp_support_state, 5863 pp_state); 5864 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5865 } 5866 5867 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 5868 pp_support_state = PP_STATE_SUPPORT_LS; 5869 5870 if (state == AMD_CG_STATE_UNGATE) 5871 pp_state = 0; 5872 else 5873 pp_state = PP_STATE_LS; 5874 msg_id = PP_CG_MSG_ID(PP_GROUP_GFX, 5875 PP_BLOCK_GFX_CP, 5876 pp_support_state, 5877 pp_state); 5878 amdgpu_dpm_set_clockgating_by_smu(adev, msg_id); 5879 } 5880 5881 return 0; 5882 } 5883 5884 static int gfx_v8_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5885 enum amd_clockgating_state state) 5886 { 5887 struct amdgpu_device *adev = ip_block->adev; 5888 5889 if (amdgpu_sriov_vf(adev)) 5890 return 0; 5891 5892 switch (adev->asic_type) { 5893 case CHIP_FIJI: 5894 case CHIP_CARRIZO: 5895 case CHIP_STONEY: 5896 gfx_v8_0_update_gfx_clock_gating(adev, 5897 state == AMD_CG_STATE_GATE); 5898 break; 5899 case CHIP_TONGA: 5900 gfx_v8_0_tonga_update_gfx_clock_gating(adev, state); 5901 break; 5902 case CHIP_POLARIS10: 5903 case CHIP_POLARIS11: 5904 case CHIP_POLARIS12: 5905 case CHIP_VEGAM: 5906 gfx_v8_0_polaris_update_gfx_clock_gating(adev, state); 5907 break; 5908 default: 5909 break; 5910 } 5911 return 0; 5912 } 5913 5914 static u64 gfx_v8_0_ring_get_rptr(struct amdgpu_ring *ring) 5915 { 5916 return *ring->rptr_cpu_addr; 5917 } 5918 5919 static u64 gfx_v8_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5920 { 5921 struct amdgpu_device *adev = ring->adev; 5922 5923 if (ring->use_doorbell) 5924 /* XXX check if swapping is necessary on BE */ 5925 return *ring->wptr_cpu_addr; 5926 else 5927 return RREG32(mmCP_RB0_WPTR); 5928 } 5929 5930 static void gfx_v8_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5931 { 5932 struct amdgpu_device *adev = ring->adev; 5933 5934 if (ring->use_doorbell) { 5935 /* XXX check if swapping is necessary on BE */ 5936 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 5937 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 5938 } else { 5939 WREG32(mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5940 (void)RREG32(mmCP_RB0_WPTR); 5941 } 5942 } 5943 5944 static void gfx_v8_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5945 { 5946 u32 ref_and_mask, reg_mem_engine; 5947 struct amdgpu_device *adev = ring->adev; 5948 5949 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5950 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5951 return; 5952 } 5953 5954 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 5955 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 5956 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */ 5957 WAIT_REG_MEM_FUNCTION(3) | /* == */ 5958 reg_mem_engine)); 5959 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_REQ); 5960 amdgpu_ring_write(ring, mmGPU_HDP_FLUSH_DONE); 5961 amdgpu_ring_write(ring, ref_and_mask); 5962 amdgpu_ring_write(ring, ref_and_mask); 5963 amdgpu_ring_write(ring, 0x20); /* poll interval */ 5964 } 5965 5966 static void gfx_v8_0_ring_emit_vgt_flush(struct amdgpu_ring *ring) 5967 { 5968 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 5969 amdgpu_ring_write(ring, EVENT_TYPE(VS_PARTIAL_FLUSH) | 5970 EVENT_INDEX(4)); 5971 5972 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 5973 amdgpu_ring_write(ring, EVENT_TYPE(VGT_FLUSH) | 5974 EVENT_INDEX(0)); 5975 } 5976 5977 static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5978 struct amdgpu_job *job, 5979 struct amdgpu_ib *ib, 5980 uint32_t flags) 5981 { 5982 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5983 u32 header, control = 0; 5984 5985 if (ib->flags & AMDGPU_IB_FLAG_CE) 5986 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5987 else 5988 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5989 5990 control |= ib->length_dw | (vmid << 24); 5991 5992 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5993 control |= INDIRECT_BUFFER_PRE_ENB(1); 5994 5995 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5996 gfx_v8_0_ring_emit_de_meta(ring); 5997 } 5998 5999 amdgpu_ring_write(ring, header); 6000 amdgpu_ring_write(ring, 6001 #ifdef __BIG_ENDIAN 6002 (2 << 0) | 6003 #endif 6004 (ib->gpu_addr & 0xFFFFFFFC)); 6005 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6006 amdgpu_ring_write(ring, control); 6007 } 6008 6009 static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 6010 struct amdgpu_job *job, 6011 struct amdgpu_ib *ib, 6012 uint32_t flags) 6013 { 6014 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6015 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 6016 6017 /* Currently, there is a high possibility to get wave ID mismatch 6018 * between ME and GDS, leading to a hw deadlock, because ME generates 6019 * different wave IDs than the GDS expects. This situation happens 6020 * randomly when at least 5 compute pipes use GDS ordered append. 6021 * The wave IDs generated by ME are also wrong after suspend/resume. 6022 * Those are probably bugs somewhere else in the kernel driver. 6023 * 6024 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6025 * GDS to 0 for this ring (me/pipe). 6026 */ 6027 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6028 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6029 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID - PACKET3_SET_CONFIG_REG_START); 6030 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6031 } 6032 6033 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6034 amdgpu_ring_write(ring, 6035 #ifdef __BIG_ENDIAN 6036 (2 << 0) | 6037 #endif 6038 (ib->gpu_addr & 0xFFFFFFFC)); 6039 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF); 6040 amdgpu_ring_write(ring, control); 6041 } 6042 6043 static void gfx_v8_0_ring_emit_fence_gfx(struct amdgpu_ring *ring, u64 addr, 6044 u64 seq, unsigned flags) 6045 { 6046 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6047 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6048 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC; 6049 6050 /* Workaround for cache flush problems. First send a dummy EOP 6051 * event down the pipe with seq one below. 6052 */ 6053 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6054 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6055 EOP_TC_ACTION_EN | 6056 EOP_TC_WB_ACTION_EN | 6057 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6058 EVENT_INDEX(5))); 6059 amdgpu_ring_write(ring, addr & 0xfffffffc); 6060 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6061 DATA_SEL(1) | INT_SEL(0)); 6062 amdgpu_ring_write(ring, lower_32_bits(seq - 1)); 6063 amdgpu_ring_write(ring, upper_32_bits(seq - 1)); 6064 6065 /* Then send the real EOP event down the pipe: 6066 * EVENT_WRITE_EOP - flush caches, send int */ 6067 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4)); 6068 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6069 EOP_TC_ACTION_EN | 6070 EOP_TC_WB_ACTION_EN | 6071 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6072 EVENT_INDEX(5) | 6073 (exec ? EOP_EXEC : 0))); 6074 amdgpu_ring_write(ring, addr & 0xfffffffc); 6075 amdgpu_ring_write(ring, (upper_32_bits(addr) & 0xffff) | 6076 DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6077 amdgpu_ring_write(ring, lower_32_bits(seq)); 6078 amdgpu_ring_write(ring, upper_32_bits(seq)); 6079 6080 } 6081 6082 static void gfx_v8_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6083 { 6084 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6085 uint32_t seq = ring->fence_drv.sync_seq; 6086 uint64_t addr = ring->fence_drv.gpu_addr; 6087 6088 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6089 amdgpu_ring_write(ring, (WAIT_REG_MEM_MEM_SPACE(1) | /* memory */ 6090 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 6091 WAIT_REG_MEM_ENGINE(usepfp))); /* pfp or me */ 6092 amdgpu_ring_write(ring, addr & 0xfffffffc); 6093 amdgpu_ring_write(ring, upper_32_bits(addr) & 0xffffffff); 6094 amdgpu_ring_write(ring, seq); 6095 amdgpu_ring_write(ring, 0xffffffff); 6096 amdgpu_ring_write(ring, 4); /* poll interval */ 6097 } 6098 6099 static void gfx_v8_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6100 unsigned vmid, uint64_t pd_addr) 6101 { 6102 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6103 6104 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6105 6106 /* wait for the invalidate to complete */ 6107 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 6108 amdgpu_ring_write(ring, (WAIT_REG_MEM_OPERATION(0) | /* wait */ 6109 WAIT_REG_MEM_FUNCTION(0) | /* always */ 6110 WAIT_REG_MEM_ENGINE(0))); /* me */ 6111 amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST); 6112 amdgpu_ring_write(ring, 0); 6113 amdgpu_ring_write(ring, 0); /* ref */ 6114 amdgpu_ring_write(ring, 0); /* mask */ 6115 amdgpu_ring_write(ring, 0x20); /* poll interval */ 6116 6117 /* compute doesn't have PFP */ 6118 if (usepfp) { 6119 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6120 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6121 amdgpu_ring_write(ring, 0x0); 6122 } 6123 } 6124 6125 static u64 gfx_v8_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 6126 { 6127 return *ring->wptr_cpu_addr; 6128 } 6129 6130 static void gfx_v8_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 6131 { 6132 struct amdgpu_device *adev = ring->adev; 6133 6134 /* XXX check if swapping is necessary on BE */ 6135 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 6136 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 6137 } 6138 6139 static void gfx_v8_0_ring_emit_fence_compute(struct amdgpu_ring *ring, 6140 u64 addr, u64 seq, 6141 unsigned flags) 6142 { 6143 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6144 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6145 6146 /* RELEASE_MEM - flush caches, send int */ 6147 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5)); 6148 amdgpu_ring_write(ring, (EOP_TCL1_ACTION_EN | 6149 EOP_TC_ACTION_EN | 6150 EOP_TC_WB_ACTION_EN | 6151 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6152 EVENT_INDEX(5))); 6153 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 6154 amdgpu_ring_write(ring, addr & 0xfffffffc); 6155 amdgpu_ring_write(ring, upper_32_bits(addr)); 6156 amdgpu_ring_write(ring, lower_32_bits(seq)); 6157 amdgpu_ring_write(ring, upper_32_bits(seq)); 6158 } 6159 6160 static void gfx_v8_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6161 u64 seq, unsigned int flags) 6162 { 6163 /* write fence seq to the "addr" */ 6164 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6165 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6166 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6167 amdgpu_ring_write(ring, lower_32_bits(addr)); 6168 amdgpu_ring_write(ring, upper_32_bits(addr)); 6169 amdgpu_ring_write(ring, lower_32_bits(seq)); 6170 6171 if (flags & AMDGPU_FENCE_FLAG_INT) { 6172 /* set register to trigger INT */ 6173 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6174 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6175 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6176 amdgpu_ring_write(ring, mmCPC_INT_STATUS); 6177 amdgpu_ring_write(ring, 0); 6178 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6179 } 6180 } 6181 6182 static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring) 6183 { 6184 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 6185 amdgpu_ring_write(ring, 0); 6186 } 6187 6188 static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 6189 { 6190 uint32_t dw2 = 0; 6191 6192 if (amdgpu_sriov_vf(ring->adev)) 6193 gfx_v8_0_ring_emit_ce_meta(ring); 6194 6195 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6196 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6197 gfx_v8_0_ring_emit_vgt_flush(ring); 6198 /* set load_global_config & load_global_uconfig */ 6199 dw2 |= 0x8001; 6200 /* set load_cs_sh_regs */ 6201 dw2 |= 0x01000000; 6202 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6203 dw2 |= 0x10002; 6204 6205 /* set load_ce_ram if preamble presented */ 6206 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 6207 dw2 |= 0x10000000; 6208 } else { 6209 /* still load_ce_ram if this is the first time preamble presented 6210 * although there is no context switch happens. 6211 */ 6212 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 6213 dw2 |= 0x10000000; 6214 } 6215 6216 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6217 amdgpu_ring_write(ring, dw2); 6218 amdgpu_ring_write(ring, 0); 6219 } 6220 6221 static unsigned gfx_v8_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 6222 uint64_t addr) 6223 { 6224 unsigned ret; 6225 6226 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6227 amdgpu_ring_write(ring, lower_32_bits(addr)); 6228 amdgpu_ring_write(ring, upper_32_bits(addr)); 6229 /* discard following DWs if *cond_exec_gpu_addr==0 */ 6230 amdgpu_ring_write(ring, 0); 6231 ret = ring->wptr & ring->buf_mask; 6232 /* patch dummy value later */ 6233 amdgpu_ring_write(ring, 0); 6234 return ret; 6235 } 6236 6237 static void gfx_v8_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6238 uint32_t reg_val_offs) 6239 { 6240 struct amdgpu_device *adev = ring->adev; 6241 6242 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6243 amdgpu_ring_write(ring, 0 | /* src: register*/ 6244 (5 << 8) | /* dst: memory */ 6245 (1 << 20)); /* write confirm */ 6246 amdgpu_ring_write(ring, reg); 6247 amdgpu_ring_write(ring, 0); 6248 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6249 reg_val_offs * 4)); 6250 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6251 reg_val_offs * 4)); 6252 } 6253 6254 static void gfx_v8_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6255 uint32_t val) 6256 { 6257 uint32_t cmd; 6258 6259 switch (ring->funcs->type) { 6260 case AMDGPU_RING_TYPE_GFX: 6261 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6262 break; 6263 case AMDGPU_RING_TYPE_KIQ: 6264 cmd = 1 << 16; /* no inc addr */ 6265 break; 6266 default: 6267 cmd = WR_CONFIRM; 6268 break; 6269 } 6270 6271 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6272 amdgpu_ring_write(ring, cmd); 6273 amdgpu_ring_write(ring, reg); 6274 amdgpu_ring_write(ring, 0); 6275 amdgpu_ring_write(ring, val); 6276 } 6277 6278 static void gfx_v8_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 6279 { 6280 struct amdgpu_device *adev = ring->adev; 6281 uint32_t value = 0; 6282 6283 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 6284 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 6285 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 6286 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 6287 WREG32(mmSQ_CMD, value); 6288 } 6289 6290 static void gfx_v8_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6291 enum amdgpu_interrupt_state state) 6292 { 6293 WREG32_FIELD(CP_INT_CNTL_RING0, TIME_STAMP_INT_ENABLE, 6294 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6295 } 6296 6297 static void gfx_v8_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6298 int me, int pipe, 6299 enum amdgpu_interrupt_state state) 6300 { 6301 u32 mec_int_cntl, mec_int_cntl_reg; 6302 6303 /* 6304 * amdgpu controls only the first MEC. That's why this function only 6305 * handles the setting of interrupts for this specific MEC. All other 6306 * pipes' interrupts are set by amdkfd. 6307 */ 6308 6309 if (me == 1) { 6310 switch (pipe) { 6311 case 0: 6312 mec_int_cntl_reg = mmCP_ME1_PIPE0_INT_CNTL; 6313 break; 6314 case 1: 6315 mec_int_cntl_reg = mmCP_ME1_PIPE1_INT_CNTL; 6316 break; 6317 case 2: 6318 mec_int_cntl_reg = mmCP_ME1_PIPE2_INT_CNTL; 6319 break; 6320 case 3: 6321 mec_int_cntl_reg = mmCP_ME1_PIPE3_INT_CNTL; 6322 break; 6323 default: 6324 DRM_DEBUG("invalid pipe %d\n", pipe); 6325 return; 6326 } 6327 } else { 6328 DRM_DEBUG("invalid me %d\n", me); 6329 return; 6330 } 6331 6332 switch (state) { 6333 case AMDGPU_IRQ_STATE_DISABLE: 6334 mec_int_cntl = RREG32(mec_int_cntl_reg); 6335 mec_int_cntl &= ~CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6336 WREG32(mec_int_cntl_reg, mec_int_cntl); 6337 break; 6338 case AMDGPU_IRQ_STATE_ENABLE: 6339 mec_int_cntl = RREG32(mec_int_cntl_reg); 6340 mec_int_cntl |= CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK; 6341 WREG32(mec_int_cntl_reg, mec_int_cntl); 6342 break; 6343 default: 6344 break; 6345 } 6346 } 6347 6348 static int gfx_v8_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6349 struct amdgpu_irq_src *source, 6350 unsigned type, 6351 enum amdgpu_interrupt_state state) 6352 { 6353 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_REG_INT_ENABLE, 6354 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6355 6356 return 0; 6357 } 6358 6359 static int gfx_v8_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6360 struct amdgpu_irq_src *source, 6361 unsigned type, 6362 enum amdgpu_interrupt_state state) 6363 { 6364 WREG32_FIELD(CP_INT_CNTL_RING0, PRIV_INSTR_INT_ENABLE, 6365 state == AMDGPU_IRQ_STATE_DISABLE ? 0 : 1); 6366 6367 return 0; 6368 } 6369 6370 static int gfx_v8_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6371 struct amdgpu_irq_src *src, 6372 unsigned type, 6373 enum amdgpu_interrupt_state state) 6374 { 6375 switch (type) { 6376 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6377 gfx_v8_0_set_gfx_eop_interrupt_state(adev, state); 6378 break; 6379 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6380 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6381 break; 6382 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6383 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6384 break; 6385 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6386 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6387 break; 6388 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6389 gfx_v8_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6390 break; 6391 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 6392 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 6393 break; 6394 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 6395 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 6396 break; 6397 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 6398 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 6399 break; 6400 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 6401 gfx_v8_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 6402 break; 6403 default: 6404 break; 6405 } 6406 return 0; 6407 } 6408 6409 static int gfx_v8_0_set_cp_ecc_int_state(struct amdgpu_device *adev, 6410 struct amdgpu_irq_src *source, 6411 unsigned int type, 6412 enum amdgpu_interrupt_state state) 6413 { 6414 int enable_flag; 6415 6416 switch (state) { 6417 case AMDGPU_IRQ_STATE_DISABLE: 6418 enable_flag = 0; 6419 break; 6420 6421 case AMDGPU_IRQ_STATE_ENABLE: 6422 enable_flag = 1; 6423 break; 6424 6425 default: 6426 return -EINVAL; 6427 } 6428 6429 WREG32_FIELD(CP_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6430 WREG32_FIELD(CP_INT_CNTL_RING0, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6431 WREG32_FIELD(CP_INT_CNTL_RING1, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6432 WREG32_FIELD(CP_INT_CNTL_RING2, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6433 WREG32_FIELD(CPC_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, enable_flag); 6434 WREG32_FIELD(CP_ME1_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6435 enable_flag); 6436 WREG32_FIELD(CP_ME1_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6437 enable_flag); 6438 WREG32_FIELD(CP_ME1_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6439 enable_flag); 6440 WREG32_FIELD(CP_ME1_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6441 enable_flag); 6442 WREG32_FIELD(CP_ME2_PIPE0_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6443 enable_flag); 6444 WREG32_FIELD(CP_ME2_PIPE1_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6445 enable_flag); 6446 WREG32_FIELD(CP_ME2_PIPE2_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6447 enable_flag); 6448 WREG32_FIELD(CP_ME2_PIPE3_INT_CNTL, CP_ECC_ERROR_INT_ENABLE, 6449 enable_flag); 6450 6451 return 0; 6452 } 6453 6454 static int gfx_v8_0_set_sq_int_state(struct amdgpu_device *adev, 6455 struct amdgpu_irq_src *source, 6456 unsigned int type, 6457 enum amdgpu_interrupt_state state) 6458 { 6459 int enable_flag; 6460 6461 switch (state) { 6462 case AMDGPU_IRQ_STATE_DISABLE: 6463 enable_flag = 1; 6464 break; 6465 6466 case AMDGPU_IRQ_STATE_ENABLE: 6467 enable_flag = 0; 6468 break; 6469 6470 default: 6471 return -EINVAL; 6472 } 6473 6474 WREG32_FIELD(SQ_INTERRUPT_MSG_CTRL, STALL, 6475 enable_flag); 6476 6477 return 0; 6478 } 6479 6480 static int gfx_v8_0_eop_irq(struct amdgpu_device *adev, 6481 struct amdgpu_irq_src *source, 6482 struct amdgpu_iv_entry *entry) 6483 { 6484 int i; 6485 u8 me_id, pipe_id, queue_id; 6486 struct amdgpu_ring *ring; 6487 6488 DRM_DEBUG("IH: CP EOP\n"); 6489 me_id = (entry->ring_id & 0x0c) >> 2; 6490 pipe_id = (entry->ring_id & 0x03) >> 0; 6491 queue_id = (entry->ring_id & 0x70) >> 4; 6492 6493 switch (me_id) { 6494 case 0: 6495 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 6496 break; 6497 case 1: 6498 case 2: 6499 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6500 ring = &adev->gfx.compute_ring[i]; 6501 /* Per-queue interrupt is supported for MEC starting from VI. 6502 * The interrupt can only be enabled/disabled per pipe instead of per queue. 6503 */ 6504 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 6505 amdgpu_fence_process(ring); 6506 } 6507 break; 6508 } 6509 return 0; 6510 } 6511 6512 static void gfx_v8_0_fault(struct amdgpu_device *adev, 6513 struct amdgpu_iv_entry *entry) 6514 { 6515 u8 me_id, pipe_id, queue_id; 6516 struct amdgpu_ring *ring; 6517 int i; 6518 6519 me_id = (entry->ring_id & 0x0c) >> 2; 6520 pipe_id = (entry->ring_id & 0x03) >> 0; 6521 queue_id = (entry->ring_id & 0x70) >> 4; 6522 6523 switch (me_id) { 6524 case 0: 6525 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 6526 break; 6527 case 1: 6528 case 2: 6529 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6530 ring = &adev->gfx.compute_ring[i]; 6531 if (ring->me == me_id && ring->pipe == pipe_id && 6532 ring->queue == queue_id) 6533 drm_sched_fault(&ring->sched); 6534 } 6535 break; 6536 } 6537 } 6538 6539 static int gfx_v8_0_priv_reg_irq(struct amdgpu_device *adev, 6540 struct amdgpu_irq_src *source, 6541 struct amdgpu_iv_entry *entry) 6542 { 6543 DRM_ERROR("Illegal register access in command stream\n"); 6544 gfx_v8_0_fault(adev, entry); 6545 return 0; 6546 } 6547 6548 static int gfx_v8_0_priv_inst_irq(struct amdgpu_device *adev, 6549 struct amdgpu_irq_src *source, 6550 struct amdgpu_iv_entry *entry) 6551 { 6552 DRM_ERROR("Illegal instruction in command stream\n"); 6553 gfx_v8_0_fault(adev, entry); 6554 return 0; 6555 } 6556 6557 static int gfx_v8_0_cp_ecc_error_irq(struct amdgpu_device *adev, 6558 struct amdgpu_irq_src *source, 6559 struct amdgpu_iv_entry *entry) 6560 { 6561 DRM_ERROR("CP EDC/ECC error detected."); 6562 return 0; 6563 } 6564 6565 static void gfx_v8_0_parse_sq_irq(struct amdgpu_device *adev, unsigned ih_data, 6566 bool from_wq) 6567 { 6568 u32 enc, se_id, sh_id, cu_id; 6569 char type[20]; 6570 int sq_edc_source = -1; 6571 6572 enc = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, ENCODING); 6573 se_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_CMN, SE_ID); 6574 6575 switch (enc) { 6576 case 0: 6577 drm_info(adev_to_drm(adev), "SQ general purpose intr detected:" 6578 "se_id %d, immed_overflow %d, host_reg_overflow %d," 6579 "host_cmd_overflow %d, cmd_timestamp %d," 6580 "reg_timestamp %d, thread_trace_buff_full %d," 6581 "wlt %d, thread_trace %d.\n", 6582 se_id, 6583 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, IMMED_OVERFLOW), 6584 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_REG_OVERFLOW), 6585 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, HOST_CMD_OVERFLOW), 6586 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, CMD_TIMESTAMP), 6587 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, REG_TIMESTAMP), 6588 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE_BUF_FULL), 6589 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, WLT), 6590 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_AUTO, THREAD_TRACE) 6591 ); 6592 break; 6593 case 1: 6594 case 2: 6595 6596 cu_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, CU_ID); 6597 sh_id = REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SH_ID); 6598 6599 /* 6600 * This function can be called either directly from ISR 6601 * or from BH in which case we can access SQ_EDC_INFO 6602 * instance 6603 */ 6604 if (from_wq) { 6605 mutex_lock(&adev->grbm_idx_mutex); 6606 gfx_v8_0_select_se_sh(adev, se_id, sh_id, cu_id, 0); 6607 6608 sq_edc_source = REG_GET_FIELD(RREG32(mmSQ_EDC_INFO), SQ_EDC_INFO, SOURCE); 6609 6610 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 6611 mutex_unlock(&adev->grbm_idx_mutex); 6612 } 6613 6614 if (enc == 1) 6615 sprintf(type, "instruction intr"); 6616 else 6617 sprintf(type, "EDC/ECC error"); 6618 6619 drm_info(adev_to_drm(adev), 6620 "SQ %s detected: " 6621 "se_id %d, sh_id %d, cu_id %d, simd_id %d, wave_id %d, vm_id %d " 6622 "trap %s, sq_ed_info.source %s.\n", 6623 type, se_id, sh_id, cu_id, 6624 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, SIMD_ID), 6625 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, WAVE_ID), 6626 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, VM_ID), 6627 REG_GET_FIELD(ih_data, SQ_INTERRUPT_WORD_WAVE, PRIV) ? "true" : "false", 6628 (sq_edc_source != -1) ? sq_edc_source_names[sq_edc_source] : "unavailable" 6629 ); 6630 break; 6631 default: 6632 DRM_ERROR("SQ invalid encoding type\n."); 6633 } 6634 } 6635 6636 static void gfx_v8_0_sq_irq_work_func(struct work_struct *work) 6637 { 6638 6639 struct amdgpu_device *adev = container_of(work, struct amdgpu_device, gfx.sq_work.work); 6640 struct sq_work *sq_work = container_of(work, struct sq_work, work); 6641 6642 gfx_v8_0_parse_sq_irq(adev, sq_work->ih_data, true); 6643 } 6644 6645 static int gfx_v8_0_sq_irq(struct amdgpu_device *adev, 6646 struct amdgpu_irq_src *source, 6647 struct amdgpu_iv_entry *entry) 6648 { 6649 unsigned ih_data = entry->src_data[0]; 6650 6651 /* 6652 * Try to submit work so SQ_EDC_INFO can be accessed from 6653 * BH. If previous work submission hasn't finished yet 6654 * just print whatever info is possible directly from the ISR. 6655 */ 6656 if (work_pending(&adev->gfx.sq_work.work)) { 6657 gfx_v8_0_parse_sq_irq(adev, ih_data, false); 6658 } else { 6659 adev->gfx.sq_work.ih_data = ih_data; 6660 schedule_work(&adev->gfx.sq_work.work); 6661 } 6662 6663 return 0; 6664 } 6665 6666 static void gfx_v8_0_emit_mem_sync(struct amdgpu_ring *ring) 6667 { 6668 amdgpu_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3)); 6669 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6670 PACKET3_TC_ACTION_ENA | 6671 PACKET3_SH_KCACHE_ACTION_ENA | 6672 PACKET3_SH_ICACHE_ACTION_ENA | 6673 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6674 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6675 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6676 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6677 } 6678 6679 static void gfx_v8_0_emit_mem_sync_compute(struct amdgpu_ring *ring) 6680 { 6681 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6682 amdgpu_ring_write(ring, PACKET3_TCL1_ACTION_ENA | 6683 PACKET3_TC_ACTION_ENA | 6684 PACKET3_SH_KCACHE_ACTION_ENA | 6685 PACKET3_SH_ICACHE_ACTION_ENA | 6686 PACKET3_TC_WB_ACTION_ENA); /* CP_COHER_CNTL */ 6687 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6688 amdgpu_ring_write(ring, 0xff); /* CP_COHER_SIZE_HI */ 6689 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6690 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6691 amdgpu_ring_write(ring, 0x0000000A); /* poll interval */ 6692 } 6693 6694 6695 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6696 #define mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT 0x0000007f 6697 static void gfx_v8_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6698 uint32_t pipe, bool enable) 6699 { 6700 uint32_t val; 6701 uint32_t wcl_cs_reg; 6702 6703 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS_DEFAULT; 6704 6705 switch (pipe) { 6706 case 0: 6707 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS0; 6708 break; 6709 case 1: 6710 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS1; 6711 break; 6712 case 2: 6713 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS2; 6714 break; 6715 case 3: 6716 wcl_cs_reg = mmSPI_WCL_PIPE_PERCENT_CS3; 6717 break; 6718 default: 6719 DRM_DEBUG("invalid pipe %d\n", pipe); 6720 return; 6721 } 6722 6723 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6724 6725 } 6726 6727 #define mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT 0x07ffffff 6728 static void gfx_v8_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6729 { 6730 struct amdgpu_device *adev = ring->adev; 6731 uint32_t val; 6732 int i; 6733 6734 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6735 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6736 * around 25% of gpu resources. 6737 */ 6738 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6739 amdgpu_ring_emit_wreg(ring, mmSPI_WCL_PIPE_PERCENT_GFX, val); 6740 6741 /* Restrict waves for normal/low priority compute queues as well 6742 * to get best QoS for high priority compute jobs. 6743 * 6744 * amdgpu controls only 1st ME(0-3 CS pipes). 6745 */ 6746 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6747 if (i != ring->pipe) 6748 gfx_v8_0_emit_wave_limit_cs(ring, i, enable); 6749 6750 } 6751 6752 } 6753 6754 static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { 6755 .name = "gfx_v8_0", 6756 .early_init = gfx_v8_0_early_init, 6757 .late_init = gfx_v8_0_late_init, 6758 .sw_init = gfx_v8_0_sw_init, 6759 .sw_fini = gfx_v8_0_sw_fini, 6760 .hw_init = gfx_v8_0_hw_init, 6761 .hw_fini = gfx_v8_0_hw_fini, 6762 .suspend = gfx_v8_0_suspend, 6763 .resume = gfx_v8_0_resume, 6764 .is_idle = gfx_v8_0_is_idle, 6765 .wait_for_idle = gfx_v8_0_wait_for_idle, 6766 .soft_reset = gfx_v8_0_soft_reset, 6767 .set_clockgating_state = gfx_v8_0_set_clockgating_state, 6768 .set_powergating_state = gfx_v8_0_set_powergating_state, 6769 .get_clockgating_state = gfx_v8_0_get_clockgating_state, 6770 }; 6771 6772 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_gfx = { 6773 .type = AMDGPU_RING_TYPE_GFX, 6774 .align_mask = 0xff, 6775 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6776 .support_64bit_ptrs = false, 6777 .get_rptr = gfx_v8_0_ring_get_rptr, 6778 .get_wptr = gfx_v8_0_ring_get_wptr_gfx, 6779 .set_wptr = gfx_v8_0_ring_set_wptr_gfx, 6780 .emit_frame_size = /* maximum 215dw if count 16 IBs in */ 6781 5 + /* COND_EXEC */ 6782 7 + /* PIPELINE_SYNC */ 6783 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 9 + /* VM_FLUSH */ 6784 12 + /* FENCE for VM_FLUSH */ 6785 20 + /* GDS switch */ 6786 4 + /* double SWITCH_BUFFER, 6787 the first COND_EXEC jump to the place just 6788 prior to this double SWITCH_BUFFER */ 6789 5 + /* COND_EXEC */ 6790 7 + /* HDP_flush */ 6791 4 + /* VGT_flush */ 6792 14 + /* CE_META */ 6793 31 + /* DE_META */ 6794 3 + /* CNTX_CTRL */ 6795 5 + /* HDP_INVL */ 6796 12 + 12 + /* FENCE x2 */ 6797 2 + /* SWITCH_BUFFER */ 6798 5, /* SURFACE_SYNC */ 6799 .emit_ib_size = 4, /* gfx_v8_0_ring_emit_ib_gfx */ 6800 .emit_ib = gfx_v8_0_ring_emit_ib_gfx, 6801 .emit_fence = gfx_v8_0_ring_emit_fence_gfx, 6802 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6803 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6804 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6805 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6806 .test_ring = gfx_v8_0_ring_test_ring, 6807 .test_ib = gfx_v8_0_ring_test_ib, 6808 .insert_nop = amdgpu_ring_insert_nop, 6809 .pad_ib = amdgpu_ring_generic_pad_ib, 6810 .emit_switch_buffer = gfx_v8_ring_emit_sb, 6811 .emit_cntxcntl = gfx_v8_ring_emit_cntxcntl, 6812 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, 6813 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6814 .soft_recovery = gfx_v8_0_ring_soft_recovery, 6815 .emit_mem_sync = gfx_v8_0_emit_mem_sync, 6816 }; 6817 6818 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { 6819 .type = AMDGPU_RING_TYPE_COMPUTE, 6820 .align_mask = 0xff, 6821 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6822 .support_64bit_ptrs = false, 6823 .get_rptr = gfx_v8_0_ring_get_rptr, 6824 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6825 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6826 .emit_frame_size = 6827 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_ib_schedule) */ 6828 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6829 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6830 5 + /* hdp_invalidate */ 6831 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6832 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_vm_flush) */ 6833 VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ 6834 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ 6835 7 + /* gfx_v8_0_emit_mem_sync_compute */ 6836 5 + /* gfx_v8_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6837 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6838 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6839 .emit_ib = gfx_v8_0_ring_emit_ib_compute, 6840 .emit_fence = gfx_v8_0_ring_emit_fence_compute, 6841 .emit_pipeline_sync = gfx_v8_0_ring_emit_pipeline_sync, 6842 .emit_vm_flush = gfx_v8_0_ring_emit_vm_flush, 6843 .emit_gds_switch = gfx_v8_0_ring_emit_gds_switch, 6844 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6845 .test_ring = gfx_v8_0_ring_test_ring, 6846 .test_ib = gfx_v8_0_ring_test_ib, 6847 .insert_nop = amdgpu_ring_insert_nop, 6848 .pad_ib = amdgpu_ring_generic_pad_ib, 6849 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6850 .soft_recovery = gfx_v8_0_ring_soft_recovery, 6851 .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute, 6852 .emit_wave_limit = gfx_v8_0_emit_wave_limit, 6853 .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, 6854 }; 6855 6856 static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { 6857 .type = AMDGPU_RING_TYPE_KIQ, 6858 .align_mask = 0xff, 6859 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6860 .support_64bit_ptrs = false, 6861 .get_rptr = gfx_v8_0_ring_get_rptr, 6862 .get_wptr = gfx_v8_0_ring_get_wptr_compute, 6863 .set_wptr = gfx_v8_0_ring_set_wptr_compute, 6864 .emit_frame_size = 6865 20 + /* gfx_v8_0_ring_emit_gds_switch */ 6866 7 + /* gfx_v8_0_ring_emit_hdp_flush */ 6867 5 + /* hdp_invalidate */ 6868 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ 6869 17 + /* gfx_v8_0_ring_emit_vm_flush */ 6870 7 + 7 + 7, /* gfx_v8_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6871 .emit_ib_size = 7, /* gfx_v8_0_ring_emit_ib_compute */ 6872 .emit_fence = gfx_v8_0_ring_emit_fence_kiq, 6873 .test_ring = gfx_v8_0_ring_test_ring, 6874 .insert_nop = amdgpu_ring_insert_nop, 6875 .pad_ib = amdgpu_ring_generic_pad_ib, 6876 .emit_rreg = gfx_v8_0_ring_emit_rreg, 6877 .emit_wreg = gfx_v8_0_ring_emit_wreg, 6878 .emit_hdp_flush = gfx_v8_0_ring_emit_hdp_flush, 6879 }; 6880 6881 static void gfx_v8_0_set_ring_funcs(struct amdgpu_device *adev) 6882 { 6883 int i; 6884 6885 adev->gfx.kiq[0].ring.funcs = &gfx_v8_0_ring_funcs_kiq; 6886 6887 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6888 adev->gfx.gfx_ring[i].funcs = &gfx_v8_0_ring_funcs_gfx; 6889 6890 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6891 adev->gfx.compute_ring[i].funcs = &gfx_v8_0_ring_funcs_compute; 6892 } 6893 6894 static const struct amdgpu_irq_src_funcs gfx_v8_0_eop_irq_funcs = { 6895 .set = gfx_v8_0_set_eop_interrupt_state, 6896 .process = gfx_v8_0_eop_irq, 6897 }; 6898 6899 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_reg_irq_funcs = { 6900 .set = gfx_v8_0_set_priv_reg_fault_state, 6901 .process = gfx_v8_0_priv_reg_irq, 6902 }; 6903 6904 static const struct amdgpu_irq_src_funcs gfx_v8_0_priv_inst_irq_funcs = { 6905 .set = gfx_v8_0_set_priv_inst_fault_state, 6906 .process = gfx_v8_0_priv_inst_irq, 6907 }; 6908 6909 static const struct amdgpu_irq_src_funcs gfx_v8_0_cp_ecc_error_irq_funcs = { 6910 .set = gfx_v8_0_set_cp_ecc_int_state, 6911 .process = gfx_v8_0_cp_ecc_error_irq, 6912 }; 6913 6914 static const struct amdgpu_irq_src_funcs gfx_v8_0_sq_irq_funcs = { 6915 .set = gfx_v8_0_set_sq_int_state, 6916 .process = gfx_v8_0_sq_irq, 6917 }; 6918 6919 static void gfx_v8_0_set_irq_funcs(struct amdgpu_device *adev) 6920 { 6921 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6922 adev->gfx.eop_irq.funcs = &gfx_v8_0_eop_irq_funcs; 6923 6924 adev->gfx.priv_reg_irq.num_types = 1; 6925 adev->gfx.priv_reg_irq.funcs = &gfx_v8_0_priv_reg_irq_funcs; 6926 6927 adev->gfx.priv_inst_irq.num_types = 1; 6928 adev->gfx.priv_inst_irq.funcs = &gfx_v8_0_priv_inst_irq_funcs; 6929 6930 adev->gfx.cp_ecc_error_irq.num_types = 1; 6931 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v8_0_cp_ecc_error_irq_funcs; 6932 6933 adev->gfx.sq_irq.num_types = 1; 6934 adev->gfx.sq_irq.funcs = &gfx_v8_0_sq_irq_funcs; 6935 } 6936 6937 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev) 6938 { 6939 adev->gfx.rlc.funcs = &iceland_rlc_funcs; 6940 } 6941 6942 static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev) 6943 { 6944 /* init asci gds info */ 6945 adev->gds.gds_size = RREG32(mmGDS_VMID0_SIZE); 6946 adev->gds.gws_size = 64; 6947 adev->gds.oa_size = 16; 6948 adev->gds.gds_compute_max_wave_id = RREG32(mmGDS_COMPUTE_MAX_WAVE_ID); 6949 } 6950 6951 static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6952 u32 bitmap) 6953 { 6954 u32 data; 6955 6956 if (!bitmap) 6957 return; 6958 6959 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6960 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6961 6962 WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data); 6963 } 6964 6965 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6966 { 6967 u32 data, mask; 6968 6969 data = RREG32(mmCC_GC_SHADER_ARRAY_CONFIG) | 6970 RREG32(mmGC_USER_SHADER_ARRAY_CONFIG); 6971 6972 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6973 6974 return ~REG_GET_FIELD(data, CC_GC_SHADER_ARRAY_CONFIG, INACTIVE_CUS) & mask; 6975 } 6976 6977 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev) 6978 { 6979 int i, j, k, counter, active_cu_number = 0; 6980 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6981 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 6982 unsigned disable_masks[4 * 2]; 6983 u32 ao_cu_num; 6984 6985 memset(cu_info, 0, sizeof(*cu_info)); 6986 6987 if (adev->flags & AMD_IS_APU) 6988 ao_cu_num = 2; 6989 else 6990 ao_cu_num = adev->gfx.config.max_cu_per_sh; 6991 6992 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 4, 2); 6993 6994 mutex_lock(&adev->grbm_idx_mutex); 6995 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6996 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 6997 mask = 1; 6998 ao_bitmap = 0; 6999 counter = 0; 7000 gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7001 if (i < 4 && j < 2) 7002 gfx_v8_0_set_user_cu_inactive_bitmap( 7003 adev, disable_masks[i * 2 + j]); 7004 bitmap = gfx_v8_0_get_cu_active_bitmap(adev); 7005 cu_info->bitmap[0][i][j] = bitmap; 7006 7007 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7008 if (bitmap & mask) { 7009 if (counter < ao_cu_num) 7010 ao_bitmap |= mask; 7011 counter ++; 7012 } 7013 mask <<= 1; 7014 } 7015 active_cu_number += counter; 7016 if (i < 2 && j < 2) 7017 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7018 cu_info->ao_cu_bitmap[i][j] = ao_bitmap; 7019 } 7020 } 7021 gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7022 mutex_unlock(&adev->grbm_idx_mutex); 7023 7024 cu_info->number = active_cu_number; 7025 cu_info->ao_cu_mask = ao_cu_mask; 7026 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7027 cu_info->max_waves_per_simd = 10; 7028 cu_info->max_scratch_slots_per_cu = 32; 7029 cu_info->wave_front_size = 64; 7030 cu_info->lds_size = 64; 7031 } 7032 7033 const struct amdgpu_ip_block_version gfx_v8_0_ip_block = 7034 { 7035 .type = AMD_IP_BLOCK_TYPE_GFX, 7036 .major = 8, 7037 .minor = 0, 7038 .rev = 0, 7039 .funcs = &gfx_v8_0_ip_funcs, 7040 }; 7041 7042 const struct amdgpu_ip_block_version gfx_v8_1_ip_block = 7043 { 7044 .type = AMD_IP_BLOCK_TYPE_GFX, 7045 .major = 8, 7046 .minor = 1, 7047 .rev = 0, 7048 .funcs = &gfx_v8_0_ip_funcs, 7049 }; 7050 7051 static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 7052 { 7053 uint64_t ce_payload_addr; 7054 int cnt_ce; 7055 union { 7056 struct vi_ce_ib_state regular; 7057 struct vi_ce_ib_state_chained_ib chained; 7058 } ce_payload = {}; 7059 7060 if (ring->adev->virt.chained_ib_support) { 7061 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7062 offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload); 7063 cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2; 7064 } else { 7065 ce_payload_addr = amdgpu_csa_vaddr(ring->adev) + 7066 offsetof(struct vi_gfx_meta_data, ce_payload); 7067 cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2; 7068 } 7069 7070 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_ce)); 7071 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 7072 WRITE_DATA_DST_SEL(8) | 7073 WR_CONFIRM) | 7074 WRITE_DATA_CACHE_POLICY(0)); 7075 amdgpu_ring_write(ring, lower_32_bits(ce_payload_addr)); 7076 amdgpu_ring_write(ring, upper_32_bits(ce_payload_addr)); 7077 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2); 7078 } 7079 7080 static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring) 7081 { 7082 uint64_t de_payload_addr, gds_addr, csa_addr; 7083 int cnt_de; 7084 union { 7085 struct vi_de_ib_state regular; 7086 struct vi_de_ib_state_chained_ib chained; 7087 } de_payload = {}; 7088 7089 csa_addr = amdgpu_csa_vaddr(ring->adev); 7090 gds_addr = csa_addr + 4096; 7091 if (ring->adev->virt.chained_ib_support) { 7092 de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr); 7093 de_payload.chained.gds_backup_addrhi = upper_32_bits(gds_addr); 7094 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data_chained_ib, de_payload); 7095 cnt_de = (sizeof(de_payload.chained) >> 2) + 4 - 2; 7096 } else { 7097 de_payload.regular.gds_backup_addrlo = lower_32_bits(gds_addr); 7098 de_payload.regular.gds_backup_addrhi = upper_32_bits(gds_addr); 7099 de_payload_addr = csa_addr + offsetof(struct vi_gfx_meta_data, de_payload); 7100 cnt_de = (sizeof(de_payload.regular) >> 2) + 4 - 2; 7101 } 7102 7103 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt_de)); 7104 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 7105 WRITE_DATA_DST_SEL(8) | 7106 WR_CONFIRM) | 7107 WRITE_DATA_CACHE_POLICY(0)); 7108 amdgpu_ring_write(ring, lower_32_bits(de_payload_addr)); 7109 amdgpu_ring_write(ring, upper_32_bits(de_payload_addr)); 7110 amdgpu_ring_write_multiple(ring, (void *)&de_payload, cnt_de - 2); 7111 } 7112