1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "gfx_v9_4.h" 51 #include "gfx_v9_0.h" 52 #include "gfx_v9_4_2.h" 53 54 #include "asic_reg/pwr/pwr_10_0_offset.h" 55 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 56 #include "asic_reg/gc/gc_9_0_default.h" 57 58 #define GFX9_NUM_GFX_RINGS 1 59 #define GFX9_MEC_HPD_SIZE 4096 60 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 61 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 62 63 #define mmGCEA_PROBE_MAP 0x070c 64 #define mmGCEA_PROBE_MAP_BASE_IDX 0 65 66 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 72 73 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 79 80 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 86 87 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 93 94 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 101 102 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 109 110 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 125 126 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 128 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 129 130 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 131 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 132 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 133 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 134 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 135 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 136 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 137 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 138 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 139 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 140 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 141 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 142 143 enum ta_ras_gfx_subblock { 144 /*CPC*/ 145 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 146 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 147 TA_RAS_BLOCK__GFX_CPC_UCODE, 148 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 149 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 150 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 151 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 152 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 153 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 154 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 155 /* CPF*/ 156 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 157 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 158 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 159 TA_RAS_BLOCK__GFX_CPF_TAG, 160 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 161 /* CPG*/ 162 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 163 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 164 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 165 TA_RAS_BLOCK__GFX_CPG_TAG, 166 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 167 /* GDS*/ 168 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 169 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 170 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 171 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 172 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 173 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 174 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 175 /* SPI*/ 176 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 177 /* SQ*/ 178 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 179 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 180 TA_RAS_BLOCK__GFX_SQ_LDS_D, 181 TA_RAS_BLOCK__GFX_SQ_LDS_I, 182 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 183 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 184 /* SQC (3 ranges)*/ 185 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 186 /* SQC range 0*/ 187 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 188 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 189 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 190 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 191 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 192 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 193 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 194 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 195 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 196 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 197 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 198 /* SQC range 1*/ 199 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 201 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 202 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 203 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 204 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 206 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 207 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 208 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 209 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 210 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 211 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 212 /* SQC range 2*/ 213 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 215 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 216 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 217 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 218 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 220 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 221 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 222 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 223 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 224 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 225 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 226 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 227 /* TA*/ 228 TA_RAS_BLOCK__GFX_TA_INDEX_START, 229 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 230 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 231 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 232 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 233 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 234 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 235 /* TCA*/ 236 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 237 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 238 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 239 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 240 /* TCC (5 sub-ranges)*/ 241 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 242 /* TCC range 0*/ 243 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 244 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 245 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 246 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 247 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 248 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 249 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 250 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 251 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 252 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 253 /* TCC range 1*/ 254 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 255 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 256 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 257 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 258 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 259 /* TCC range 2*/ 260 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 261 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 262 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 263 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 264 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 265 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 266 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 267 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 268 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 269 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 270 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 271 /* TCC range 3*/ 272 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 273 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 274 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 275 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 276 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 277 /* TCC range 4*/ 278 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 279 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 280 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 281 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 282 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 283 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 284 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 285 /* TCI*/ 286 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 287 /* TCP*/ 288 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 289 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 290 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 291 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 292 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 293 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 294 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 295 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 296 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 297 /* TD*/ 298 TA_RAS_BLOCK__GFX_TD_INDEX_START, 299 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 300 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 301 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 302 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 303 /* EA (3 sub-ranges)*/ 304 TA_RAS_BLOCK__GFX_EA_INDEX_START, 305 /* EA range 0*/ 306 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 307 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 308 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 309 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 310 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 311 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 312 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 313 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 314 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 315 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 316 /* EA range 1*/ 317 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 318 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 319 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 320 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 321 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 322 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 323 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 324 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 325 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 326 /* EA range 2*/ 327 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 328 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 329 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 330 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 331 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 332 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 333 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 334 /* UTC VM L2 bank*/ 335 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 336 /* UTC VM walker*/ 337 TA_RAS_BLOCK__UTC_VML2_WALKER, 338 /* UTC ATC L2 2MB cache*/ 339 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 340 /* UTC ATC L2 4KB cache*/ 341 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 342 TA_RAS_BLOCK__GFX_MAX 343 }; 344 345 struct ras_gfx_subblock { 346 unsigned char *name; 347 int ta_subblock; 348 int hw_supported_error_type; 349 int sw_supported_error_type; 350 }; 351 352 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 353 [AMDGPU_RAS_BLOCK__##subblock] = { \ 354 #subblock, \ 355 TA_RAS_BLOCK__##subblock, \ 356 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 357 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 358 } 359 360 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 361 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 362 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 368 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 369 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 371 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 372 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 374 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 375 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 376 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 377 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 378 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 380 0), 381 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 384 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 385 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 388 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 389 0, 0), 390 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 391 0), 392 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 393 0, 0), 394 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 395 0), 396 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 397 0, 0), 398 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 399 0), 400 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 401 1), 402 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 403 0, 0, 0), 404 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 405 0), 406 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 407 0), 408 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 409 0), 410 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 411 0), 412 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 413 0), 414 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 415 0, 0), 416 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 417 0), 418 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 419 0), 420 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 421 0, 0, 0), 422 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 423 0), 424 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 425 0), 426 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 427 0), 428 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 429 0), 430 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 431 0), 432 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 433 0, 0), 434 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 435 0), 436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 438 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 441 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 442 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 445 1), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 447 1), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 449 1), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 451 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 453 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 462 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 466 0), 467 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 469 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 471 0, 0), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 473 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 482 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 483 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 504 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 505 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 506 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 507 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 508 }; 509 510 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 511 { 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 528 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 532 }; 533 534 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 535 { 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 550 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 554 }; 555 556 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 557 { 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 565 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 569 }; 570 571 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 572 { 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 593 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 597 }; 598 599 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 600 { 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 604 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 608 }; 609 610 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 611 { 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 627 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 631 }; 632 633 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 634 { 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 643 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 647 }; 648 649 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 650 { 651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 654 }; 655 656 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 657 { 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 670 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 674 }; 675 676 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 677 { 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 691 }; 692 693 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 694 { 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 706 }; 707 708 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 709 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 710 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 711 }; 712 713 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 714 { 715 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 720 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 721 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 722 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 723 }; 724 725 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 726 { 727 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 732 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 733 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 734 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 735 }; 736 737 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v) 738 { 739 static void *scratch_reg0; 740 static void *scratch_reg1; 741 static void *scratch_reg2; 742 static void *scratch_reg3; 743 static void *spare_int; 744 static uint32_t grbm_cntl; 745 static uint32_t grbm_idx; 746 747 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 748 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 749 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 750 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 751 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 752 753 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 754 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 755 756 if (amdgpu_sriov_runtime(adev)) { 757 pr_err("shouldn't call rlcg write register during runtime\n"); 758 return; 759 } 760 761 if (offset == grbm_cntl || offset == grbm_idx) { 762 if (offset == grbm_cntl) 763 writel(v, scratch_reg2); 764 else if (offset == grbm_idx) 765 writel(v, scratch_reg3); 766 767 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 768 } else { 769 uint32_t i = 0; 770 uint32_t retries = 50000; 771 772 writel(v, scratch_reg0); 773 writel(offset | 0x80000000, scratch_reg1); 774 writel(1, spare_int); 775 for (i = 0; i < retries; i++) { 776 u32 tmp; 777 778 tmp = readl(scratch_reg1); 779 if (!(tmp & 0x80000000)) 780 break; 781 782 udelay(10); 783 } 784 if (i >= retries) 785 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 786 } 787 788 } 789 790 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 791 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 792 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 793 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 794 795 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 796 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 797 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 798 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 799 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 800 struct amdgpu_cu_info *cu_info); 801 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 802 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 803 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 804 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 805 void *ras_error_status); 806 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 807 void *inject_if); 808 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 809 810 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 811 uint64_t queue_mask) 812 { 813 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 814 amdgpu_ring_write(kiq_ring, 815 PACKET3_SET_RESOURCES_VMID_MASK(0) | 816 /* vmid_mask:0* queue_type:0 (KIQ) */ 817 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 818 amdgpu_ring_write(kiq_ring, 819 lower_32_bits(queue_mask)); /* queue mask lo */ 820 amdgpu_ring_write(kiq_ring, 821 upper_32_bits(queue_mask)); /* queue mask hi */ 822 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 823 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 824 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 825 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 826 } 827 828 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 829 struct amdgpu_ring *ring) 830 { 831 struct amdgpu_device *adev = kiq_ring->adev; 832 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 833 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 834 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 835 836 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 837 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 838 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 839 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 840 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 841 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 842 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 843 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 844 /*queue_type: normal compute queue */ 845 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 846 /* alloc format: all_on_one_pipe */ 847 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 848 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 849 /* num_queues: must be 1 */ 850 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 851 amdgpu_ring_write(kiq_ring, 852 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 853 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 854 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 855 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 856 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 857 } 858 859 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 860 struct amdgpu_ring *ring, 861 enum amdgpu_unmap_queues_action action, 862 u64 gpu_addr, u64 seq) 863 { 864 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 865 866 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 867 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 868 PACKET3_UNMAP_QUEUES_ACTION(action) | 869 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 870 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 871 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 872 amdgpu_ring_write(kiq_ring, 873 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 874 875 if (action == PREEMPT_QUEUES_NO_UNMAP) { 876 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 877 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 878 amdgpu_ring_write(kiq_ring, seq); 879 } else { 880 amdgpu_ring_write(kiq_ring, 0); 881 amdgpu_ring_write(kiq_ring, 0); 882 amdgpu_ring_write(kiq_ring, 0); 883 } 884 } 885 886 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 887 struct amdgpu_ring *ring, 888 u64 addr, 889 u64 seq) 890 { 891 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 892 893 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 894 amdgpu_ring_write(kiq_ring, 895 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 896 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 897 PACKET3_QUERY_STATUS_COMMAND(2)); 898 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 899 amdgpu_ring_write(kiq_ring, 900 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 901 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 902 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 903 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 904 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 905 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 906 } 907 908 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 909 uint16_t pasid, uint32_t flush_type, 910 bool all_hub) 911 { 912 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 913 amdgpu_ring_write(kiq_ring, 914 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 915 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 916 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 917 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 918 } 919 920 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 921 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 922 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 923 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 924 .kiq_query_status = gfx_v9_0_kiq_query_status, 925 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 926 .set_resources_size = 8, 927 .map_queues_size = 7, 928 .unmap_queues_size = 6, 929 .query_status_size = 7, 930 .invalidate_tlbs_size = 2, 931 }; 932 933 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 934 { 935 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 936 } 937 938 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 939 { 940 switch (adev->asic_type) { 941 case CHIP_VEGA10: 942 soc15_program_register_sequence(adev, 943 golden_settings_gc_9_0, 944 ARRAY_SIZE(golden_settings_gc_9_0)); 945 soc15_program_register_sequence(adev, 946 golden_settings_gc_9_0_vg10, 947 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 948 break; 949 case CHIP_VEGA12: 950 soc15_program_register_sequence(adev, 951 golden_settings_gc_9_2_1, 952 ARRAY_SIZE(golden_settings_gc_9_2_1)); 953 soc15_program_register_sequence(adev, 954 golden_settings_gc_9_2_1_vg12, 955 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 956 break; 957 case CHIP_VEGA20: 958 soc15_program_register_sequence(adev, 959 golden_settings_gc_9_0, 960 ARRAY_SIZE(golden_settings_gc_9_0)); 961 soc15_program_register_sequence(adev, 962 golden_settings_gc_9_0_vg20, 963 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 964 break; 965 case CHIP_ARCTURUS: 966 soc15_program_register_sequence(adev, 967 golden_settings_gc_9_4_1_arct, 968 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 969 break; 970 case CHIP_RAVEN: 971 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 972 ARRAY_SIZE(golden_settings_gc_9_1)); 973 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 974 soc15_program_register_sequence(adev, 975 golden_settings_gc_9_1_rv2, 976 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 977 else 978 soc15_program_register_sequence(adev, 979 golden_settings_gc_9_1_rv1, 980 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 981 break; 982 case CHIP_RENOIR: 983 soc15_program_register_sequence(adev, 984 golden_settings_gc_9_1_rn, 985 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 986 return; /* for renoir, don't need common goldensetting */ 987 case CHIP_ALDEBARAN: 988 gfx_v9_4_2_init_golden_registers(adev, 989 adev->smuio.funcs->get_die_id(adev)); 990 break; 991 default: 992 break; 993 } 994 995 if ((adev->asic_type != CHIP_ARCTURUS) && 996 (adev->asic_type != CHIP_ALDEBARAN)) 997 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 998 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 999 } 1000 1001 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 1002 { 1003 adev->gfx.scratch.num_reg = 8; 1004 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1005 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 1006 } 1007 1008 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1009 bool wc, uint32_t reg, uint32_t val) 1010 { 1011 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1012 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1013 WRITE_DATA_DST_SEL(0) | 1014 (wc ? WR_CONFIRM : 0)); 1015 amdgpu_ring_write(ring, reg); 1016 amdgpu_ring_write(ring, 0); 1017 amdgpu_ring_write(ring, val); 1018 } 1019 1020 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1021 int mem_space, int opt, uint32_t addr0, 1022 uint32_t addr1, uint32_t ref, uint32_t mask, 1023 uint32_t inv) 1024 { 1025 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1026 amdgpu_ring_write(ring, 1027 /* memory (1) or register (0) */ 1028 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1029 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1030 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1031 WAIT_REG_MEM_ENGINE(eng_sel))); 1032 1033 if (mem_space) 1034 BUG_ON(addr0 & 0x3); /* Dword align */ 1035 amdgpu_ring_write(ring, addr0); 1036 amdgpu_ring_write(ring, addr1); 1037 amdgpu_ring_write(ring, ref); 1038 amdgpu_ring_write(ring, mask); 1039 amdgpu_ring_write(ring, inv); /* poll interval */ 1040 } 1041 1042 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1043 { 1044 struct amdgpu_device *adev = ring->adev; 1045 uint32_t scratch; 1046 uint32_t tmp = 0; 1047 unsigned i; 1048 int r; 1049 1050 r = amdgpu_gfx_scratch_get(adev, &scratch); 1051 if (r) 1052 return r; 1053 1054 WREG32(scratch, 0xCAFEDEAD); 1055 r = amdgpu_ring_alloc(ring, 3); 1056 if (r) 1057 goto error_free_scratch; 1058 1059 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1060 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1061 amdgpu_ring_write(ring, 0xDEADBEEF); 1062 amdgpu_ring_commit(ring); 1063 1064 for (i = 0; i < adev->usec_timeout; i++) { 1065 tmp = RREG32(scratch); 1066 if (tmp == 0xDEADBEEF) 1067 break; 1068 udelay(1); 1069 } 1070 1071 if (i >= adev->usec_timeout) 1072 r = -ETIMEDOUT; 1073 1074 error_free_scratch: 1075 amdgpu_gfx_scratch_free(adev, scratch); 1076 return r; 1077 } 1078 1079 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1080 { 1081 struct amdgpu_device *adev = ring->adev; 1082 struct amdgpu_ib ib; 1083 struct dma_fence *f = NULL; 1084 1085 unsigned index; 1086 uint64_t gpu_addr; 1087 uint32_t tmp; 1088 long r; 1089 1090 r = amdgpu_device_wb_get(adev, &index); 1091 if (r) 1092 return r; 1093 1094 gpu_addr = adev->wb.gpu_addr + (index * 4); 1095 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1096 memset(&ib, 0, sizeof(ib)); 1097 r = amdgpu_ib_get(adev, NULL, 16, 1098 AMDGPU_IB_POOL_DIRECT, &ib); 1099 if (r) 1100 goto err1; 1101 1102 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1103 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1104 ib.ptr[2] = lower_32_bits(gpu_addr); 1105 ib.ptr[3] = upper_32_bits(gpu_addr); 1106 ib.ptr[4] = 0xDEADBEEF; 1107 ib.length_dw = 5; 1108 1109 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1110 if (r) 1111 goto err2; 1112 1113 r = dma_fence_wait_timeout(f, false, timeout); 1114 if (r == 0) { 1115 r = -ETIMEDOUT; 1116 goto err2; 1117 } else if (r < 0) { 1118 goto err2; 1119 } 1120 1121 tmp = adev->wb.wb[index]; 1122 if (tmp == 0xDEADBEEF) 1123 r = 0; 1124 else 1125 r = -EINVAL; 1126 1127 err2: 1128 amdgpu_ib_free(adev, &ib, NULL); 1129 dma_fence_put(f); 1130 err1: 1131 amdgpu_device_wb_free(adev, index); 1132 return r; 1133 } 1134 1135 1136 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1137 { 1138 release_firmware(adev->gfx.pfp_fw); 1139 adev->gfx.pfp_fw = NULL; 1140 release_firmware(adev->gfx.me_fw); 1141 adev->gfx.me_fw = NULL; 1142 release_firmware(adev->gfx.ce_fw); 1143 adev->gfx.ce_fw = NULL; 1144 release_firmware(adev->gfx.rlc_fw); 1145 adev->gfx.rlc_fw = NULL; 1146 release_firmware(adev->gfx.mec_fw); 1147 adev->gfx.mec_fw = NULL; 1148 release_firmware(adev->gfx.mec2_fw); 1149 adev->gfx.mec2_fw = NULL; 1150 1151 kfree(adev->gfx.rlc.register_list_format); 1152 } 1153 1154 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1155 { 1156 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1157 1158 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1159 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1160 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1161 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1162 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1163 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1164 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1165 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1166 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1167 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1168 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1169 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1170 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1171 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1172 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1173 } 1174 1175 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1176 { 1177 adev->gfx.me_fw_write_wait = false; 1178 adev->gfx.mec_fw_write_wait = false; 1179 1180 if ((adev->asic_type != CHIP_ARCTURUS) && 1181 ((adev->gfx.mec_fw_version < 0x000001a5) || 1182 (adev->gfx.mec_feature_version < 46) || 1183 (adev->gfx.pfp_fw_version < 0x000000b7) || 1184 (adev->gfx.pfp_feature_version < 46))) 1185 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1186 1187 switch (adev->asic_type) { 1188 case CHIP_VEGA10: 1189 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1190 (adev->gfx.me_feature_version >= 42) && 1191 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1192 (adev->gfx.pfp_feature_version >= 42)) 1193 adev->gfx.me_fw_write_wait = true; 1194 1195 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1196 (adev->gfx.mec_feature_version >= 42)) 1197 adev->gfx.mec_fw_write_wait = true; 1198 break; 1199 case CHIP_VEGA12: 1200 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1201 (adev->gfx.me_feature_version >= 44) && 1202 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1203 (adev->gfx.pfp_feature_version >= 44)) 1204 adev->gfx.me_fw_write_wait = true; 1205 1206 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1207 (adev->gfx.mec_feature_version >= 44)) 1208 adev->gfx.mec_fw_write_wait = true; 1209 break; 1210 case CHIP_VEGA20: 1211 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1212 (adev->gfx.me_feature_version >= 44) && 1213 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1214 (adev->gfx.pfp_feature_version >= 44)) 1215 adev->gfx.me_fw_write_wait = true; 1216 1217 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1218 (adev->gfx.mec_feature_version >= 44)) 1219 adev->gfx.mec_fw_write_wait = true; 1220 break; 1221 case CHIP_RAVEN: 1222 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1223 (adev->gfx.me_feature_version >= 42) && 1224 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1225 (adev->gfx.pfp_feature_version >= 42)) 1226 adev->gfx.me_fw_write_wait = true; 1227 1228 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1229 (adev->gfx.mec_feature_version >= 42)) 1230 adev->gfx.mec_fw_write_wait = true; 1231 break; 1232 default: 1233 adev->gfx.me_fw_write_wait = true; 1234 adev->gfx.mec_fw_write_wait = true; 1235 break; 1236 } 1237 } 1238 1239 struct amdgpu_gfxoff_quirk { 1240 u16 chip_vendor; 1241 u16 chip_device; 1242 u16 subsys_vendor; 1243 u16 subsys_device; 1244 u8 revision; 1245 }; 1246 1247 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1248 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1249 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1250 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1251 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1252 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1253 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1254 { 0, 0, 0, 0, 0 }, 1255 }; 1256 1257 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1258 { 1259 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1260 1261 while (p && p->chip_device != 0) { 1262 if (pdev->vendor == p->chip_vendor && 1263 pdev->device == p->chip_device && 1264 pdev->subsystem_vendor == p->subsys_vendor && 1265 pdev->subsystem_device == p->subsys_device && 1266 pdev->revision == p->revision) { 1267 return true; 1268 } 1269 ++p; 1270 } 1271 return false; 1272 } 1273 1274 static bool is_raven_kicker(struct amdgpu_device *adev) 1275 { 1276 if (adev->pm.fw_version >= 0x41e2b) 1277 return true; 1278 else 1279 return false; 1280 } 1281 1282 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1283 { 1284 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1285 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1286 1287 switch (adev->asic_type) { 1288 case CHIP_VEGA10: 1289 case CHIP_VEGA12: 1290 case CHIP_VEGA20: 1291 break; 1292 case CHIP_RAVEN: 1293 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1294 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1295 ((!is_raven_kicker(adev) && 1296 adev->gfx.rlc_fw_version < 531) || 1297 (adev->gfx.rlc_feature_version < 1) || 1298 !adev->gfx.rlc.is_rlc_v2_1)) 1299 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1300 1301 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1302 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1303 AMD_PG_SUPPORT_CP | 1304 AMD_PG_SUPPORT_RLC_SMU_HS; 1305 break; 1306 case CHIP_RENOIR: 1307 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1308 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1309 AMD_PG_SUPPORT_CP | 1310 AMD_PG_SUPPORT_RLC_SMU_HS; 1311 break; 1312 default: 1313 break; 1314 } 1315 } 1316 1317 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1318 const char *chip_name) 1319 { 1320 char fw_name[30]; 1321 int err; 1322 struct amdgpu_firmware_info *info = NULL; 1323 const struct common_firmware_header *header = NULL; 1324 const struct gfx_firmware_header_v1_0 *cp_hdr; 1325 1326 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1327 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1328 if (err) 1329 goto out; 1330 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1331 if (err) 1332 goto out; 1333 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1334 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1335 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1336 1337 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1338 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1339 if (err) 1340 goto out; 1341 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1342 if (err) 1343 goto out; 1344 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1345 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1346 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1347 1348 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1349 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1350 if (err) 1351 goto out; 1352 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1353 if (err) 1354 goto out; 1355 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1356 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1357 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1358 1359 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1360 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1361 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1362 info->fw = adev->gfx.pfp_fw; 1363 header = (const struct common_firmware_header *)info->fw->data; 1364 adev->firmware.fw_size += 1365 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1366 1367 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1368 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1369 info->fw = adev->gfx.me_fw; 1370 header = (const struct common_firmware_header *)info->fw->data; 1371 adev->firmware.fw_size += 1372 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1373 1374 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1375 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1376 info->fw = adev->gfx.ce_fw; 1377 header = (const struct common_firmware_header *)info->fw->data; 1378 adev->firmware.fw_size += 1379 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1380 } 1381 1382 out: 1383 if (err) { 1384 dev_err(adev->dev, 1385 "gfx9: Failed to load firmware \"%s\"\n", 1386 fw_name); 1387 release_firmware(adev->gfx.pfp_fw); 1388 adev->gfx.pfp_fw = NULL; 1389 release_firmware(adev->gfx.me_fw); 1390 adev->gfx.me_fw = NULL; 1391 release_firmware(adev->gfx.ce_fw); 1392 adev->gfx.ce_fw = NULL; 1393 } 1394 return err; 1395 } 1396 1397 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1398 const char *chip_name) 1399 { 1400 char fw_name[30]; 1401 int err; 1402 struct amdgpu_firmware_info *info = NULL; 1403 const struct common_firmware_header *header = NULL; 1404 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1405 unsigned int *tmp = NULL; 1406 unsigned int i = 0; 1407 uint16_t version_major; 1408 uint16_t version_minor; 1409 uint32_t smu_version; 1410 1411 /* 1412 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1413 * instead of picasso_rlc.bin. 1414 * Judgment method: 1415 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1416 * or revision >= 0xD8 && revision <= 0xDF 1417 * otherwise is PCO FP5 1418 */ 1419 if (!strcmp(chip_name, "picasso") && 1420 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1421 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1422 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1423 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1424 (smu_version >= 0x41e2b)) 1425 /** 1426 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1427 */ 1428 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1429 else 1430 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1431 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1432 if (err) 1433 goto out; 1434 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1435 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1436 1437 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1438 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1439 if (version_major == 2 && version_minor == 1) 1440 adev->gfx.rlc.is_rlc_v2_1 = true; 1441 1442 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1443 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1444 adev->gfx.rlc.save_and_restore_offset = 1445 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1446 adev->gfx.rlc.clear_state_descriptor_offset = 1447 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1448 adev->gfx.rlc.avail_scratch_ram_locations = 1449 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1450 adev->gfx.rlc.reg_restore_list_size = 1451 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1452 adev->gfx.rlc.reg_list_format_start = 1453 le32_to_cpu(rlc_hdr->reg_list_format_start); 1454 adev->gfx.rlc.reg_list_format_separate_start = 1455 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1456 adev->gfx.rlc.starting_offsets_start = 1457 le32_to_cpu(rlc_hdr->starting_offsets_start); 1458 adev->gfx.rlc.reg_list_format_size_bytes = 1459 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1460 adev->gfx.rlc.reg_list_size_bytes = 1461 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1462 adev->gfx.rlc.register_list_format = 1463 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1464 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1465 if (!adev->gfx.rlc.register_list_format) { 1466 err = -ENOMEM; 1467 goto out; 1468 } 1469 1470 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1471 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1472 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1473 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1474 1475 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1476 1477 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1478 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1479 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1480 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1481 1482 if (adev->gfx.rlc.is_rlc_v2_1) 1483 gfx_v9_0_init_rlc_ext_microcode(adev); 1484 1485 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1486 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1487 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1488 info->fw = adev->gfx.rlc_fw; 1489 header = (const struct common_firmware_header *)info->fw->data; 1490 adev->firmware.fw_size += 1491 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1492 1493 if (adev->gfx.rlc.is_rlc_v2_1 && 1494 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1495 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1496 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1497 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1498 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1499 info->fw = adev->gfx.rlc_fw; 1500 adev->firmware.fw_size += 1501 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1502 1503 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1504 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1505 info->fw = adev->gfx.rlc_fw; 1506 adev->firmware.fw_size += 1507 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1508 1509 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1510 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1511 info->fw = adev->gfx.rlc_fw; 1512 adev->firmware.fw_size += 1513 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1514 } 1515 } 1516 1517 out: 1518 if (err) { 1519 dev_err(adev->dev, 1520 "gfx9: Failed to load firmware \"%s\"\n", 1521 fw_name); 1522 release_firmware(adev->gfx.rlc_fw); 1523 adev->gfx.rlc_fw = NULL; 1524 } 1525 return err; 1526 } 1527 1528 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1529 { 1530 if (adev->asic_type == CHIP_ALDEBARAN || 1531 adev->asic_type == CHIP_ARCTURUS || 1532 adev->asic_type == CHIP_RENOIR) 1533 return false; 1534 1535 return true; 1536 } 1537 1538 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1539 const char *chip_name) 1540 { 1541 char fw_name[30]; 1542 int err; 1543 struct amdgpu_firmware_info *info = NULL; 1544 const struct common_firmware_header *header = NULL; 1545 const struct gfx_firmware_header_v1_0 *cp_hdr; 1546 1547 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1548 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1549 if (err) 1550 goto out; 1551 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1552 if (err) 1553 goto out; 1554 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1555 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1556 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1557 1558 1559 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1560 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1561 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1562 if (!err) { 1563 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1564 if (err) 1565 goto out; 1566 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1567 adev->gfx.mec2_fw->data; 1568 adev->gfx.mec2_fw_version = 1569 le32_to_cpu(cp_hdr->header.ucode_version); 1570 adev->gfx.mec2_feature_version = 1571 le32_to_cpu(cp_hdr->ucode_feature_version); 1572 } else { 1573 err = 0; 1574 adev->gfx.mec2_fw = NULL; 1575 } 1576 } 1577 1578 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1579 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1580 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1581 info->fw = adev->gfx.mec_fw; 1582 header = (const struct common_firmware_header *)info->fw->data; 1583 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1584 adev->firmware.fw_size += 1585 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1586 1587 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1588 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1589 info->fw = adev->gfx.mec_fw; 1590 adev->firmware.fw_size += 1591 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1592 1593 if (adev->gfx.mec2_fw) { 1594 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1595 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1596 info->fw = adev->gfx.mec2_fw; 1597 header = (const struct common_firmware_header *)info->fw->data; 1598 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1599 adev->firmware.fw_size += 1600 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1601 1602 /* TODO: Determine if MEC2 JT FW loading can be removed 1603 for all GFX V9 asic and above */ 1604 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1605 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1606 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1607 info->fw = adev->gfx.mec2_fw; 1608 adev->firmware.fw_size += 1609 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1610 PAGE_SIZE); 1611 } 1612 } 1613 } 1614 1615 out: 1616 gfx_v9_0_check_if_need_gfxoff(adev); 1617 gfx_v9_0_check_fw_write_wait(adev); 1618 if (err) { 1619 dev_err(adev->dev, 1620 "gfx9: Failed to load firmware \"%s\"\n", 1621 fw_name); 1622 release_firmware(adev->gfx.mec_fw); 1623 adev->gfx.mec_fw = NULL; 1624 release_firmware(adev->gfx.mec2_fw); 1625 adev->gfx.mec2_fw = NULL; 1626 } 1627 return err; 1628 } 1629 1630 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1631 { 1632 const char *chip_name; 1633 int r; 1634 1635 DRM_DEBUG("\n"); 1636 1637 switch (adev->asic_type) { 1638 case CHIP_VEGA10: 1639 chip_name = "vega10"; 1640 break; 1641 case CHIP_VEGA12: 1642 chip_name = "vega12"; 1643 break; 1644 case CHIP_VEGA20: 1645 chip_name = "vega20"; 1646 break; 1647 case CHIP_RAVEN: 1648 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1649 chip_name = "raven2"; 1650 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1651 chip_name = "picasso"; 1652 else 1653 chip_name = "raven"; 1654 break; 1655 case CHIP_ARCTURUS: 1656 chip_name = "arcturus"; 1657 break; 1658 case CHIP_RENOIR: 1659 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1660 chip_name = "renoir"; 1661 else 1662 chip_name = "green_sardine"; 1663 break; 1664 case CHIP_ALDEBARAN: 1665 chip_name = "aldebaran"; 1666 break; 1667 default: 1668 BUG(); 1669 } 1670 1671 /* No CPG in Arcturus */ 1672 if (adev->gfx.num_gfx_rings) { 1673 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1674 if (r) 1675 return r; 1676 } 1677 1678 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1679 if (r) 1680 return r; 1681 1682 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1683 if (r) 1684 return r; 1685 1686 return r; 1687 } 1688 1689 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1690 { 1691 u32 count = 0; 1692 const struct cs_section_def *sect = NULL; 1693 const struct cs_extent_def *ext = NULL; 1694 1695 /* begin clear state */ 1696 count += 2; 1697 /* context control state */ 1698 count += 3; 1699 1700 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1701 for (ext = sect->section; ext->extent != NULL; ++ext) { 1702 if (sect->id == SECT_CONTEXT) 1703 count += 2 + ext->reg_count; 1704 else 1705 return 0; 1706 } 1707 } 1708 1709 /* end clear state */ 1710 count += 2; 1711 /* clear state */ 1712 count += 2; 1713 1714 return count; 1715 } 1716 1717 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1718 volatile u32 *buffer) 1719 { 1720 u32 count = 0, i; 1721 const struct cs_section_def *sect = NULL; 1722 const struct cs_extent_def *ext = NULL; 1723 1724 if (adev->gfx.rlc.cs_data == NULL) 1725 return; 1726 if (buffer == NULL) 1727 return; 1728 1729 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1730 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1731 1732 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1733 buffer[count++] = cpu_to_le32(0x80000000); 1734 buffer[count++] = cpu_to_le32(0x80000000); 1735 1736 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1737 for (ext = sect->section; ext->extent != NULL; ++ext) { 1738 if (sect->id == SECT_CONTEXT) { 1739 buffer[count++] = 1740 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1741 buffer[count++] = cpu_to_le32(ext->reg_index - 1742 PACKET3_SET_CONTEXT_REG_START); 1743 for (i = 0; i < ext->reg_count; i++) 1744 buffer[count++] = cpu_to_le32(ext->extent[i]); 1745 } else { 1746 return; 1747 } 1748 } 1749 } 1750 1751 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1752 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1753 1754 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1755 buffer[count++] = cpu_to_le32(0); 1756 } 1757 1758 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1759 { 1760 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1761 uint32_t pg_always_on_cu_num = 2; 1762 uint32_t always_on_cu_num; 1763 uint32_t i, j, k; 1764 uint32_t mask, cu_bitmap, counter; 1765 1766 if (adev->flags & AMD_IS_APU) 1767 always_on_cu_num = 4; 1768 else if (adev->asic_type == CHIP_VEGA12) 1769 always_on_cu_num = 8; 1770 else 1771 always_on_cu_num = 12; 1772 1773 mutex_lock(&adev->grbm_idx_mutex); 1774 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1775 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1776 mask = 1; 1777 cu_bitmap = 0; 1778 counter = 0; 1779 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1780 1781 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1782 if (cu_info->bitmap[i][j] & mask) { 1783 if (counter == pg_always_on_cu_num) 1784 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1785 if (counter < always_on_cu_num) 1786 cu_bitmap |= mask; 1787 else 1788 break; 1789 counter++; 1790 } 1791 mask <<= 1; 1792 } 1793 1794 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1795 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1796 } 1797 } 1798 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1799 mutex_unlock(&adev->grbm_idx_mutex); 1800 } 1801 1802 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1803 { 1804 uint32_t data; 1805 1806 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1807 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1808 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1809 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1810 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1811 1812 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1813 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1814 1815 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1816 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1817 1818 mutex_lock(&adev->grbm_idx_mutex); 1819 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1820 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1821 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1822 1823 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1824 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1825 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1826 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1827 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1828 1829 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1830 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1831 data &= 0x0000FFFF; 1832 data |= 0x00C00000; 1833 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1834 1835 /* 1836 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1837 * programmed in gfx_v9_0_init_always_on_cu_mask() 1838 */ 1839 1840 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1841 * but used for RLC_LB_CNTL configuration */ 1842 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1843 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1844 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1845 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1846 mutex_unlock(&adev->grbm_idx_mutex); 1847 1848 gfx_v9_0_init_always_on_cu_mask(adev); 1849 } 1850 1851 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1852 { 1853 uint32_t data; 1854 1855 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1856 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1857 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1858 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1859 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1860 1861 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1862 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1863 1864 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1865 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1866 1867 mutex_lock(&adev->grbm_idx_mutex); 1868 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1869 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1870 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1871 1872 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1873 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1874 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1875 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1876 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1877 1878 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1879 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1880 data &= 0x0000FFFF; 1881 data |= 0x00C00000; 1882 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1883 1884 /* 1885 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1886 * programmed in gfx_v9_0_init_always_on_cu_mask() 1887 */ 1888 1889 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1890 * but used for RLC_LB_CNTL configuration */ 1891 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1892 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1893 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1894 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1895 mutex_unlock(&adev->grbm_idx_mutex); 1896 1897 gfx_v9_0_init_always_on_cu_mask(adev); 1898 } 1899 1900 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1901 { 1902 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1903 } 1904 1905 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1906 { 1907 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1908 return 5; 1909 else 1910 return 4; 1911 } 1912 1913 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1914 { 1915 const struct cs_section_def *cs_data; 1916 int r; 1917 1918 adev->gfx.rlc.cs_data = gfx9_cs_data; 1919 1920 cs_data = adev->gfx.rlc.cs_data; 1921 1922 if (cs_data) { 1923 /* init clear state block */ 1924 r = amdgpu_gfx_rlc_init_csb(adev); 1925 if (r) 1926 return r; 1927 } 1928 1929 if (adev->flags & AMD_IS_APU) { 1930 /* TODO: double check the cp_table_size for RV */ 1931 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1932 r = amdgpu_gfx_rlc_init_cpt(adev); 1933 if (r) 1934 return r; 1935 } 1936 1937 switch (adev->asic_type) { 1938 case CHIP_RAVEN: 1939 gfx_v9_0_init_lbpw(adev); 1940 break; 1941 case CHIP_VEGA20: 1942 gfx_v9_4_init_lbpw(adev); 1943 break; 1944 default: 1945 break; 1946 } 1947 1948 /* init spm vmid with 0xf */ 1949 if (adev->gfx.rlc.funcs->update_spm_vmid) 1950 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1951 1952 return 0; 1953 } 1954 1955 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1956 { 1957 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1958 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1959 } 1960 1961 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1962 { 1963 int r; 1964 u32 *hpd; 1965 const __le32 *fw_data; 1966 unsigned fw_size; 1967 u32 *fw; 1968 size_t mec_hpd_size; 1969 1970 const struct gfx_firmware_header_v1_0 *mec_hdr; 1971 1972 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1973 1974 /* take ownership of the relevant compute queues */ 1975 amdgpu_gfx_compute_queue_acquire(adev); 1976 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1977 if (mec_hpd_size) { 1978 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1979 AMDGPU_GEM_DOMAIN_VRAM, 1980 &adev->gfx.mec.hpd_eop_obj, 1981 &adev->gfx.mec.hpd_eop_gpu_addr, 1982 (void **)&hpd); 1983 if (r) { 1984 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1985 gfx_v9_0_mec_fini(adev); 1986 return r; 1987 } 1988 1989 memset(hpd, 0, mec_hpd_size); 1990 1991 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1992 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1993 } 1994 1995 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1996 1997 fw_data = (const __le32 *) 1998 (adev->gfx.mec_fw->data + 1999 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 2000 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 2001 2002 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 2003 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 2004 &adev->gfx.mec.mec_fw_obj, 2005 &adev->gfx.mec.mec_fw_gpu_addr, 2006 (void **)&fw); 2007 if (r) { 2008 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 2009 gfx_v9_0_mec_fini(adev); 2010 return r; 2011 } 2012 2013 memcpy(fw, fw_data, fw_size); 2014 2015 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2016 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2017 2018 return 0; 2019 } 2020 2021 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 2022 { 2023 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2024 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2025 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2026 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2027 (SQ_IND_INDEX__FORCE_READ_MASK)); 2028 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2029 } 2030 2031 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2032 uint32_t wave, uint32_t thread, 2033 uint32_t regno, uint32_t num, uint32_t *out) 2034 { 2035 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2036 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2037 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2038 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2039 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2040 (SQ_IND_INDEX__FORCE_READ_MASK) | 2041 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2042 while (num--) 2043 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2044 } 2045 2046 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2047 { 2048 /* type 1 wave data */ 2049 dst[(*no_fields)++] = 1; 2050 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2051 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2052 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2053 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2054 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2055 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2056 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2057 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2058 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2059 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2060 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2061 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2062 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2063 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2064 } 2065 2066 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2067 uint32_t wave, uint32_t start, 2068 uint32_t size, uint32_t *dst) 2069 { 2070 wave_read_regs( 2071 adev, simd, wave, 0, 2072 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2073 } 2074 2075 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2076 uint32_t wave, uint32_t thread, 2077 uint32_t start, uint32_t size, 2078 uint32_t *dst) 2079 { 2080 wave_read_regs( 2081 adev, simd, wave, thread, 2082 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2083 } 2084 2085 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2086 u32 me, u32 pipe, u32 q, u32 vm) 2087 { 2088 soc15_grbm_select(adev, me, pipe, q, vm); 2089 } 2090 2091 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2092 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2093 .select_se_sh = &gfx_v9_0_select_se_sh, 2094 .read_wave_data = &gfx_v9_0_read_wave_data, 2095 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2096 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2097 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2098 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2099 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2100 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2101 }; 2102 2103 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { 2104 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2105 .select_se_sh = &gfx_v9_0_select_se_sh, 2106 .read_wave_data = &gfx_v9_0_read_wave_data, 2107 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2108 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2109 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2110 .ras_error_inject = &gfx_v9_4_ras_error_inject, 2111 .query_ras_error_count = &gfx_v9_4_query_ras_error_count, 2112 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count, 2113 .query_ras_error_status = &gfx_v9_4_query_ras_error_status, 2114 }; 2115 2116 static const struct amdgpu_gfx_funcs gfx_v9_4_2_gfx_funcs = { 2117 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2118 .select_se_sh = &gfx_v9_0_select_se_sh, 2119 .read_wave_data = &gfx_v9_0_read_wave_data, 2120 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2121 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2122 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2123 .ras_error_inject = &gfx_v9_4_2_ras_error_inject, 2124 .query_ras_error_count = &gfx_v9_4_2_query_ras_error_count, 2125 .reset_ras_error_count = &gfx_v9_4_2_reset_ras_error_count, 2126 .query_ras_error_status = &gfx_v9_4_2_query_ras_error_status, 2127 .reset_ras_error_status = &gfx_v9_4_2_reset_ras_error_status, 2128 .enable_watchdog_timer = &gfx_v9_4_2_enable_watchdog_timer, 2129 }; 2130 2131 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2132 { 2133 u32 gb_addr_config; 2134 int err; 2135 2136 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2137 2138 switch (adev->asic_type) { 2139 case CHIP_VEGA10: 2140 adev->gfx.config.max_hw_contexts = 8; 2141 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2142 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2143 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2144 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2145 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2146 break; 2147 case CHIP_VEGA12: 2148 adev->gfx.config.max_hw_contexts = 8; 2149 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2150 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2151 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2152 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2153 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2154 DRM_INFO("fix gfx.config for vega12\n"); 2155 break; 2156 case CHIP_VEGA20: 2157 adev->gfx.config.max_hw_contexts = 8; 2158 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2159 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2160 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2161 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2162 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2163 gb_addr_config &= ~0xf3e777ff; 2164 gb_addr_config |= 0x22014042; 2165 /* check vbios table if gpu info is not available */ 2166 err = amdgpu_atomfirmware_get_gfx_info(adev); 2167 if (err) 2168 return err; 2169 break; 2170 case CHIP_RAVEN: 2171 adev->gfx.config.max_hw_contexts = 8; 2172 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2173 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2174 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2175 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2176 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2177 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2178 else 2179 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2180 break; 2181 case CHIP_ARCTURUS: 2182 adev->gfx.funcs = &gfx_v9_4_gfx_funcs; 2183 adev->gfx.config.max_hw_contexts = 8; 2184 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2185 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2186 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2187 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2188 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2189 gb_addr_config &= ~0xf3e777ff; 2190 gb_addr_config |= 0x22014042; 2191 break; 2192 case CHIP_RENOIR: 2193 adev->gfx.config.max_hw_contexts = 8; 2194 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2195 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2196 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2197 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2198 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2199 gb_addr_config &= ~0xf3e777ff; 2200 gb_addr_config |= 0x22010042; 2201 break; 2202 case CHIP_ALDEBARAN: 2203 adev->gfx.funcs = &gfx_v9_4_2_gfx_funcs; 2204 adev->gfx.config.max_hw_contexts = 8; 2205 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2206 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2207 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2208 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2209 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2210 gb_addr_config &= ~0xf3e777ff; 2211 gb_addr_config |= 0x22014042; 2212 /* check vbios table if gpu info is not available */ 2213 err = amdgpu_atomfirmware_get_gfx_info(adev); 2214 if (err) 2215 return err; 2216 break; 2217 default: 2218 BUG(); 2219 break; 2220 } 2221 2222 adev->gfx.config.gb_addr_config = gb_addr_config; 2223 2224 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2225 REG_GET_FIELD( 2226 adev->gfx.config.gb_addr_config, 2227 GB_ADDR_CONFIG, 2228 NUM_PIPES); 2229 2230 adev->gfx.config.max_tile_pipes = 2231 adev->gfx.config.gb_addr_config_fields.num_pipes; 2232 2233 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2234 REG_GET_FIELD( 2235 adev->gfx.config.gb_addr_config, 2236 GB_ADDR_CONFIG, 2237 NUM_BANKS); 2238 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2239 REG_GET_FIELD( 2240 adev->gfx.config.gb_addr_config, 2241 GB_ADDR_CONFIG, 2242 MAX_COMPRESSED_FRAGS); 2243 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2244 REG_GET_FIELD( 2245 adev->gfx.config.gb_addr_config, 2246 GB_ADDR_CONFIG, 2247 NUM_RB_PER_SE); 2248 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2249 REG_GET_FIELD( 2250 adev->gfx.config.gb_addr_config, 2251 GB_ADDR_CONFIG, 2252 NUM_SHADER_ENGINES); 2253 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2254 REG_GET_FIELD( 2255 adev->gfx.config.gb_addr_config, 2256 GB_ADDR_CONFIG, 2257 PIPE_INTERLEAVE_SIZE)); 2258 2259 return 0; 2260 } 2261 2262 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2263 int mec, int pipe, int queue) 2264 { 2265 unsigned irq_type; 2266 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2267 unsigned int hw_prio; 2268 2269 ring = &adev->gfx.compute_ring[ring_id]; 2270 2271 /* mec0 is me1 */ 2272 ring->me = mec + 1; 2273 ring->pipe = pipe; 2274 ring->queue = queue; 2275 2276 ring->ring_obj = NULL; 2277 ring->use_doorbell = true; 2278 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2279 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2280 + (ring_id * GFX9_MEC_HPD_SIZE); 2281 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2282 2283 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2284 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2285 + ring->pipe; 2286 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2287 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2288 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2289 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 2290 hw_prio, NULL); 2291 } 2292 2293 static int gfx_v9_0_sw_init(void *handle) 2294 { 2295 int i, j, k, r, ring_id; 2296 struct amdgpu_ring *ring; 2297 struct amdgpu_kiq *kiq; 2298 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2299 2300 switch (adev->asic_type) { 2301 case CHIP_VEGA10: 2302 case CHIP_VEGA12: 2303 case CHIP_VEGA20: 2304 case CHIP_RAVEN: 2305 case CHIP_ARCTURUS: 2306 case CHIP_RENOIR: 2307 case CHIP_ALDEBARAN: 2308 adev->gfx.mec.num_mec = 2; 2309 break; 2310 default: 2311 adev->gfx.mec.num_mec = 1; 2312 break; 2313 } 2314 2315 adev->gfx.mec.num_pipe_per_mec = 4; 2316 adev->gfx.mec.num_queue_per_pipe = 8; 2317 2318 /* EOP Event */ 2319 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2320 if (r) 2321 return r; 2322 2323 /* Privileged reg */ 2324 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2325 &adev->gfx.priv_reg_irq); 2326 if (r) 2327 return r; 2328 2329 /* Privileged inst */ 2330 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2331 &adev->gfx.priv_inst_irq); 2332 if (r) 2333 return r; 2334 2335 /* ECC error */ 2336 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2337 &adev->gfx.cp_ecc_error_irq); 2338 if (r) 2339 return r; 2340 2341 /* FUE error */ 2342 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2343 &adev->gfx.cp_ecc_error_irq); 2344 if (r) 2345 return r; 2346 2347 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2348 2349 gfx_v9_0_scratch_init(adev); 2350 2351 r = gfx_v9_0_init_microcode(adev); 2352 if (r) { 2353 DRM_ERROR("Failed to load gfx firmware!\n"); 2354 return r; 2355 } 2356 2357 r = adev->gfx.rlc.funcs->init(adev); 2358 if (r) { 2359 DRM_ERROR("Failed to init rlc BOs!\n"); 2360 return r; 2361 } 2362 2363 r = gfx_v9_0_mec_init(adev); 2364 if (r) { 2365 DRM_ERROR("Failed to init MEC BOs!\n"); 2366 return r; 2367 } 2368 2369 /* set up the gfx ring */ 2370 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2371 ring = &adev->gfx.gfx_ring[i]; 2372 ring->ring_obj = NULL; 2373 if (!i) 2374 sprintf(ring->name, "gfx"); 2375 else 2376 sprintf(ring->name, "gfx_%d", i); 2377 ring->use_doorbell = true; 2378 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2379 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2380 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2381 AMDGPU_RING_PRIO_DEFAULT, NULL); 2382 if (r) 2383 return r; 2384 } 2385 2386 /* set up the compute queues - allocate horizontally across pipes */ 2387 ring_id = 0; 2388 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2389 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2390 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2391 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2392 continue; 2393 2394 r = gfx_v9_0_compute_ring_init(adev, 2395 ring_id, 2396 i, k, j); 2397 if (r) 2398 return r; 2399 2400 ring_id++; 2401 } 2402 } 2403 } 2404 2405 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2406 if (r) { 2407 DRM_ERROR("Failed to init KIQ BOs!\n"); 2408 return r; 2409 } 2410 2411 kiq = &adev->gfx.kiq; 2412 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2413 if (r) 2414 return r; 2415 2416 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2417 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2418 if (r) 2419 return r; 2420 2421 adev->gfx.ce_ram_size = 0x8000; 2422 2423 r = gfx_v9_0_gpu_early_init(adev); 2424 if (r) 2425 return r; 2426 2427 return 0; 2428 } 2429 2430 2431 static int gfx_v9_0_sw_fini(void *handle) 2432 { 2433 int i; 2434 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2435 2436 amdgpu_gfx_ras_fini(adev); 2437 2438 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2439 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2440 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2441 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2442 2443 amdgpu_gfx_mqd_sw_fini(adev); 2444 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2445 amdgpu_gfx_kiq_fini(adev); 2446 2447 gfx_v9_0_mec_fini(adev); 2448 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2449 if (adev->flags & AMD_IS_APU) { 2450 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2451 &adev->gfx.rlc.cp_table_gpu_addr, 2452 (void **)&adev->gfx.rlc.cp_table_ptr); 2453 } 2454 gfx_v9_0_free_microcode(adev); 2455 2456 return 0; 2457 } 2458 2459 2460 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2461 { 2462 /* TODO */ 2463 } 2464 2465 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2466 u32 instance) 2467 { 2468 u32 data; 2469 2470 if (instance == 0xffffffff) 2471 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2472 else 2473 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2474 2475 if (se_num == 0xffffffff) 2476 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2477 else 2478 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2479 2480 if (sh_num == 0xffffffff) 2481 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2482 else 2483 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2484 2485 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2486 } 2487 2488 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2489 { 2490 u32 data, mask; 2491 2492 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2493 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2494 2495 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2496 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2497 2498 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2499 adev->gfx.config.max_sh_per_se); 2500 2501 return (~data) & mask; 2502 } 2503 2504 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2505 { 2506 int i, j; 2507 u32 data; 2508 u32 active_rbs = 0; 2509 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2510 adev->gfx.config.max_sh_per_se; 2511 2512 mutex_lock(&adev->grbm_idx_mutex); 2513 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2514 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2515 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2516 data = gfx_v9_0_get_rb_active_bitmap(adev); 2517 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2518 rb_bitmap_width_per_sh); 2519 } 2520 } 2521 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2522 mutex_unlock(&adev->grbm_idx_mutex); 2523 2524 adev->gfx.config.backend_enable_mask = active_rbs; 2525 adev->gfx.config.num_rbs = hweight32(active_rbs); 2526 } 2527 2528 #define DEFAULT_SH_MEM_BASES (0x6000) 2529 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2530 { 2531 int i; 2532 uint32_t sh_mem_config; 2533 uint32_t sh_mem_bases; 2534 2535 /* 2536 * Configure apertures: 2537 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2538 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2539 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2540 */ 2541 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2542 2543 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2544 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2545 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2546 2547 mutex_lock(&adev->srbm_mutex); 2548 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2549 soc15_grbm_select(adev, 0, 0, 0, i); 2550 /* CP and shaders */ 2551 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2552 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2553 } 2554 soc15_grbm_select(adev, 0, 0, 0, 0); 2555 mutex_unlock(&adev->srbm_mutex); 2556 2557 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2558 acccess. These should be enabled by FW for target VMIDs. */ 2559 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2560 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2561 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2562 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2563 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2564 } 2565 } 2566 2567 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2568 { 2569 int vmid; 2570 2571 /* 2572 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2573 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2574 * the driver can enable them for graphics. VMID0 should maintain 2575 * access so that HWS firmware can save/restore entries. 2576 */ 2577 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2578 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2579 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2580 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2581 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2582 } 2583 } 2584 2585 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2586 { 2587 uint32_t tmp; 2588 2589 switch (adev->asic_type) { 2590 case CHIP_ARCTURUS: 2591 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2592 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2593 DISABLE_BARRIER_WAITCNT, 1); 2594 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2595 break; 2596 default: 2597 break; 2598 } 2599 } 2600 2601 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2602 { 2603 u32 tmp; 2604 int i; 2605 2606 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2607 2608 gfx_v9_0_tiling_mode_table_init(adev); 2609 2610 gfx_v9_0_setup_rb(adev); 2611 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2612 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2613 2614 /* XXX SH_MEM regs */ 2615 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2616 mutex_lock(&adev->srbm_mutex); 2617 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2618 soc15_grbm_select(adev, 0, 0, 0, i); 2619 /* CP and shaders */ 2620 if (i == 0) { 2621 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2622 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2623 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2624 !!adev->gmc.noretry); 2625 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2626 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2627 } else { 2628 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2629 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2630 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2631 !!adev->gmc.noretry); 2632 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2633 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2634 (adev->gmc.private_aperture_start >> 48)); 2635 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2636 (adev->gmc.shared_aperture_start >> 48)); 2637 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2638 } 2639 } 2640 soc15_grbm_select(adev, 0, 0, 0, 0); 2641 2642 mutex_unlock(&adev->srbm_mutex); 2643 2644 gfx_v9_0_init_compute_vmid(adev); 2645 gfx_v9_0_init_gds_vmid(adev); 2646 gfx_v9_0_init_sq_config(adev); 2647 } 2648 2649 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2650 { 2651 u32 i, j, k; 2652 u32 mask; 2653 2654 mutex_lock(&adev->grbm_idx_mutex); 2655 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2656 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2657 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2658 for (k = 0; k < adev->usec_timeout; k++) { 2659 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2660 break; 2661 udelay(1); 2662 } 2663 if (k == adev->usec_timeout) { 2664 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2665 0xffffffff, 0xffffffff); 2666 mutex_unlock(&adev->grbm_idx_mutex); 2667 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2668 i, j); 2669 return; 2670 } 2671 } 2672 } 2673 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2674 mutex_unlock(&adev->grbm_idx_mutex); 2675 2676 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2677 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2678 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2679 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2680 for (k = 0; k < adev->usec_timeout; k++) { 2681 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2682 break; 2683 udelay(1); 2684 } 2685 } 2686 2687 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2688 bool enable) 2689 { 2690 u32 tmp; 2691 2692 /* These interrupts should be enabled to drive DS clock */ 2693 2694 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2695 2696 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2697 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2698 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2699 if(adev->gfx.num_gfx_rings) 2700 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2701 2702 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2703 } 2704 2705 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2706 { 2707 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2708 /* csib */ 2709 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2710 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2711 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2712 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2713 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2714 adev->gfx.rlc.clear_state_size); 2715 } 2716 2717 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2718 int indirect_offset, 2719 int list_size, 2720 int *unique_indirect_regs, 2721 int unique_indirect_reg_count, 2722 int *indirect_start_offsets, 2723 int *indirect_start_offsets_count, 2724 int max_start_offsets_count) 2725 { 2726 int idx; 2727 2728 for (; indirect_offset < list_size; indirect_offset++) { 2729 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2730 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2731 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2732 2733 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2734 indirect_offset += 2; 2735 2736 /* look for the matching indice */ 2737 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2738 if (unique_indirect_regs[idx] == 2739 register_list_format[indirect_offset] || 2740 !unique_indirect_regs[idx]) 2741 break; 2742 } 2743 2744 BUG_ON(idx >= unique_indirect_reg_count); 2745 2746 if (!unique_indirect_regs[idx]) 2747 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2748 2749 indirect_offset++; 2750 } 2751 } 2752 } 2753 2754 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2755 { 2756 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2757 int unique_indirect_reg_count = 0; 2758 2759 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2760 int indirect_start_offsets_count = 0; 2761 2762 int list_size = 0; 2763 int i = 0, j = 0; 2764 u32 tmp = 0; 2765 2766 u32 *register_list_format = 2767 kmemdup(adev->gfx.rlc.register_list_format, 2768 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2769 if (!register_list_format) 2770 return -ENOMEM; 2771 2772 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2773 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2774 gfx_v9_1_parse_ind_reg_list(register_list_format, 2775 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2776 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2777 unique_indirect_regs, 2778 unique_indirect_reg_count, 2779 indirect_start_offsets, 2780 &indirect_start_offsets_count, 2781 ARRAY_SIZE(indirect_start_offsets)); 2782 2783 /* enable auto inc in case it is disabled */ 2784 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2785 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2786 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2787 2788 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2789 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2790 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2791 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2792 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2793 adev->gfx.rlc.register_restore[i]); 2794 2795 /* load indirect register */ 2796 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2797 adev->gfx.rlc.reg_list_format_start); 2798 2799 /* direct register portion */ 2800 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2801 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2802 register_list_format[i]); 2803 2804 /* indirect register portion */ 2805 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2806 if (register_list_format[i] == 0xFFFFFFFF) { 2807 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2808 continue; 2809 } 2810 2811 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2812 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2813 2814 for (j = 0; j < unique_indirect_reg_count; j++) { 2815 if (register_list_format[i] == unique_indirect_regs[j]) { 2816 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2817 break; 2818 } 2819 } 2820 2821 BUG_ON(j >= unique_indirect_reg_count); 2822 2823 i++; 2824 } 2825 2826 /* set save/restore list size */ 2827 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2828 list_size = list_size >> 1; 2829 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2830 adev->gfx.rlc.reg_restore_list_size); 2831 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2832 2833 /* write the starting offsets to RLC scratch ram */ 2834 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2835 adev->gfx.rlc.starting_offsets_start); 2836 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2837 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2838 indirect_start_offsets[i]); 2839 2840 /* load unique indirect regs*/ 2841 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2842 if (unique_indirect_regs[i] != 0) { 2843 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2844 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2845 unique_indirect_regs[i] & 0x3FFFF); 2846 2847 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2848 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2849 unique_indirect_regs[i] >> 20); 2850 } 2851 } 2852 2853 kfree(register_list_format); 2854 return 0; 2855 } 2856 2857 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2858 { 2859 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2860 } 2861 2862 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2863 bool enable) 2864 { 2865 uint32_t data = 0; 2866 uint32_t default_data = 0; 2867 2868 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2869 if (enable) { 2870 /* enable GFXIP control over CGPG */ 2871 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2872 if(default_data != data) 2873 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2874 2875 /* update status */ 2876 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2877 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2878 if(default_data != data) 2879 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2880 } else { 2881 /* restore GFXIP control over GCPG */ 2882 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2883 if(default_data != data) 2884 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2885 } 2886 } 2887 2888 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2889 { 2890 uint32_t data = 0; 2891 2892 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2893 AMD_PG_SUPPORT_GFX_SMG | 2894 AMD_PG_SUPPORT_GFX_DMG)) { 2895 /* init IDLE_POLL_COUNT = 60 */ 2896 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2897 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2898 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2899 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2900 2901 /* init RLC PG Delay */ 2902 data = 0; 2903 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2904 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2905 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2906 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2907 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2908 2909 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2910 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2911 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2912 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2913 2914 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2915 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2916 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2917 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2918 2919 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2920 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2921 2922 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2923 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2924 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2925 if (adev->asic_type != CHIP_RENOIR) 2926 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2927 } 2928 } 2929 2930 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2931 bool enable) 2932 { 2933 uint32_t data = 0; 2934 uint32_t default_data = 0; 2935 2936 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2937 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2938 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2939 enable ? 1 : 0); 2940 if (default_data != data) 2941 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2942 } 2943 2944 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2945 bool enable) 2946 { 2947 uint32_t data = 0; 2948 uint32_t default_data = 0; 2949 2950 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2951 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2952 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2953 enable ? 1 : 0); 2954 if(default_data != data) 2955 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2956 } 2957 2958 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2959 bool enable) 2960 { 2961 uint32_t data = 0; 2962 uint32_t default_data = 0; 2963 2964 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2965 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2966 CP_PG_DISABLE, 2967 enable ? 0 : 1); 2968 if(default_data != data) 2969 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2970 } 2971 2972 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2973 bool enable) 2974 { 2975 uint32_t data, default_data; 2976 2977 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2978 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2979 GFX_POWER_GATING_ENABLE, 2980 enable ? 1 : 0); 2981 if(default_data != data) 2982 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2983 } 2984 2985 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2986 bool enable) 2987 { 2988 uint32_t data, default_data; 2989 2990 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2991 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2992 GFX_PIPELINE_PG_ENABLE, 2993 enable ? 1 : 0); 2994 if(default_data != data) 2995 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2996 2997 if (!enable) 2998 /* read any GFX register to wake up GFX */ 2999 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 3000 } 3001 3002 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 3003 bool enable) 3004 { 3005 uint32_t data, default_data; 3006 3007 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3008 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3009 STATIC_PER_CU_PG_ENABLE, 3010 enable ? 1 : 0); 3011 if(default_data != data) 3012 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3013 } 3014 3015 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 3016 bool enable) 3017 { 3018 uint32_t data, default_data; 3019 3020 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3021 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3022 DYN_PER_CU_PG_ENABLE, 3023 enable ? 1 : 0); 3024 if(default_data != data) 3025 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3026 } 3027 3028 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 3029 { 3030 gfx_v9_0_init_csb(adev); 3031 3032 /* 3033 * Rlc save restore list is workable since v2_1. 3034 * And it's needed by gfxoff feature. 3035 */ 3036 if (adev->gfx.rlc.is_rlc_v2_1) { 3037 if (adev->asic_type == CHIP_VEGA12 || 3038 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 3039 gfx_v9_1_init_rlc_save_restore_list(adev); 3040 gfx_v9_0_enable_save_restore_machine(adev); 3041 } 3042 3043 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 3044 AMD_PG_SUPPORT_GFX_SMG | 3045 AMD_PG_SUPPORT_GFX_DMG | 3046 AMD_PG_SUPPORT_CP | 3047 AMD_PG_SUPPORT_GDS | 3048 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3049 WREG32(mmRLC_JUMP_TABLE_RESTORE, 3050 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3051 gfx_v9_0_init_gfx_power_gating(adev); 3052 } 3053 } 3054 3055 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3056 { 3057 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3058 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3059 gfx_v9_0_wait_for_rlc_serdes(adev); 3060 } 3061 3062 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3063 { 3064 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3065 udelay(50); 3066 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3067 udelay(50); 3068 } 3069 3070 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3071 { 3072 #ifdef AMDGPU_RLC_DEBUG_RETRY 3073 u32 rlc_ucode_ver; 3074 #endif 3075 3076 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3077 udelay(50); 3078 3079 /* carrizo do enable cp interrupt after cp inited */ 3080 if (!(adev->flags & AMD_IS_APU)) { 3081 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3082 udelay(50); 3083 } 3084 3085 #ifdef AMDGPU_RLC_DEBUG_RETRY 3086 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3087 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3088 if(rlc_ucode_ver == 0x108) { 3089 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3090 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3091 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3092 * default is 0x9C4 to create a 100us interval */ 3093 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3094 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3095 * to disable the page fault retry interrupts, default is 3096 * 0x100 (256) */ 3097 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3098 } 3099 #endif 3100 } 3101 3102 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3103 { 3104 const struct rlc_firmware_header_v2_0 *hdr; 3105 const __le32 *fw_data; 3106 unsigned i, fw_size; 3107 3108 if (!adev->gfx.rlc_fw) 3109 return -EINVAL; 3110 3111 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3112 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3113 3114 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3115 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3116 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3117 3118 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3119 RLCG_UCODE_LOADING_START_ADDRESS); 3120 for (i = 0; i < fw_size; i++) 3121 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3122 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3123 3124 return 0; 3125 } 3126 3127 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3128 { 3129 int r; 3130 3131 if (amdgpu_sriov_vf(adev)) { 3132 gfx_v9_0_init_csb(adev); 3133 return 0; 3134 } 3135 3136 adev->gfx.rlc.funcs->stop(adev); 3137 3138 /* disable CG */ 3139 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3140 3141 gfx_v9_0_init_pg(adev); 3142 3143 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3144 /* legacy rlc firmware loading */ 3145 r = gfx_v9_0_rlc_load_microcode(adev); 3146 if (r) 3147 return r; 3148 } 3149 3150 switch (adev->asic_type) { 3151 case CHIP_RAVEN: 3152 if (amdgpu_lbpw == 0) 3153 gfx_v9_0_enable_lbpw(adev, false); 3154 else 3155 gfx_v9_0_enable_lbpw(adev, true); 3156 break; 3157 case CHIP_VEGA20: 3158 if (amdgpu_lbpw > 0) 3159 gfx_v9_0_enable_lbpw(adev, true); 3160 else 3161 gfx_v9_0_enable_lbpw(adev, false); 3162 break; 3163 default: 3164 break; 3165 } 3166 3167 adev->gfx.rlc.funcs->start(adev); 3168 3169 return 0; 3170 } 3171 3172 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3173 { 3174 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3175 3176 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3177 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3178 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3179 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3180 udelay(50); 3181 } 3182 3183 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3184 { 3185 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3186 const struct gfx_firmware_header_v1_0 *ce_hdr; 3187 const struct gfx_firmware_header_v1_0 *me_hdr; 3188 const __le32 *fw_data; 3189 unsigned i, fw_size; 3190 3191 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3192 return -EINVAL; 3193 3194 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3195 adev->gfx.pfp_fw->data; 3196 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3197 adev->gfx.ce_fw->data; 3198 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3199 adev->gfx.me_fw->data; 3200 3201 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3202 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3203 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3204 3205 gfx_v9_0_cp_gfx_enable(adev, false); 3206 3207 /* PFP */ 3208 fw_data = (const __le32 *) 3209 (adev->gfx.pfp_fw->data + 3210 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3211 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3212 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3213 for (i = 0; i < fw_size; i++) 3214 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3215 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3216 3217 /* CE */ 3218 fw_data = (const __le32 *) 3219 (adev->gfx.ce_fw->data + 3220 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3221 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3222 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3223 for (i = 0; i < fw_size; i++) 3224 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3225 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3226 3227 /* ME */ 3228 fw_data = (const __le32 *) 3229 (adev->gfx.me_fw->data + 3230 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3231 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3232 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3233 for (i = 0; i < fw_size; i++) 3234 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3235 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3236 3237 return 0; 3238 } 3239 3240 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3241 { 3242 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3243 const struct cs_section_def *sect = NULL; 3244 const struct cs_extent_def *ext = NULL; 3245 int r, i, tmp; 3246 3247 /* init the CP */ 3248 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3249 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3250 3251 gfx_v9_0_cp_gfx_enable(adev, true); 3252 3253 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3254 if (r) { 3255 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3256 return r; 3257 } 3258 3259 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3260 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3261 3262 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3263 amdgpu_ring_write(ring, 0x80000000); 3264 amdgpu_ring_write(ring, 0x80000000); 3265 3266 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3267 for (ext = sect->section; ext->extent != NULL; ++ext) { 3268 if (sect->id == SECT_CONTEXT) { 3269 amdgpu_ring_write(ring, 3270 PACKET3(PACKET3_SET_CONTEXT_REG, 3271 ext->reg_count)); 3272 amdgpu_ring_write(ring, 3273 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3274 for (i = 0; i < ext->reg_count; i++) 3275 amdgpu_ring_write(ring, ext->extent[i]); 3276 } 3277 } 3278 } 3279 3280 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3281 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3282 3283 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3284 amdgpu_ring_write(ring, 0); 3285 3286 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3287 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3288 amdgpu_ring_write(ring, 0x8000); 3289 amdgpu_ring_write(ring, 0x8000); 3290 3291 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3292 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3293 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3294 amdgpu_ring_write(ring, tmp); 3295 amdgpu_ring_write(ring, 0); 3296 3297 amdgpu_ring_commit(ring); 3298 3299 return 0; 3300 } 3301 3302 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3303 { 3304 struct amdgpu_ring *ring; 3305 u32 tmp; 3306 u32 rb_bufsz; 3307 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3308 3309 /* Set the write pointer delay */ 3310 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3311 3312 /* set the RB to use vmid 0 */ 3313 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3314 3315 /* Set ring buffer size */ 3316 ring = &adev->gfx.gfx_ring[0]; 3317 rb_bufsz = order_base_2(ring->ring_size / 8); 3318 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3319 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3320 #ifdef __BIG_ENDIAN 3321 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3322 #endif 3323 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3324 3325 /* Initialize the ring buffer's write pointers */ 3326 ring->wptr = 0; 3327 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3328 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3329 3330 /* set the wb address wether it's enabled or not */ 3331 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3332 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3333 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3334 3335 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3336 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3337 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3338 3339 mdelay(1); 3340 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3341 3342 rb_addr = ring->gpu_addr >> 8; 3343 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3344 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3345 3346 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3347 if (ring->use_doorbell) { 3348 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3349 DOORBELL_OFFSET, ring->doorbell_index); 3350 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3351 DOORBELL_EN, 1); 3352 } else { 3353 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3354 } 3355 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3356 3357 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3358 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3359 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3360 3361 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3362 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3363 3364 3365 /* start the ring */ 3366 gfx_v9_0_cp_gfx_start(adev); 3367 ring->sched.ready = true; 3368 3369 return 0; 3370 } 3371 3372 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3373 { 3374 if (enable) { 3375 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3376 } else { 3377 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3378 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3379 adev->gfx.kiq.ring.sched.ready = false; 3380 } 3381 udelay(50); 3382 } 3383 3384 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3385 { 3386 const struct gfx_firmware_header_v1_0 *mec_hdr; 3387 const __le32 *fw_data; 3388 unsigned i; 3389 u32 tmp; 3390 3391 if (!adev->gfx.mec_fw) 3392 return -EINVAL; 3393 3394 gfx_v9_0_cp_compute_enable(adev, false); 3395 3396 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3397 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3398 3399 fw_data = (const __le32 *) 3400 (adev->gfx.mec_fw->data + 3401 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3402 tmp = 0; 3403 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3404 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3405 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3406 3407 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3408 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3409 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3410 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3411 3412 /* MEC1 */ 3413 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3414 mec_hdr->jt_offset); 3415 for (i = 0; i < mec_hdr->jt_size; i++) 3416 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3417 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3418 3419 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3420 adev->gfx.mec_fw_version); 3421 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3422 3423 return 0; 3424 } 3425 3426 /* KIQ functions */ 3427 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3428 { 3429 uint32_t tmp; 3430 struct amdgpu_device *adev = ring->adev; 3431 3432 /* tell RLC which is KIQ queue */ 3433 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3434 tmp &= 0xffffff00; 3435 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3436 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3437 tmp |= 0x80; 3438 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3439 } 3440 3441 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3442 { 3443 struct amdgpu_device *adev = ring->adev; 3444 3445 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3446 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3447 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3448 mqd->cp_hqd_queue_priority = 3449 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3450 } 3451 } 3452 } 3453 3454 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3455 { 3456 struct amdgpu_device *adev = ring->adev; 3457 struct v9_mqd *mqd = ring->mqd_ptr; 3458 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3459 uint32_t tmp; 3460 3461 mqd->header = 0xC0310800; 3462 mqd->compute_pipelinestat_enable = 0x00000001; 3463 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3464 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3465 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3466 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3467 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3468 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3469 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3470 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3471 mqd->compute_misc_reserved = 0x00000003; 3472 3473 mqd->dynamic_cu_mask_addr_lo = 3474 lower_32_bits(ring->mqd_gpu_addr 3475 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3476 mqd->dynamic_cu_mask_addr_hi = 3477 upper_32_bits(ring->mqd_gpu_addr 3478 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3479 3480 eop_base_addr = ring->eop_gpu_addr >> 8; 3481 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3482 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3483 3484 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3485 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3486 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3487 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3488 3489 mqd->cp_hqd_eop_control = tmp; 3490 3491 /* enable doorbell? */ 3492 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3493 3494 if (ring->use_doorbell) { 3495 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3496 DOORBELL_OFFSET, ring->doorbell_index); 3497 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3498 DOORBELL_EN, 1); 3499 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3500 DOORBELL_SOURCE, 0); 3501 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3502 DOORBELL_HIT, 0); 3503 } else { 3504 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3505 DOORBELL_EN, 0); 3506 } 3507 3508 mqd->cp_hqd_pq_doorbell_control = tmp; 3509 3510 /* disable the queue if it's active */ 3511 ring->wptr = 0; 3512 mqd->cp_hqd_dequeue_request = 0; 3513 mqd->cp_hqd_pq_rptr = 0; 3514 mqd->cp_hqd_pq_wptr_lo = 0; 3515 mqd->cp_hqd_pq_wptr_hi = 0; 3516 3517 /* set the pointer to the MQD */ 3518 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3519 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3520 3521 /* set MQD vmid to 0 */ 3522 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3523 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3524 mqd->cp_mqd_control = tmp; 3525 3526 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3527 hqd_gpu_addr = ring->gpu_addr >> 8; 3528 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3529 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3530 3531 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3532 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3533 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3534 (order_base_2(ring->ring_size / 4) - 1)); 3535 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3536 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3537 #ifdef __BIG_ENDIAN 3538 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3539 #endif 3540 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3541 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3542 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3543 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3544 mqd->cp_hqd_pq_control = tmp; 3545 3546 /* set the wb address whether it's enabled or not */ 3547 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3548 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3549 mqd->cp_hqd_pq_rptr_report_addr_hi = 3550 upper_32_bits(wb_gpu_addr) & 0xffff; 3551 3552 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3553 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3554 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3555 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3556 3557 tmp = 0; 3558 /* enable the doorbell if requested */ 3559 if (ring->use_doorbell) { 3560 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3561 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3562 DOORBELL_OFFSET, ring->doorbell_index); 3563 3564 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3565 DOORBELL_EN, 1); 3566 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3567 DOORBELL_SOURCE, 0); 3568 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3569 DOORBELL_HIT, 0); 3570 } 3571 3572 mqd->cp_hqd_pq_doorbell_control = tmp; 3573 3574 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3575 ring->wptr = 0; 3576 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3577 3578 /* set the vmid for the queue */ 3579 mqd->cp_hqd_vmid = 0; 3580 3581 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3582 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3583 mqd->cp_hqd_persistent_state = tmp; 3584 3585 /* set MIN_IB_AVAIL_SIZE */ 3586 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3587 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3588 mqd->cp_hqd_ib_control = tmp; 3589 3590 /* set static priority for a queue/ring */ 3591 gfx_v9_0_mqd_set_priority(ring, mqd); 3592 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3593 3594 /* map_queues packet doesn't need activate the queue, 3595 * so only kiq need set this field. 3596 */ 3597 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3598 mqd->cp_hqd_active = 1; 3599 3600 return 0; 3601 } 3602 3603 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3604 { 3605 struct amdgpu_device *adev = ring->adev; 3606 struct v9_mqd *mqd = ring->mqd_ptr; 3607 int j; 3608 3609 /* disable wptr polling */ 3610 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3611 3612 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3613 mqd->cp_hqd_eop_base_addr_lo); 3614 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3615 mqd->cp_hqd_eop_base_addr_hi); 3616 3617 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3618 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3619 mqd->cp_hqd_eop_control); 3620 3621 /* enable doorbell? */ 3622 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3623 mqd->cp_hqd_pq_doorbell_control); 3624 3625 /* disable the queue if it's active */ 3626 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3627 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3628 for (j = 0; j < adev->usec_timeout; j++) { 3629 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3630 break; 3631 udelay(1); 3632 } 3633 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3634 mqd->cp_hqd_dequeue_request); 3635 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3636 mqd->cp_hqd_pq_rptr); 3637 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3638 mqd->cp_hqd_pq_wptr_lo); 3639 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3640 mqd->cp_hqd_pq_wptr_hi); 3641 } 3642 3643 /* set the pointer to the MQD */ 3644 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3645 mqd->cp_mqd_base_addr_lo); 3646 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3647 mqd->cp_mqd_base_addr_hi); 3648 3649 /* set MQD vmid to 0 */ 3650 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3651 mqd->cp_mqd_control); 3652 3653 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3654 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3655 mqd->cp_hqd_pq_base_lo); 3656 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3657 mqd->cp_hqd_pq_base_hi); 3658 3659 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3660 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3661 mqd->cp_hqd_pq_control); 3662 3663 /* set the wb address whether it's enabled or not */ 3664 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3665 mqd->cp_hqd_pq_rptr_report_addr_lo); 3666 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3667 mqd->cp_hqd_pq_rptr_report_addr_hi); 3668 3669 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3670 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3671 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3672 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3673 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3674 3675 /* enable the doorbell if requested */ 3676 if (ring->use_doorbell) { 3677 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3678 (adev->doorbell_index.kiq * 2) << 2); 3679 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3680 (adev->doorbell_index.userqueue_end * 2) << 2); 3681 } 3682 3683 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3684 mqd->cp_hqd_pq_doorbell_control); 3685 3686 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3687 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3688 mqd->cp_hqd_pq_wptr_lo); 3689 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3690 mqd->cp_hqd_pq_wptr_hi); 3691 3692 /* set the vmid for the queue */ 3693 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3694 3695 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3696 mqd->cp_hqd_persistent_state); 3697 3698 /* activate the queue */ 3699 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3700 mqd->cp_hqd_active); 3701 3702 if (ring->use_doorbell) 3703 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3704 3705 return 0; 3706 } 3707 3708 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3709 { 3710 struct amdgpu_device *adev = ring->adev; 3711 int j; 3712 3713 /* disable the queue if it's active */ 3714 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3715 3716 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3717 3718 for (j = 0; j < adev->usec_timeout; j++) { 3719 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3720 break; 3721 udelay(1); 3722 } 3723 3724 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3725 DRM_DEBUG("KIQ dequeue request failed.\n"); 3726 3727 /* Manual disable if dequeue request times out */ 3728 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3729 } 3730 3731 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3732 0); 3733 } 3734 3735 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3736 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3737 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3738 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3739 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3740 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3741 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3742 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3743 3744 return 0; 3745 } 3746 3747 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3748 { 3749 struct amdgpu_device *adev = ring->adev; 3750 struct v9_mqd *mqd = ring->mqd_ptr; 3751 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3752 struct v9_mqd *tmp_mqd; 3753 3754 gfx_v9_0_kiq_setting(ring); 3755 3756 /* GPU could be in bad state during probe, driver trigger the reset 3757 * after load the SMU, in this case , the mqd is not be initialized. 3758 * driver need to re-init the mqd. 3759 * check mqd->cp_hqd_pq_control since this value should not be 0 3760 */ 3761 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3762 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3763 /* for GPU_RESET case , reset MQD to a clean status */ 3764 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3765 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3766 3767 /* reset ring buffer */ 3768 ring->wptr = 0; 3769 amdgpu_ring_clear_ring(ring); 3770 3771 mutex_lock(&adev->srbm_mutex); 3772 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3773 gfx_v9_0_kiq_init_register(ring); 3774 soc15_grbm_select(adev, 0, 0, 0, 0); 3775 mutex_unlock(&adev->srbm_mutex); 3776 } else { 3777 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3778 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3779 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3780 mutex_lock(&adev->srbm_mutex); 3781 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3782 gfx_v9_0_mqd_init(ring); 3783 gfx_v9_0_kiq_init_register(ring); 3784 soc15_grbm_select(adev, 0, 0, 0, 0); 3785 mutex_unlock(&adev->srbm_mutex); 3786 3787 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3788 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3789 } 3790 3791 return 0; 3792 } 3793 3794 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3795 { 3796 struct amdgpu_device *adev = ring->adev; 3797 struct v9_mqd *mqd = ring->mqd_ptr; 3798 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3799 struct v9_mqd *tmp_mqd; 3800 3801 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3802 * is not be initialized before 3803 */ 3804 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3805 3806 if (!tmp_mqd->cp_hqd_pq_control || 3807 (!amdgpu_in_reset(adev) && !adev->in_suspend)) { 3808 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3809 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3810 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3811 mutex_lock(&adev->srbm_mutex); 3812 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3813 gfx_v9_0_mqd_init(ring); 3814 soc15_grbm_select(adev, 0, 0, 0, 0); 3815 mutex_unlock(&adev->srbm_mutex); 3816 3817 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3818 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3819 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3820 /* reset MQD to a clean status */ 3821 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3822 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3823 3824 /* reset ring buffer */ 3825 ring->wptr = 0; 3826 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3827 amdgpu_ring_clear_ring(ring); 3828 } else { 3829 amdgpu_ring_clear_ring(ring); 3830 } 3831 3832 return 0; 3833 } 3834 3835 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3836 { 3837 struct amdgpu_ring *ring; 3838 int r; 3839 3840 ring = &adev->gfx.kiq.ring; 3841 3842 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3843 if (unlikely(r != 0)) 3844 return r; 3845 3846 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3847 if (unlikely(r != 0)) 3848 return r; 3849 3850 gfx_v9_0_kiq_init_queue(ring); 3851 amdgpu_bo_kunmap(ring->mqd_obj); 3852 ring->mqd_ptr = NULL; 3853 amdgpu_bo_unreserve(ring->mqd_obj); 3854 ring->sched.ready = true; 3855 return 0; 3856 } 3857 3858 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3859 { 3860 struct amdgpu_ring *ring = NULL; 3861 int r = 0, i; 3862 3863 gfx_v9_0_cp_compute_enable(adev, true); 3864 3865 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3866 ring = &adev->gfx.compute_ring[i]; 3867 3868 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3869 if (unlikely(r != 0)) 3870 goto done; 3871 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3872 if (!r) { 3873 r = gfx_v9_0_kcq_init_queue(ring); 3874 amdgpu_bo_kunmap(ring->mqd_obj); 3875 ring->mqd_ptr = NULL; 3876 } 3877 amdgpu_bo_unreserve(ring->mqd_obj); 3878 if (r) 3879 goto done; 3880 } 3881 3882 r = amdgpu_gfx_enable_kcq(adev); 3883 done: 3884 return r; 3885 } 3886 3887 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3888 { 3889 int r, i; 3890 struct amdgpu_ring *ring; 3891 3892 if (!(adev->flags & AMD_IS_APU)) 3893 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3894 3895 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3896 if (adev->gfx.num_gfx_rings) { 3897 /* legacy firmware loading */ 3898 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3899 if (r) 3900 return r; 3901 } 3902 3903 r = gfx_v9_0_cp_compute_load_microcode(adev); 3904 if (r) 3905 return r; 3906 } 3907 3908 r = gfx_v9_0_kiq_resume(adev); 3909 if (r) 3910 return r; 3911 3912 if (adev->gfx.num_gfx_rings) { 3913 r = gfx_v9_0_cp_gfx_resume(adev); 3914 if (r) 3915 return r; 3916 } 3917 3918 r = gfx_v9_0_kcq_resume(adev); 3919 if (r) 3920 return r; 3921 3922 if (adev->gfx.num_gfx_rings) { 3923 ring = &adev->gfx.gfx_ring[0]; 3924 r = amdgpu_ring_test_helper(ring); 3925 if (r) 3926 return r; 3927 } 3928 3929 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3930 ring = &adev->gfx.compute_ring[i]; 3931 amdgpu_ring_test_helper(ring); 3932 } 3933 3934 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3935 3936 return 0; 3937 } 3938 3939 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3940 { 3941 u32 tmp; 3942 3943 if (adev->asic_type != CHIP_ARCTURUS) 3944 return; 3945 3946 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3947 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3948 adev->df.hash_status.hash_64k); 3949 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3950 adev->df.hash_status.hash_2m); 3951 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3952 adev->df.hash_status.hash_1g); 3953 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3954 } 3955 3956 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3957 { 3958 if (adev->gfx.num_gfx_rings) 3959 gfx_v9_0_cp_gfx_enable(adev, enable); 3960 gfx_v9_0_cp_compute_enable(adev, enable); 3961 } 3962 3963 static int gfx_v9_0_hw_init(void *handle) 3964 { 3965 int r; 3966 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3967 3968 if (!amdgpu_sriov_vf(adev)) 3969 gfx_v9_0_init_golden_registers(adev); 3970 3971 gfx_v9_0_constants_init(adev); 3972 3973 gfx_v9_0_init_tcp_config(adev); 3974 3975 r = adev->gfx.rlc.funcs->resume(adev); 3976 if (r) 3977 return r; 3978 3979 r = gfx_v9_0_cp_resume(adev); 3980 if (r) 3981 return r; 3982 3983 if (adev->asic_type == CHIP_ALDEBARAN) 3984 gfx_v9_4_2_set_power_brake_sequence(adev); 3985 3986 return r; 3987 } 3988 3989 static int gfx_v9_0_hw_fini(void *handle) 3990 { 3991 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3992 3993 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3994 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3995 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3996 3997 /* DF freeze and kcq disable will fail */ 3998 if (!amdgpu_ras_intr_triggered()) 3999 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4000 amdgpu_gfx_disable_kcq(adev); 4001 4002 if (amdgpu_sriov_vf(adev)) { 4003 gfx_v9_0_cp_gfx_enable(adev, false); 4004 /* must disable polling for SRIOV when hw finished, otherwise 4005 * CPC engine may still keep fetching WB address which is already 4006 * invalid after sw finished and trigger DMAR reading error in 4007 * hypervisor side. 4008 */ 4009 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4010 return 0; 4011 } 4012 4013 /* Use deinitialize sequence from CAIL when unbinding device from driver, 4014 * otherwise KIQ is hanging when binding back 4015 */ 4016 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4017 mutex_lock(&adev->srbm_mutex); 4018 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 4019 adev->gfx.kiq.ring.pipe, 4020 adev->gfx.kiq.ring.queue, 0); 4021 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 4022 soc15_grbm_select(adev, 0, 0, 0, 0); 4023 mutex_unlock(&adev->srbm_mutex); 4024 } 4025 4026 gfx_v9_0_cp_enable(adev, false); 4027 4028 /* Skip suspend with A+A reset */ 4029 if (adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) { 4030 dev_dbg(adev->dev, "Device in reset. Skipping RLC halt\n"); 4031 return 0; 4032 } 4033 4034 adev->gfx.rlc.funcs->stop(adev); 4035 return 0; 4036 } 4037 4038 static int gfx_v9_0_suspend(void *handle) 4039 { 4040 return gfx_v9_0_hw_fini(handle); 4041 } 4042 4043 static int gfx_v9_0_resume(void *handle) 4044 { 4045 return gfx_v9_0_hw_init(handle); 4046 } 4047 4048 static bool gfx_v9_0_is_idle(void *handle) 4049 { 4050 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4051 4052 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 4053 GRBM_STATUS, GUI_ACTIVE)) 4054 return false; 4055 else 4056 return true; 4057 } 4058 4059 static int gfx_v9_0_wait_for_idle(void *handle) 4060 { 4061 unsigned i; 4062 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4063 4064 for (i = 0; i < adev->usec_timeout; i++) { 4065 if (gfx_v9_0_is_idle(handle)) 4066 return 0; 4067 udelay(1); 4068 } 4069 return -ETIMEDOUT; 4070 } 4071 4072 static int gfx_v9_0_soft_reset(void *handle) 4073 { 4074 u32 grbm_soft_reset = 0; 4075 u32 tmp; 4076 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4077 4078 /* GRBM_STATUS */ 4079 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4080 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4081 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4082 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4083 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4084 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4085 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4086 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4087 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4088 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4089 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4090 } 4091 4092 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4093 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4094 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4095 } 4096 4097 /* GRBM_STATUS2 */ 4098 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4099 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4100 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4101 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4102 4103 4104 if (grbm_soft_reset) { 4105 /* stop the rlc */ 4106 adev->gfx.rlc.funcs->stop(adev); 4107 4108 if (adev->gfx.num_gfx_rings) 4109 /* Disable GFX parsing/prefetching */ 4110 gfx_v9_0_cp_gfx_enable(adev, false); 4111 4112 /* Disable MEC parsing/prefetching */ 4113 gfx_v9_0_cp_compute_enable(adev, false); 4114 4115 if (grbm_soft_reset) { 4116 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4117 tmp |= grbm_soft_reset; 4118 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4119 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4120 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4121 4122 udelay(50); 4123 4124 tmp &= ~grbm_soft_reset; 4125 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4126 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4127 } 4128 4129 /* Wait a little for things to settle down */ 4130 udelay(50); 4131 } 4132 return 0; 4133 } 4134 4135 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4136 { 4137 signed long r, cnt = 0; 4138 unsigned long flags; 4139 uint32_t seq, reg_val_offs = 0; 4140 uint64_t value = 0; 4141 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4142 struct amdgpu_ring *ring = &kiq->ring; 4143 4144 BUG_ON(!ring->funcs->emit_rreg); 4145 4146 spin_lock_irqsave(&kiq->ring_lock, flags); 4147 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4148 pr_err("critical bug! too many kiq readers\n"); 4149 goto failed_unlock; 4150 } 4151 amdgpu_ring_alloc(ring, 32); 4152 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4153 amdgpu_ring_write(ring, 9 | /* src: register*/ 4154 (5 << 8) | /* dst: memory */ 4155 (1 << 16) | /* count sel */ 4156 (1 << 20)); /* write confirm */ 4157 amdgpu_ring_write(ring, 0); 4158 amdgpu_ring_write(ring, 0); 4159 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4160 reg_val_offs * 4)); 4161 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4162 reg_val_offs * 4)); 4163 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4164 if (r) 4165 goto failed_undo; 4166 4167 amdgpu_ring_commit(ring); 4168 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4169 4170 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4171 4172 /* don't wait anymore for gpu reset case because this way may 4173 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4174 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4175 * never return if we keep waiting in virt_kiq_rreg, which cause 4176 * gpu_recover() hang there. 4177 * 4178 * also don't wait anymore for IRQ context 4179 * */ 4180 if (r < 1 && (amdgpu_in_reset(adev))) 4181 goto failed_kiq_read; 4182 4183 might_sleep(); 4184 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4185 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4186 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4187 } 4188 4189 if (cnt > MAX_KIQ_REG_TRY) 4190 goto failed_kiq_read; 4191 4192 mb(); 4193 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4194 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4195 amdgpu_device_wb_free(adev, reg_val_offs); 4196 return value; 4197 4198 failed_undo: 4199 amdgpu_ring_undo(ring); 4200 failed_unlock: 4201 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4202 failed_kiq_read: 4203 if (reg_val_offs) 4204 amdgpu_device_wb_free(adev, reg_val_offs); 4205 pr_err("failed to read gpu clock\n"); 4206 return ~0; 4207 } 4208 4209 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4210 { 4211 uint64_t clock; 4212 4213 amdgpu_gfx_off_ctrl(adev, false); 4214 mutex_lock(&adev->gfx.gpu_clock_mutex); 4215 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4216 clock = gfx_v9_0_kiq_read_clock(adev); 4217 } else { 4218 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4219 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4220 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4221 } 4222 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4223 amdgpu_gfx_off_ctrl(adev, true); 4224 return clock; 4225 } 4226 4227 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4228 uint32_t vmid, 4229 uint32_t gds_base, uint32_t gds_size, 4230 uint32_t gws_base, uint32_t gws_size, 4231 uint32_t oa_base, uint32_t oa_size) 4232 { 4233 struct amdgpu_device *adev = ring->adev; 4234 4235 /* GDS Base */ 4236 gfx_v9_0_write_data_to_reg(ring, 0, false, 4237 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4238 gds_base); 4239 4240 /* GDS Size */ 4241 gfx_v9_0_write_data_to_reg(ring, 0, false, 4242 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4243 gds_size); 4244 4245 /* GWS */ 4246 gfx_v9_0_write_data_to_reg(ring, 0, false, 4247 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4248 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4249 4250 /* OA */ 4251 gfx_v9_0_write_data_to_reg(ring, 0, false, 4252 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4253 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4254 } 4255 4256 static const u32 vgpr_init_compute_shader[] = 4257 { 4258 0xb07c0000, 0xbe8000ff, 4259 0x000000f8, 0xbf110800, 4260 0x7e000280, 0x7e020280, 4261 0x7e040280, 0x7e060280, 4262 0x7e080280, 0x7e0a0280, 4263 0x7e0c0280, 0x7e0e0280, 4264 0x80808800, 0xbe803200, 4265 0xbf84fff5, 0xbf9c0000, 4266 0xd28c0001, 0x0001007f, 4267 0xd28d0001, 0x0002027e, 4268 0x10020288, 0xb8810904, 4269 0xb7814000, 0xd1196a01, 4270 0x00000301, 0xbe800087, 4271 0xbefc00c1, 0xd89c4000, 4272 0x00020201, 0xd89cc080, 4273 0x00040401, 0x320202ff, 4274 0x00000800, 0x80808100, 4275 0xbf84fff8, 0x7e020280, 4276 0xbf810000, 0x00000000, 4277 }; 4278 4279 static const u32 sgpr_init_compute_shader[] = 4280 { 4281 0xb07c0000, 0xbe8000ff, 4282 0x0000005f, 0xbee50080, 4283 0xbe812c65, 0xbe822c65, 4284 0xbe832c65, 0xbe842c65, 4285 0xbe852c65, 0xb77c0005, 4286 0x80808500, 0xbf84fff8, 4287 0xbe800080, 0xbf810000, 4288 }; 4289 4290 static const u32 vgpr_init_compute_shader_arcturus[] = { 4291 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4292 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4293 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4294 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4295 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4296 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4297 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4298 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4299 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4300 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4301 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4302 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4303 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4304 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4305 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4306 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4307 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4308 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4309 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4310 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4311 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4312 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4313 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4314 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4315 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4316 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4317 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4318 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4319 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4320 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4321 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4322 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4323 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4324 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4325 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4326 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4327 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4328 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4329 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4330 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4331 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4332 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4333 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4334 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4335 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4336 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4337 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4338 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4339 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4340 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4341 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4342 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4343 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4344 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4345 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4346 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4347 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4348 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4349 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4350 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4351 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4352 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4353 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4354 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4355 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4356 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4357 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4358 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4359 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4360 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4361 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4362 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4363 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4364 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4365 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4366 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4367 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4368 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4369 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4370 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4371 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4372 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4373 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4374 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4375 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4376 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4377 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4378 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4379 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4380 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4381 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4382 0xbf84fff8, 0xbf810000, 4383 }; 4384 4385 /* When below register arrays changed, please update gpr_reg_size, 4386 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4387 to cover all gfx9 ASICs */ 4388 static const struct soc15_reg_entry vgpr_init_regs[] = { 4389 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4390 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4391 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4392 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4393 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4394 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4395 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4396 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4397 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4398 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4399 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4400 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4401 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4402 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4403 }; 4404 4405 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4406 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4407 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4408 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4409 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4410 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4411 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4412 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4413 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4414 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4415 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4416 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4417 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4418 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4419 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4420 }; 4421 4422 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4423 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4424 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4425 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4426 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4427 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4428 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4429 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4430 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4431 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4432 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4433 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4434 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4435 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4436 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4437 }; 4438 4439 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4440 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4441 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4442 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4443 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4444 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4445 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4446 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4447 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4448 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4449 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4450 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4451 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4452 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4453 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4454 }; 4455 4456 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4457 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4458 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4459 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4460 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4461 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4462 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4463 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4464 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4465 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4466 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4467 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4468 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4469 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4470 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4471 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4472 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4473 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4474 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4475 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4476 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4477 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4478 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4479 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4480 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4481 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4482 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4483 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4484 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4485 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4486 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4487 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4488 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4489 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4490 }; 4491 4492 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4493 { 4494 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4495 int i, r; 4496 4497 /* only support when RAS is enabled */ 4498 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4499 return 0; 4500 4501 r = amdgpu_ring_alloc(ring, 7); 4502 if (r) { 4503 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4504 ring->name, r); 4505 return r; 4506 } 4507 4508 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4509 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4510 4511 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4512 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4513 PACKET3_DMA_DATA_DST_SEL(1) | 4514 PACKET3_DMA_DATA_SRC_SEL(2) | 4515 PACKET3_DMA_DATA_ENGINE(0))); 4516 amdgpu_ring_write(ring, 0); 4517 amdgpu_ring_write(ring, 0); 4518 amdgpu_ring_write(ring, 0); 4519 amdgpu_ring_write(ring, 0); 4520 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4521 adev->gds.gds_size); 4522 4523 amdgpu_ring_commit(ring); 4524 4525 for (i = 0; i < adev->usec_timeout; i++) { 4526 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4527 break; 4528 udelay(1); 4529 } 4530 4531 if (i >= adev->usec_timeout) 4532 r = -ETIMEDOUT; 4533 4534 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4535 4536 return r; 4537 } 4538 4539 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4540 { 4541 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4542 struct amdgpu_ib ib; 4543 struct dma_fence *f = NULL; 4544 int r, i; 4545 unsigned total_size, vgpr_offset, sgpr_offset; 4546 u64 gpu_addr; 4547 4548 int compute_dim_x = adev->gfx.config.max_shader_engines * 4549 adev->gfx.config.max_cu_per_sh * 4550 adev->gfx.config.max_sh_per_se; 4551 int sgpr_work_group_size = 5; 4552 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4553 int vgpr_init_shader_size; 4554 const u32 *vgpr_init_shader_ptr; 4555 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4556 4557 /* only support when RAS is enabled */ 4558 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4559 return 0; 4560 4561 /* bail if the compute ring is not ready */ 4562 if (!ring->sched.ready) 4563 return 0; 4564 4565 if (adev->asic_type == CHIP_ARCTURUS || 4566 adev->asic_type == CHIP_ALDEBARAN) { 4567 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4568 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4569 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4570 } else { 4571 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4572 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4573 vgpr_init_regs_ptr = vgpr_init_regs; 4574 } 4575 4576 total_size = 4577 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4578 total_size += 4579 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4580 total_size += 4581 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4582 total_size = ALIGN(total_size, 256); 4583 vgpr_offset = total_size; 4584 total_size += ALIGN(vgpr_init_shader_size, 256); 4585 sgpr_offset = total_size; 4586 total_size += sizeof(sgpr_init_compute_shader); 4587 4588 /* allocate an indirect buffer to put the commands in */ 4589 memset(&ib, 0, sizeof(ib)); 4590 r = amdgpu_ib_get(adev, NULL, total_size, 4591 AMDGPU_IB_POOL_DIRECT, &ib); 4592 if (r) { 4593 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4594 return r; 4595 } 4596 4597 /* load the compute shaders */ 4598 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4599 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4600 4601 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4602 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4603 4604 /* init the ib length to 0 */ 4605 ib.length_dw = 0; 4606 4607 /* VGPR */ 4608 /* write the register state for the compute dispatch */ 4609 for (i = 0; i < gpr_reg_size; i++) { 4610 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4611 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4612 - PACKET3_SET_SH_REG_START; 4613 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4614 } 4615 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4616 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4617 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4618 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4619 - PACKET3_SET_SH_REG_START; 4620 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4621 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4622 4623 /* write dispatch packet */ 4624 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4625 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4626 ib.ptr[ib.length_dw++] = 1; /* y */ 4627 ib.ptr[ib.length_dw++] = 1; /* z */ 4628 ib.ptr[ib.length_dw++] = 4629 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4630 4631 /* write CS partial flush packet */ 4632 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4633 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4634 4635 /* SGPR1 */ 4636 /* write the register state for the compute dispatch */ 4637 for (i = 0; i < gpr_reg_size; i++) { 4638 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4639 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4640 - PACKET3_SET_SH_REG_START; 4641 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4642 } 4643 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4644 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4645 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4646 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4647 - PACKET3_SET_SH_REG_START; 4648 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4649 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4650 4651 /* write dispatch packet */ 4652 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4653 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4654 ib.ptr[ib.length_dw++] = 1; /* y */ 4655 ib.ptr[ib.length_dw++] = 1; /* z */ 4656 ib.ptr[ib.length_dw++] = 4657 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4658 4659 /* write CS partial flush packet */ 4660 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4661 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4662 4663 /* SGPR2 */ 4664 /* write the register state for the compute dispatch */ 4665 for (i = 0; i < gpr_reg_size; i++) { 4666 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4667 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4668 - PACKET3_SET_SH_REG_START; 4669 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4670 } 4671 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4672 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4673 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4674 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4675 - PACKET3_SET_SH_REG_START; 4676 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4677 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4678 4679 /* write dispatch packet */ 4680 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4681 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4682 ib.ptr[ib.length_dw++] = 1; /* y */ 4683 ib.ptr[ib.length_dw++] = 1; /* z */ 4684 ib.ptr[ib.length_dw++] = 4685 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4686 4687 /* write CS partial flush packet */ 4688 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4689 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4690 4691 /* shedule the ib on the ring */ 4692 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4693 if (r) { 4694 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4695 goto fail; 4696 } 4697 4698 /* wait for the GPU to finish processing the IB */ 4699 r = dma_fence_wait(f, false); 4700 if (r) { 4701 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4702 goto fail; 4703 } 4704 4705 fail: 4706 amdgpu_ib_free(adev, &ib, NULL); 4707 dma_fence_put(f); 4708 4709 return r; 4710 } 4711 4712 static int gfx_v9_0_early_init(void *handle) 4713 { 4714 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4715 4716 if (adev->asic_type == CHIP_ARCTURUS || 4717 adev->asic_type == CHIP_ALDEBARAN) 4718 adev->gfx.num_gfx_rings = 0; 4719 else 4720 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4721 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4722 AMDGPU_MAX_COMPUTE_RINGS); 4723 gfx_v9_0_set_kiq_pm4_funcs(adev); 4724 gfx_v9_0_set_ring_funcs(adev); 4725 gfx_v9_0_set_irq_funcs(adev); 4726 gfx_v9_0_set_gds_init(adev); 4727 gfx_v9_0_set_rlc_funcs(adev); 4728 4729 return 0; 4730 } 4731 4732 static int gfx_v9_0_ecc_late_init(void *handle) 4733 { 4734 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4735 int r; 4736 4737 /* 4738 * Temp workaround to fix the issue that CP firmware fails to 4739 * update read pointer when CPDMA is writing clearing operation 4740 * to GDS in suspend/resume sequence on several cards. So just 4741 * limit this operation in cold boot sequence. 4742 */ 4743 if ((!adev->in_suspend) && 4744 (adev->gds.gds_size)) { 4745 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4746 if (r) 4747 return r; 4748 } 4749 4750 /* requires IBs so do in late init after IB pool is initialized */ 4751 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4752 if (r) 4753 return r; 4754 4755 r = amdgpu_gfx_ras_late_init(adev); 4756 if (r) 4757 return r; 4758 4759 if (adev->gfx.funcs->enable_watchdog_timer) 4760 adev->gfx.funcs->enable_watchdog_timer(adev); 4761 4762 return 0; 4763 } 4764 4765 static int gfx_v9_0_late_init(void *handle) 4766 { 4767 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4768 int r; 4769 4770 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4771 if (r) 4772 return r; 4773 4774 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4775 if (r) 4776 return r; 4777 4778 r = gfx_v9_0_ecc_late_init(handle); 4779 if (r) 4780 return r; 4781 4782 return 0; 4783 } 4784 4785 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4786 { 4787 uint32_t rlc_setting; 4788 4789 /* if RLC is not enabled, do nothing */ 4790 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4791 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4792 return false; 4793 4794 return true; 4795 } 4796 4797 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4798 { 4799 uint32_t data; 4800 unsigned i; 4801 4802 data = RLC_SAFE_MODE__CMD_MASK; 4803 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4804 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4805 4806 /* wait for RLC_SAFE_MODE */ 4807 for (i = 0; i < adev->usec_timeout; i++) { 4808 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4809 break; 4810 udelay(1); 4811 } 4812 } 4813 4814 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4815 { 4816 uint32_t data; 4817 4818 data = RLC_SAFE_MODE__CMD_MASK; 4819 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4820 } 4821 4822 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4823 bool enable) 4824 { 4825 amdgpu_gfx_rlc_enter_safe_mode(adev); 4826 4827 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4828 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4829 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4830 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4831 } else { 4832 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4833 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4834 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4835 } 4836 4837 amdgpu_gfx_rlc_exit_safe_mode(adev); 4838 } 4839 4840 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4841 bool enable) 4842 { 4843 /* TODO: double check if we need to perform under safe mode */ 4844 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4845 4846 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4847 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4848 else 4849 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4850 4851 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4852 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4853 else 4854 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4855 4856 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4857 } 4858 4859 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4860 bool enable) 4861 { 4862 uint32_t data, def; 4863 4864 amdgpu_gfx_rlc_enter_safe_mode(adev); 4865 4866 /* It is disabled by HW by default */ 4867 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4868 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4869 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4870 4871 if (adev->asic_type != CHIP_VEGA12) 4872 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4873 4874 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4875 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4876 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4877 4878 /* only for Vega10 & Raven1 */ 4879 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4880 4881 if (def != data) 4882 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4883 4884 /* MGLS is a global flag to control all MGLS in GFX */ 4885 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4886 /* 2 - RLC memory Light sleep */ 4887 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4888 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4889 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4890 if (def != data) 4891 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4892 } 4893 /* 3 - CP memory Light sleep */ 4894 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4895 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4896 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4897 if (def != data) 4898 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4899 } 4900 } 4901 } else { 4902 /* 1 - MGCG_OVERRIDE */ 4903 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4904 4905 if (adev->asic_type != CHIP_VEGA12) 4906 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4907 4908 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4909 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4910 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4911 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4912 4913 if (def != data) 4914 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4915 4916 /* 2 - disable MGLS in RLC */ 4917 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4918 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4919 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4920 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4921 } 4922 4923 /* 3 - disable MGLS in CP */ 4924 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4925 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4926 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4927 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4928 } 4929 } 4930 4931 amdgpu_gfx_rlc_exit_safe_mode(adev); 4932 } 4933 4934 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4935 bool enable) 4936 { 4937 uint32_t data, def; 4938 4939 if (!adev->gfx.num_gfx_rings) 4940 return; 4941 4942 amdgpu_gfx_rlc_enter_safe_mode(adev); 4943 4944 /* Enable 3D CGCG/CGLS */ 4945 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) { 4946 /* write cmd to clear cgcg/cgls ov */ 4947 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4948 /* unset CGCG override */ 4949 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4950 /* update CGCG and CGLS override bits */ 4951 if (def != data) 4952 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4953 4954 /* enable 3Dcgcg FSM(0x0000363f) */ 4955 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4956 4957 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4958 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4959 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4960 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4961 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4962 if (def != data) 4963 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4964 4965 /* set IDLE_POLL_COUNT(0x00900100) */ 4966 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4967 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4968 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4969 if (def != data) 4970 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4971 } else { 4972 /* Disable CGCG/CGLS */ 4973 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4974 /* disable cgcg, cgls should be disabled */ 4975 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4976 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4977 /* disable cgcg and cgls in FSM */ 4978 if (def != data) 4979 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4980 } 4981 4982 amdgpu_gfx_rlc_exit_safe_mode(adev); 4983 } 4984 4985 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4986 bool enable) 4987 { 4988 uint32_t def, data; 4989 4990 amdgpu_gfx_rlc_enter_safe_mode(adev); 4991 4992 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4993 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4994 /* unset CGCG override */ 4995 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4996 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4997 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4998 else 4999 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5000 /* update CGCG and CGLS override bits */ 5001 if (def != data) 5002 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5003 5004 /* enable cgcg FSM(0x0000363F) */ 5005 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5006 5007 if (adev->asic_type == CHIP_ARCTURUS) 5008 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5009 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5010 else 5011 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5012 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5013 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5014 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5015 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5016 if (def != data) 5017 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5018 5019 /* set IDLE_POLL_COUNT(0x00900100) */ 5020 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5021 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5022 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5023 if (def != data) 5024 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5025 } else { 5026 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5027 /* reset CGCG/CGLS bits */ 5028 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5029 /* disable cgcg and cgls in FSM */ 5030 if (def != data) 5031 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5032 } 5033 5034 amdgpu_gfx_rlc_exit_safe_mode(adev); 5035 } 5036 5037 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5038 bool enable) 5039 { 5040 if (enable) { 5041 /* CGCG/CGLS should be enabled after MGCG/MGLS 5042 * === MGCG + MGLS === 5043 */ 5044 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5045 /* === CGCG /CGLS for GFX 3D Only === */ 5046 gfx_v9_0_update_3d_clock_gating(adev, enable); 5047 /* === CGCG + CGLS === */ 5048 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5049 } else { 5050 /* CGCG/CGLS should be disabled before MGCG/MGLS 5051 * === CGCG + CGLS === 5052 */ 5053 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5054 /* === CGCG /CGLS for GFX 3D Only === */ 5055 gfx_v9_0_update_3d_clock_gating(adev, enable); 5056 /* === MGCG + MGLS === */ 5057 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5058 } 5059 return 0; 5060 } 5061 5062 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 5063 { 5064 u32 reg, data; 5065 5066 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5067 if (amdgpu_sriov_is_pp_one_vf(adev)) 5068 data = RREG32_NO_KIQ(reg); 5069 else 5070 data = RREG32(reg); 5071 5072 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5073 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5074 5075 if (amdgpu_sriov_is_pp_one_vf(adev)) 5076 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5077 else 5078 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5079 } 5080 5081 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5082 uint32_t offset, 5083 struct soc15_reg_rlcg *entries, int arr_size) 5084 { 5085 int i; 5086 uint32_t reg; 5087 5088 if (!entries) 5089 return false; 5090 5091 for (i = 0; i < arr_size; i++) { 5092 const struct soc15_reg_rlcg *entry; 5093 5094 entry = &entries[i]; 5095 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5096 if (offset == reg) 5097 return true; 5098 } 5099 5100 return false; 5101 } 5102 5103 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5104 { 5105 return gfx_v9_0_check_rlcg_range(adev, offset, 5106 (void *)rlcg_access_gc_9_0, 5107 ARRAY_SIZE(rlcg_access_gc_9_0)); 5108 } 5109 5110 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5111 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5112 .set_safe_mode = gfx_v9_0_set_safe_mode, 5113 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5114 .init = gfx_v9_0_rlc_init, 5115 .get_csb_size = gfx_v9_0_get_csb_size, 5116 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5117 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5118 .resume = gfx_v9_0_rlc_resume, 5119 .stop = gfx_v9_0_rlc_stop, 5120 .reset = gfx_v9_0_rlc_reset, 5121 .start = gfx_v9_0_rlc_start, 5122 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5123 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5124 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5125 }; 5126 5127 static int gfx_v9_0_set_powergating_state(void *handle, 5128 enum amd_powergating_state state) 5129 { 5130 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5131 bool enable = (state == AMD_PG_STATE_GATE); 5132 5133 switch (adev->asic_type) { 5134 case CHIP_RAVEN: 5135 case CHIP_RENOIR: 5136 if (!enable) 5137 amdgpu_gfx_off_ctrl(adev, false); 5138 5139 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5140 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5141 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5142 } else { 5143 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5144 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5145 } 5146 5147 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5148 gfx_v9_0_enable_cp_power_gating(adev, true); 5149 else 5150 gfx_v9_0_enable_cp_power_gating(adev, false); 5151 5152 /* update gfx cgpg state */ 5153 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5154 5155 /* update mgcg state */ 5156 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5157 5158 if (enable) 5159 amdgpu_gfx_off_ctrl(adev, true); 5160 break; 5161 case CHIP_VEGA12: 5162 amdgpu_gfx_off_ctrl(adev, enable); 5163 break; 5164 default: 5165 break; 5166 } 5167 5168 return 0; 5169 } 5170 5171 static int gfx_v9_0_set_clockgating_state(void *handle, 5172 enum amd_clockgating_state state) 5173 { 5174 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5175 5176 if (amdgpu_sriov_vf(adev)) 5177 return 0; 5178 5179 switch (adev->asic_type) { 5180 case CHIP_VEGA10: 5181 case CHIP_VEGA12: 5182 case CHIP_VEGA20: 5183 case CHIP_RAVEN: 5184 case CHIP_ARCTURUS: 5185 case CHIP_RENOIR: 5186 case CHIP_ALDEBARAN: 5187 gfx_v9_0_update_gfx_clock_gating(adev, 5188 state == AMD_CG_STATE_GATE); 5189 break; 5190 default: 5191 break; 5192 } 5193 return 0; 5194 } 5195 5196 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5197 { 5198 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5199 int data; 5200 5201 if (amdgpu_sriov_vf(adev)) 5202 *flags = 0; 5203 5204 /* AMD_CG_SUPPORT_GFX_MGCG */ 5205 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5206 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5207 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5208 5209 /* AMD_CG_SUPPORT_GFX_CGCG */ 5210 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5211 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5212 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5213 5214 /* AMD_CG_SUPPORT_GFX_CGLS */ 5215 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5216 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5217 5218 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5219 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5220 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5221 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5222 5223 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5224 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5225 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5226 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5227 5228 if (adev->asic_type != CHIP_ARCTURUS) { 5229 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5230 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5231 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5232 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5233 5234 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5235 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5236 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5237 } 5238 } 5239 5240 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5241 { 5242 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5243 } 5244 5245 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5246 { 5247 struct amdgpu_device *adev = ring->adev; 5248 u64 wptr; 5249 5250 /* XXX check if swapping is necessary on BE */ 5251 if (ring->use_doorbell) { 5252 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5253 } else { 5254 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5255 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5256 } 5257 5258 return wptr; 5259 } 5260 5261 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5262 { 5263 struct amdgpu_device *adev = ring->adev; 5264 5265 if (ring->use_doorbell) { 5266 /* XXX check if swapping is necessary on BE */ 5267 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5268 WDOORBELL64(ring->doorbell_index, ring->wptr); 5269 } else { 5270 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5271 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5272 } 5273 } 5274 5275 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5276 { 5277 struct amdgpu_device *adev = ring->adev; 5278 u32 ref_and_mask, reg_mem_engine; 5279 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5280 5281 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5282 switch (ring->me) { 5283 case 1: 5284 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5285 break; 5286 case 2: 5287 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5288 break; 5289 default: 5290 return; 5291 } 5292 reg_mem_engine = 0; 5293 } else { 5294 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5295 reg_mem_engine = 1; /* pfp */ 5296 } 5297 5298 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5299 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5300 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5301 ref_and_mask, ref_and_mask, 0x20); 5302 } 5303 5304 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5305 struct amdgpu_job *job, 5306 struct amdgpu_ib *ib, 5307 uint32_t flags) 5308 { 5309 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5310 u32 header, control = 0; 5311 5312 if (ib->flags & AMDGPU_IB_FLAG_CE) 5313 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5314 else 5315 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5316 5317 control |= ib->length_dw | (vmid << 24); 5318 5319 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5320 control |= INDIRECT_BUFFER_PRE_ENB(1); 5321 5322 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5323 gfx_v9_0_ring_emit_de_meta(ring); 5324 } 5325 5326 amdgpu_ring_write(ring, header); 5327 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5328 amdgpu_ring_write(ring, 5329 #ifdef __BIG_ENDIAN 5330 (2 << 0) | 5331 #endif 5332 lower_32_bits(ib->gpu_addr)); 5333 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5334 amdgpu_ring_write(ring, control); 5335 } 5336 5337 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5338 struct amdgpu_job *job, 5339 struct amdgpu_ib *ib, 5340 uint32_t flags) 5341 { 5342 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5343 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5344 5345 /* Currently, there is a high possibility to get wave ID mismatch 5346 * between ME and GDS, leading to a hw deadlock, because ME generates 5347 * different wave IDs than the GDS expects. This situation happens 5348 * randomly when at least 5 compute pipes use GDS ordered append. 5349 * The wave IDs generated by ME are also wrong after suspend/resume. 5350 * Those are probably bugs somewhere else in the kernel driver. 5351 * 5352 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5353 * GDS to 0 for this ring (me/pipe). 5354 */ 5355 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5356 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5357 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5358 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5359 } 5360 5361 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5362 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5363 amdgpu_ring_write(ring, 5364 #ifdef __BIG_ENDIAN 5365 (2 << 0) | 5366 #endif 5367 lower_32_bits(ib->gpu_addr)); 5368 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5369 amdgpu_ring_write(ring, control); 5370 } 5371 5372 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5373 u64 seq, unsigned flags) 5374 { 5375 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5376 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5377 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5378 5379 /* RELEASE_MEM - flush caches, send int */ 5380 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5381 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5382 EOP_TC_NC_ACTION_EN) : 5383 (EOP_TCL1_ACTION_EN | 5384 EOP_TC_ACTION_EN | 5385 EOP_TC_WB_ACTION_EN | 5386 EOP_TC_MD_ACTION_EN)) | 5387 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5388 EVENT_INDEX(5))); 5389 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5390 5391 /* 5392 * the address should be Qword aligned if 64bit write, Dword 5393 * aligned if only send 32bit data low (discard data high) 5394 */ 5395 if (write64bit) 5396 BUG_ON(addr & 0x7); 5397 else 5398 BUG_ON(addr & 0x3); 5399 amdgpu_ring_write(ring, lower_32_bits(addr)); 5400 amdgpu_ring_write(ring, upper_32_bits(addr)); 5401 amdgpu_ring_write(ring, lower_32_bits(seq)); 5402 amdgpu_ring_write(ring, upper_32_bits(seq)); 5403 amdgpu_ring_write(ring, 0); 5404 } 5405 5406 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5407 { 5408 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5409 uint32_t seq = ring->fence_drv.sync_seq; 5410 uint64_t addr = ring->fence_drv.gpu_addr; 5411 5412 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5413 lower_32_bits(addr), upper_32_bits(addr), 5414 seq, 0xffffffff, 4); 5415 } 5416 5417 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5418 unsigned vmid, uint64_t pd_addr) 5419 { 5420 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5421 5422 /* compute doesn't have PFP */ 5423 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5424 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5425 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5426 amdgpu_ring_write(ring, 0x0); 5427 } 5428 } 5429 5430 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5431 { 5432 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5433 } 5434 5435 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5436 { 5437 u64 wptr; 5438 5439 /* XXX check if swapping is necessary on BE */ 5440 if (ring->use_doorbell) 5441 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5442 else 5443 BUG(); 5444 return wptr; 5445 } 5446 5447 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5448 { 5449 struct amdgpu_device *adev = ring->adev; 5450 5451 /* XXX check if swapping is necessary on BE */ 5452 if (ring->use_doorbell) { 5453 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5454 WDOORBELL64(ring->doorbell_index, ring->wptr); 5455 } else{ 5456 BUG(); /* only DOORBELL method supported on gfx9 now */ 5457 } 5458 } 5459 5460 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5461 u64 seq, unsigned int flags) 5462 { 5463 struct amdgpu_device *adev = ring->adev; 5464 5465 /* we only allocate 32bit for each seq wb address */ 5466 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5467 5468 /* write fence seq to the "addr" */ 5469 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5470 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5471 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5472 amdgpu_ring_write(ring, lower_32_bits(addr)); 5473 amdgpu_ring_write(ring, upper_32_bits(addr)); 5474 amdgpu_ring_write(ring, lower_32_bits(seq)); 5475 5476 if (flags & AMDGPU_FENCE_FLAG_INT) { 5477 /* set register to trigger INT */ 5478 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5479 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5480 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5481 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5482 amdgpu_ring_write(ring, 0); 5483 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5484 } 5485 } 5486 5487 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5488 { 5489 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5490 amdgpu_ring_write(ring, 0); 5491 } 5492 5493 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5494 { 5495 struct v9_ce_ib_state ce_payload = {0}; 5496 uint64_t csa_addr; 5497 int cnt; 5498 5499 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5500 csa_addr = amdgpu_csa_vaddr(ring->adev); 5501 5502 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5503 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5504 WRITE_DATA_DST_SEL(8) | 5505 WR_CONFIRM) | 5506 WRITE_DATA_CACHE_POLICY(0)); 5507 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5508 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5509 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5510 } 5511 5512 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5513 { 5514 struct v9_de_ib_state de_payload = {0}; 5515 uint64_t csa_addr, gds_addr; 5516 int cnt; 5517 5518 csa_addr = amdgpu_csa_vaddr(ring->adev); 5519 gds_addr = csa_addr + 4096; 5520 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5521 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5522 5523 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5524 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5525 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5526 WRITE_DATA_DST_SEL(8) | 5527 WR_CONFIRM) | 5528 WRITE_DATA_CACHE_POLICY(0)); 5529 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5530 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5531 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5532 } 5533 5534 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5535 bool secure) 5536 { 5537 uint32_t v = secure ? FRAME_TMZ : 0; 5538 5539 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5540 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5541 } 5542 5543 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5544 { 5545 uint32_t dw2 = 0; 5546 5547 if (amdgpu_sriov_vf(ring->adev)) 5548 gfx_v9_0_ring_emit_ce_meta(ring); 5549 5550 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5551 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5552 /* set load_global_config & load_global_uconfig */ 5553 dw2 |= 0x8001; 5554 /* set load_cs_sh_regs */ 5555 dw2 |= 0x01000000; 5556 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5557 dw2 |= 0x10002; 5558 5559 /* set load_ce_ram if preamble presented */ 5560 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5561 dw2 |= 0x10000000; 5562 } else { 5563 /* still load_ce_ram if this is the first time preamble presented 5564 * although there is no context switch happens. 5565 */ 5566 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5567 dw2 |= 0x10000000; 5568 } 5569 5570 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5571 amdgpu_ring_write(ring, dw2); 5572 amdgpu_ring_write(ring, 0); 5573 } 5574 5575 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5576 { 5577 unsigned ret; 5578 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5579 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5580 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5581 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5582 ret = ring->wptr & ring->buf_mask; 5583 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5584 return ret; 5585 } 5586 5587 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5588 { 5589 unsigned cur; 5590 BUG_ON(offset > ring->buf_mask); 5591 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5592 5593 cur = (ring->wptr & ring->buf_mask) - 1; 5594 if (likely(cur > offset)) 5595 ring->ring[offset] = cur - offset; 5596 else 5597 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5598 } 5599 5600 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5601 uint32_t reg_val_offs) 5602 { 5603 struct amdgpu_device *adev = ring->adev; 5604 5605 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5606 amdgpu_ring_write(ring, 0 | /* src: register*/ 5607 (5 << 8) | /* dst: memory */ 5608 (1 << 20)); /* write confirm */ 5609 amdgpu_ring_write(ring, reg); 5610 amdgpu_ring_write(ring, 0); 5611 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5612 reg_val_offs * 4)); 5613 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5614 reg_val_offs * 4)); 5615 } 5616 5617 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5618 uint32_t val) 5619 { 5620 uint32_t cmd = 0; 5621 5622 switch (ring->funcs->type) { 5623 case AMDGPU_RING_TYPE_GFX: 5624 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5625 break; 5626 case AMDGPU_RING_TYPE_KIQ: 5627 cmd = (1 << 16); /* no inc addr */ 5628 break; 5629 default: 5630 cmd = WR_CONFIRM; 5631 break; 5632 } 5633 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5634 amdgpu_ring_write(ring, cmd); 5635 amdgpu_ring_write(ring, reg); 5636 amdgpu_ring_write(ring, 0); 5637 amdgpu_ring_write(ring, val); 5638 } 5639 5640 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5641 uint32_t val, uint32_t mask) 5642 { 5643 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5644 } 5645 5646 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5647 uint32_t reg0, uint32_t reg1, 5648 uint32_t ref, uint32_t mask) 5649 { 5650 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5651 struct amdgpu_device *adev = ring->adev; 5652 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5653 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5654 5655 if (fw_version_ok) 5656 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5657 ref, mask, 0x20); 5658 else 5659 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5660 ref, mask); 5661 } 5662 5663 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5664 { 5665 struct amdgpu_device *adev = ring->adev; 5666 uint32_t value = 0; 5667 5668 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5669 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5670 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5671 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5672 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5673 } 5674 5675 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5676 enum amdgpu_interrupt_state state) 5677 { 5678 switch (state) { 5679 case AMDGPU_IRQ_STATE_DISABLE: 5680 case AMDGPU_IRQ_STATE_ENABLE: 5681 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5682 TIME_STAMP_INT_ENABLE, 5683 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5684 break; 5685 default: 5686 break; 5687 } 5688 } 5689 5690 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5691 int me, int pipe, 5692 enum amdgpu_interrupt_state state) 5693 { 5694 u32 mec_int_cntl, mec_int_cntl_reg; 5695 5696 /* 5697 * amdgpu controls only the first MEC. That's why this function only 5698 * handles the setting of interrupts for this specific MEC. All other 5699 * pipes' interrupts are set by amdkfd. 5700 */ 5701 5702 if (me == 1) { 5703 switch (pipe) { 5704 case 0: 5705 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5706 break; 5707 case 1: 5708 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5709 break; 5710 case 2: 5711 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5712 break; 5713 case 3: 5714 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5715 break; 5716 default: 5717 DRM_DEBUG("invalid pipe %d\n", pipe); 5718 return; 5719 } 5720 } else { 5721 DRM_DEBUG("invalid me %d\n", me); 5722 return; 5723 } 5724 5725 switch (state) { 5726 case AMDGPU_IRQ_STATE_DISABLE: 5727 mec_int_cntl = RREG32(mec_int_cntl_reg); 5728 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5729 TIME_STAMP_INT_ENABLE, 0); 5730 WREG32(mec_int_cntl_reg, mec_int_cntl); 5731 break; 5732 case AMDGPU_IRQ_STATE_ENABLE: 5733 mec_int_cntl = RREG32(mec_int_cntl_reg); 5734 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5735 TIME_STAMP_INT_ENABLE, 1); 5736 WREG32(mec_int_cntl_reg, mec_int_cntl); 5737 break; 5738 default: 5739 break; 5740 } 5741 } 5742 5743 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5744 struct amdgpu_irq_src *source, 5745 unsigned type, 5746 enum amdgpu_interrupt_state state) 5747 { 5748 switch (state) { 5749 case AMDGPU_IRQ_STATE_DISABLE: 5750 case AMDGPU_IRQ_STATE_ENABLE: 5751 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5752 PRIV_REG_INT_ENABLE, 5753 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5754 break; 5755 default: 5756 break; 5757 } 5758 5759 return 0; 5760 } 5761 5762 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5763 struct amdgpu_irq_src *source, 5764 unsigned type, 5765 enum amdgpu_interrupt_state state) 5766 { 5767 switch (state) { 5768 case AMDGPU_IRQ_STATE_DISABLE: 5769 case AMDGPU_IRQ_STATE_ENABLE: 5770 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5771 PRIV_INSTR_INT_ENABLE, 5772 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5773 break; 5774 default: 5775 break; 5776 } 5777 5778 return 0; 5779 } 5780 5781 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5782 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5783 CP_ECC_ERROR_INT_ENABLE, 1) 5784 5785 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5786 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5787 CP_ECC_ERROR_INT_ENABLE, 0) 5788 5789 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5790 struct amdgpu_irq_src *source, 5791 unsigned type, 5792 enum amdgpu_interrupt_state state) 5793 { 5794 switch (state) { 5795 case AMDGPU_IRQ_STATE_DISABLE: 5796 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5797 CP_ECC_ERROR_INT_ENABLE, 0); 5798 DISABLE_ECC_ON_ME_PIPE(1, 0); 5799 DISABLE_ECC_ON_ME_PIPE(1, 1); 5800 DISABLE_ECC_ON_ME_PIPE(1, 2); 5801 DISABLE_ECC_ON_ME_PIPE(1, 3); 5802 break; 5803 5804 case AMDGPU_IRQ_STATE_ENABLE: 5805 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5806 CP_ECC_ERROR_INT_ENABLE, 1); 5807 ENABLE_ECC_ON_ME_PIPE(1, 0); 5808 ENABLE_ECC_ON_ME_PIPE(1, 1); 5809 ENABLE_ECC_ON_ME_PIPE(1, 2); 5810 ENABLE_ECC_ON_ME_PIPE(1, 3); 5811 break; 5812 default: 5813 break; 5814 } 5815 5816 return 0; 5817 } 5818 5819 5820 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5821 struct amdgpu_irq_src *src, 5822 unsigned type, 5823 enum amdgpu_interrupt_state state) 5824 { 5825 switch (type) { 5826 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5827 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5828 break; 5829 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5830 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5831 break; 5832 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5833 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5834 break; 5835 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5836 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5837 break; 5838 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5839 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5840 break; 5841 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5842 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5843 break; 5844 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5845 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5846 break; 5847 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5848 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5849 break; 5850 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5851 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5852 break; 5853 default: 5854 break; 5855 } 5856 return 0; 5857 } 5858 5859 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5860 struct amdgpu_irq_src *source, 5861 struct amdgpu_iv_entry *entry) 5862 { 5863 int i; 5864 u8 me_id, pipe_id, queue_id; 5865 struct amdgpu_ring *ring; 5866 5867 DRM_DEBUG("IH: CP EOP\n"); 5868 me_id = (entry->ring_id & 0x0c) >> 2; 5869 pipe_id = (entry->ring_id & 0x03) >> 0; 5870 queue_id = (entry->ring_id & 0x70) >> 4; 5871 5872 switch (me_id) { 5873 case 0: 5874 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5875 break; 5876 case 1: 5877 case 2: 5878 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5879 ring = &adev->gfx.compute_ring[i]; 5880 /* Per-queue interrupt is supported for MEC starting from VI. 5881 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5882 */ 5883 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5884 amdgpu_fence_process(ring); 5885 } 5886 break; 5887 } 5888 return 0; 5889 } 5890 5891 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5892 struct amdgpu_iv_entry *entry) 5893 { 5894 u8 me_id, pipe_id, queue_id; 5895 struct amdgpu_ring *ring; 5896 int i; 5897 5898 me_id = (entry->ring_id & 0x0c) >> 2; 5899 pipe_id = (entry->ring_id & 0x03) >> 0; 5900 queue_id = (entry->ring_id & 0x70) >> 4; 5901 5902 switch (me_id) { 5903 case 0: 5904 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5905 break; 5906 case 1: 5907 case 2: 5908 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5909 ring = &adev->gfx.compute_ring[i]; 5910 if (ring->me == me_id && ring->pipe == pipe_id && 5911 ring->queue == queue_id) 5912 drm_sched_fault(&ring->sched); 5913 } 5914 break; 5915 } 5916 } 5917 5918 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5919 struct amdgpu_irq_src *source, 5920 struct amdgpu_iv_entry *entry) 5921 { 5922 DRM_ERROR("Illegal register access in command stream\n"); 5923 gfx_v9_0_fault(adev, entry); 5924 return 0; 5925 } 5926 5927 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5928 struct amdgpu_irq_src *source, 5929 struct amdgpu_iv_entry *entry) 5930 { 5931 DRM_ERROR("Illegal instruction in command stream\n"); 5932 gfx_v9_0_fault(adev, entry); 5933 return 0; 5934 } 5935 5936 5937 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5938 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5939 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5940 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5941 }, 5942 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5943 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5944 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5945 }, 5946 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5947 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5948 0, 0 5949 }, 5950 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5951 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5952 0, 0 5953 }, 5954 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5955 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5956 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5957 }, 5958 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5959 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5960 0, 0 5961 }, 5962 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5963 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5964 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5965 }, 5966 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5967 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5968 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5969 }, 5970 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5971 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5972 0, 0 5973 }, 5974 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5975 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5976 0, 0 5977 }, 5978 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5979 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5980 0, 0 5981 }, 5982 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5983 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5984 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5985 }, 5986 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5987 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5988 0, 0 5989 }, 5990 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5991 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5992 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5993 }, 5994 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5995 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5996 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5997 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5998 }, 5999 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6000 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6001 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6002 0, 0 6003 }, 6004 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6005 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6006 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6007 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6008 }, 6009 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6010 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6011 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6012 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6013 }, 6014 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6015 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6016 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6017 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6018 }, 6019 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6020 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6021 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6022 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6023 }, 6024 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6025 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6026 0, 0 6027 }, 6028 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6029 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6030 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6031 }, 6032 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6033 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6034 0, 0 6035 }, 6036 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6037 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6038 0, 0 6039 }, 6040 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6041 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6042 0, 0 6043 }, 6044 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6045 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6046 0, 0 6047 }, 6048 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6049 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6050 0, 0 6051 }, 6052 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6053 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6054 0, 0 6055 }, 6056 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6057 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6058 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6059 }, 6060 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6061 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6062 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6063 }, 6064 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6065 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6066 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6067 }, 6068 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6069 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6070 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6071 }, 6072 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6073 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6074 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6075 }, 6076 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6077 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6078 0, 0 6079 }, 6080 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6081 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6082 0, 0 6083 }, 6084 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6085 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6086 0, 0 6087 }, 6088 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6089 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6090 0, 0 6091 }, 6092 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6093 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6094 0, 0 6095 }, 6096 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6097 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6098 0, 0 6099 }, 6100 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6101 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6102 0, 0 6103 }, 6104 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6105 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6106 0, 0 6107 }, 6108 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6109 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6110 0, 0 6111 }, 6112 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6113 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6114 0, 0 6115 }, 6116 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6117 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6118 0, 0 6119 }, 6120 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6121 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6122 0, 0 6123 }, 6124 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6125 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6126 0, 0 6127 }, 6128 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6129 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6130 0, 0 6131 }, 6132 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6133 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6134 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6135 }, 6136 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6137 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6138 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6139 }, 6140 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6141 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6142 0, 0 6143 }, 6144 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6145 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6146 0, 0 6147 }, 6148 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6149 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6150 0, 0 6151 }, 6152 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6153 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6154 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6155 }, 6156 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6157 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6158 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6159 }, 6160 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6161 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6162 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6163 }, 6164 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6165 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6166 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6167 }, 6168 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6169 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6170 0, 0 6171 }, 6172 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6173 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6174 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6175 }, 6176 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6177 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6178 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6179 }, 6180 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6181 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6182 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6183 }, 6184 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6185 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6186 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6187 }, 6188 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6189 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6190 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6191 }, 6192 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6193 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6194 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6195 }, 6196 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6197 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6198 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6199 }, 6200 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6201 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6202 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6203 }, 6204 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6205 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6206 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6207 }, 6208 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6209 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6210 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6211 }, 6212 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6213 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6214 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6215 }, 6216 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6217 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6218 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6219 }, 6220 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6221 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6222 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6223 }, 6224 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6225 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6226 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6227 }, 6228 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6229 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6230 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6231 }, 6232 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6233 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6234 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6235 }, 6236 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6237 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6238 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6239 }, 6240 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6241 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6242 0, 0 6243 }, 6244 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6245 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6246 0, 0 6247 }, 6248 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6249 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6250 0, 0 6251 }, 6252 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6253 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6254 0, 0 6255 }, 6256 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6257 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6258 0, 0 6259 }, 6260 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6261 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6262 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6263 }, 6264 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6265 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6266 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6267 }, 6268 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6269 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6270 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6271 }, 6272 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6273 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6274 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6275 }, 6276 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6277 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6278 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6279 }, 6280 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6281 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6282 0, 0 6283 }, 6284 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6285 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6286 0, 0 6287 }, 6288 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6289 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6290 0, 0 6291 }, 6292 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6293 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6294 0, 0 6295 }, 6296 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6297 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6298 0, 0 6299 }, 6300 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6301 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6302 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6303 }, 6304 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6305 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6306 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6307 }, 6308 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6309 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6310 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6311 }, 6312 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6313 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6314 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6315 }, 6316 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6317 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6318 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6319 }, 6320 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6321 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6322 0, 0 6323 }, 6324 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6325 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6326 0, 0 6327 }, 6328 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6329 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6330 0, 0 6331 }, 6332 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6333 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6334 0, 0 6335 }, 6336 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6337 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6338 0, 0 6339 }, 6340 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6341 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6342 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6343 }, 6344 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6345 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6346 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6347 }, 6348 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6349 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6350 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6351 }, 6352 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6353 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6354 0, 0 6355 }, 6356 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6357 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6358 0, 0 6359 }, 6360 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6361 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6362 0, 0 6363 }, 6364 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6365 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6366 0, 0 6367 }, 6368 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6369 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6370 0, 0 6371 }, 6372 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6373 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6374 0, 0 6375 } 6376 }; 6377 6378 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6379 void *inject_if) 6380 { 6381 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6382 int ret; 6383 struct ta_ras_trigger_error_input block_info = { 0 }; 6384 6385 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6386 return -EINVAL; 6387 6388 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6389 return -EINVAL; 6390 6391 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6392 return -EPERM; 6393 6394 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6395 info->head.type)) { 6396 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6397 ras_gfx_subblocks[info->head.sub_block_index].name, 6398 info->head.type); 6399 return -EPERM; 6400 } 6401 6402 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6403 info->head.type)) { 6404 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6405 ras_gfx_subblocks[info->head.sub_block_index].name, 6406 info->head.type); 6407 return -EPERM; 6408 } 6409 6410 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6411 block_info.sub_block_index = 6412 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6413 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6414 block_info.address = info->address; 6415 block_info.value = info->value; 6416 6417 mutex_lock(&adev->grbm_idx_mutex); 6418 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6419 mutex_unlock(&adev->grbm_idx_mutex); 6420 6421 return ret; 6422 } 6423 6424 static const char *vml2_mems[] = { 6425 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6426 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6427 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6428 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6429 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6430 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6431 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6432 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6433 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6434 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6435 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6436 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6437 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6438 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6439 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6440 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6441 }; 6442 6443 static const char *vml2_walker_mems[] = { 6444 "UTC_VML2_CACHE_PDE0_MEM0", 6445 "UTC_VML2_CACHE_PDE0_MEM1", 6446 "UTC_VML2_CACHE_PDE1_MEM0", 6447 "UTC_VML2_CACHE_PDE1_MEM1", 6448 "UTC_VML2_CACHE_PDE2_MEM0", 6449 "UTC_VML2_CACHE_PDE2_MEM1", 6450 "UTC_VML2_RDIF_LOG_FIFO", 6451 }; 6452 6453 static const char *atc_l2_cache_2m_mems[] = { 6454 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6455 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6456 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6457 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6458 }; 6459 6460 static const char *atc_l2_cache_4k_mems[] = { 6461 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6462 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6463 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6464 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6465 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6466 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6467 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6468 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6469 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6470 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6471 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6472 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6473 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6474 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6475 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6476 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6477 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6478 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6479 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6480 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6481 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6482 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6483 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6484 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6485 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6486 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6487 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6488 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6489 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6490 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6491 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6492 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6493 }; 6494 6495 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6496 struct ras_err_data *err_data) 6497 { 6498 uint32_t i, data; 6499 uint32_t sec_count, ded_count; 6500 6501 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6502 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6503 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6504 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6505 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6506 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6507 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6508 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6509 6510 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6511 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6512 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6513 6514 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6515 if (sec_count) { 6516 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6517 "SEC %d\n", i, vml2_mems[i], sec_count); 6518 err_data->ce_count += sec_count; 6519 } 6520 6521 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6522 if (ded_count) { 6523 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6524 "DED %d\n", i, vml2_mems[i], ded_count); 6525 err_data->ue_count += ded_count; 6526 } 6527 } 6528 6529 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6530 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6531 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6532 6533 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6534 SEC_COUNT); 6535 if (sec_count) { 6536 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6537 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6538 err_data->ce_count += sec_count; 6539 } 6540 6541 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6542 DED_COUNT); 6543 if (ded_count) { 6544 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6545 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6546 err_data->ue_count += ded_count; 6547 } 6548 } 6549 6550 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6551 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6552 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6553 6554 sec_count = (data & 0x00006000L) >> 0xd; 6555 if (sec_count) { 6556 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6557 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6558 sec_count); 6559 err_data->ce_count += sec_count; 6560 } 6561 } 6562 6563 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6564 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6565 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6566 6567 sec_count = (data & 0x00006000L) >> 0xd; 6568 if (sec_count) { 6569 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6570 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6571 sec_count); 6572 err_data->ce_count += sec_count; 6573 } 6574 6575 ded_count = (data & 0x00018000L) >> 0xf; 6576 if (ded_count) { 6577 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6578 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6579 ded_count); 6580 err_data->ue_count += ded_count; 6581 } 6582 } 6583 6584 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6585 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6586 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6587 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6588 6589 return 0; 6590 } 6591 6592 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6593 const struct soc15_reg_entry *reg, 6594 uint32_t se_id, uint32_t inst_id, uint32_t value, 6595 uint32_t *sec_count, uint32_t *ded_count) 6596 { 6597 uint32_t i; 6598 uint32_t sec_cnt, ded_cnt; 6599 6600 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6601 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6602 gfx_v9_0_ras_fields[i].seg != reg->seg || 6603 gfx_v9_0_ras_fields[i].inst != reg->inst) 6604 continue; 6605 6606 sec_cnt = (value & 6607 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6608 gfx_v9_0_ras_fields[i].sec_count_shift; 6609 if (sec_cnt) { 6610 dev_info(adev->dev, "GFX SubBlock %s, " 6611 "Instance[%d][%d], SEC %d\n", 6612 gfx_v9_0_ras_fields[i].name, 6613 se_id, inst_id, 6614 sec_cnt); 6615 *sec_count += sec_cnt; 6616 } 6617 6618 ded_cnt = (value & 6619 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6620 gfx_v9_0_ras_fields[i].ded_count_shift; 6621 if (ded_cnt) { 6622 dev_info(adev->dev, "GFX SubBlock %s, " 6623 "Instance[%d][%d], DED %d\n", 6624 gfx_v9_0_ras_fields[i].name, 6625 se_id, inst_id, 6626 ded_cnt); 6627 *ded_count += ded_cnt; 6628 } 6629 } 6630 6631 return 0; 6632 } 6633 6634 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6635 { 6636 int i, j, k; 6637 6638 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6639 return; 6640 6641 /* read back registers to clear the counters */ 6642 mutex_lock(&adev->grbm_idx_mutex); 6643 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6644 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6645 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6646 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6647 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6648 } 6649 } 6650 } 6651 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6652 mutex_unlock(&adev->grbm_idx_mutex); 6653 6654 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6655 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6656 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6657 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6658 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6659 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6660 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6661 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6662 6663 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6664 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6665 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6666 } 6667 6668 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6669 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6670 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6671 } 6672 6673 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6674 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6675 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6676 } 6677 6678 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6679 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6680 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6681 } 6682 6683 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6684 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6685 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6686 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6687 } 6688 6689 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6690 void *ras_error_status) 6691 { 6692 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6693 uint32_t sec_count = 0, ded_count = 0; 6694 uint32_t i, j, k; 6695 uint32_t reg_value; 6696 6697 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6698 return -EINVAL; 6699 6700 err_data->ue_count = 0; 6701 err_data->ce_count = 0; 6702 6703 mutex_lock(&adev->grbm_idx_mutex); 6704 6705 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6706 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6707 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6708 gfx_v9_0_select_se_sh(adev, j, 0, k); 6709 reg_value = 6710 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6711 if (reg_value) 6712 gfx_v9_0_ras_error_count(adev, 6713 &gfx_v9_0_edc_counter_regs[i], 6714 j, k, reg_value, 6715 &sec_count, &ded_count); 6716 } 6717 } 6718 } 6719 6720 err_data->ce_count += sec_count; 6721 err_data->ue_count += ded_count; 6722 6723 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6724 mutex_unlock(&adev->grbm_idx_mutex); 6725 6726 gfx_v9_0_query_utc_edc_status(adev, err_data); 6727 6728 return 0; 6729 } 6730 6731 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6732 { 6733 const unsigned int cp_coher_cntl = 6734 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6735 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6736 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6737 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6738 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6739 6740 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6741 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6742 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6743 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6744 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6745 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6746 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6747 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6748 } 6749 6750 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 6751 uint32_t pipe, bool enable) 6752 { 6753 struct amdgpu_device *adev = ring->adev; 6754 uint32_t val; 6755 uint32_t wcl_cs_reg; 6756 6757 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 6758 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 6759 6760 switch (pipe) { 6761 case 0: 6762 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 6763 break; 6764 case 1: 6765 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 6766 break; 6767 case 2: 6768 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 6769 break; 6770 case 3: 6771 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 6772 break; 6773 default: 6774 DRM_DEBUG("invalid pipe %d\n", pipe); 6775 return; 6776 } 6777 6778 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 6779 6780 } 6781 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 6782 { 6783 struct amdgpu_device *adev = ring->adev; 6784 uint32_t val; 6785 int i; 6786 6787 6788 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 6789 * number of gfx waves. Setting 5 bit will make sure gfx only gets 6790 * around 25% of gpu resources. 6791 */ 6792 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 6793 amdgpu_ring_emit_wreg(ring, 6794 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 6795 val); 6796 6797 /* Restrict waves for normal/low priority compute queues as well 6798 * to get best QoS for high priority compute jobs. 6799 * 6800 * amdgpu controls only 1st ME(0-3 CS pipes). 6801 */ 6802 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 6803 if (i != ring->pipe) 6804 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 6805 6806 } 6807 } 6808 6809 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6810 .name = "gfx_v9_0", 6811 .early_init = gfx_v9_0_early_init, 6812 .late_init = gfx_v9_0_late_init, 6813 .sw_init = gfx_v9_0_sw_init, 6814 .sw_fini = gfx_v9_0_sw_fini, 6815 .hw_init = gfx_v9_0_hw_init, 6816 .hw_fini = gfx_v9_0_hw_fini, 6817 .suspend = gfx_v9_0_suspend, 6818 .resume = gfx_v9_0_resume, 6819 .is_idle = gfx_v9_0_is_idle, 6820 .wait_for_idle = gfx_v9_0_wait_for_idle, 6821 .soft_reset = gfx_v9_0_soft_reset, 6822 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6823 .set_powergating_state = gfx_v9_0_set_powergating_state, 6824 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6825 }; 6826 6827 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6828 .type = AMDGPU_RING_TYPE_GFX, 6829 .align_mask = 0xff, 6830 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6831 .support_64bit_ptrs = true, 6832 .vmhub = AMDGPU_GFXHUB_0, 6833 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6834 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6835 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6836 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6837 5 + /* COND_EXEC */ 6838 7 + /* PIPELINE_SYNC */ 6839 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6840 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6841 2 + /* VM_FLUSH */ 6842 8 + /* FENCE for VM_FLUSH */ 6843 20 + /* GDS switch */ 6844 4 + /* double SWITCH_BUFFER, 6845 the first COND_EXEC jump to the place just 6846 prior to this double SWITCH_BUFFER */ 6847 5 + /* COND_EXEC */ 6848 7 + /* HDP_flush */ 6849 4 + /* VGT_flush */ 6850 14 + /* CE_META */ 6851 31 + /* DE_META */ 6852 3 + /* CNTX_CTRL */ 6853 5 + /* HDP_INVL */ 6854 8 + 8 + /* FENCE x2 */ 6855 2 + /* SWITCH_BUFFER */ 6856 7, /* gfx_v9_0_emit_mem_sync */ 6857 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6858 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6859 .emit_fence = gfx_v9_0_ring_emit_fence, 6860 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6861 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6862 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6863 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6864 .test_ring = gfx_v9_0_ring_test_ring, 6865 .test_ib = gfx_v9_0_ring_test_ib, 6866 .insert_nop = amdgpu_ring_insert_nop, 6867 .pad_ib = amdgpu_ring_generic_pad_ib, 6868 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6869 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6870 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6871 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6872 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6873 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6874 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6875 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6876 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6877 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6878 }; 6879 6880 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6881 .type = AMDGPU_RING_TYPE_COMPUTE, 6882 .align_mask = 0xff, 6883 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6884 .support_64bit_ptrs = true, 6885 .vmhub = AMDGPU_GFXHUB_0, 6886 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6887 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6888 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6889 .emit_frame_size = 6890 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6891 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6892 5 + /* hdp invalidate */ 6893 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6894 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6895 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6896 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6897 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6898 7 + /* gfx_v9_0_emit_mem_sync */ 6899 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 6900 15, /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 6901 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6902 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6903 .emit_fence = gfx_v9_0_ring_emit_fence, 6904 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6905 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6906 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6907 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6908 .test_ring = gfx_v9_0_ring_test_ring, 6909 .test_ib = gfx_v9_0_ring_test_ib, 6910 .insert_nop = amdgpu_ring_insert_nop, 6911 .pad_ib = amdgpu_ring_generic_pad_ib, 6912 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6913 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6914 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6915 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6916 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 6917 }; 6918 6919 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6920 .type = AMDGPU_RING_TYPE_KIQ, 6921 .align_mask = 0xff, 6922 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6923 .support_64bit_ptrs = true, 6924 .vmhub = AMDGPU_GFXHUB_0, 6925 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6926 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6927 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6928 .emit_frame_size = 6929 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6930 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6931 5 + /* hdp invalidate */ 6932 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6933 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6934 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6935 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6936 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6937 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6938 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6939 .test_ring = gfx_v9_0_ring_test_ring, 6940 .insert_nop = amdgpu_ring_insert_nop, 6941 .pad_ib = amdgpu_ring_generic_pad_ib, 6942 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6943 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6944 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6945 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6946 }; 6947 6948 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6949 { 6950 int i; 6951 6952 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6953 6954 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6955 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6956 6957 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6958 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6959 } 6960 6961 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6962 .set = gfx_v9_0_set_eop_interrupt_state, 6963 .process = gfx_v9_0_eop_irq, 6964 }; 6965 6966 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6967 .set = gfx_v9_0_set_priv_reg_fault_state, 6968 .process = gfx_v9_0_priv_reg_irq, 6969 }; 6970 6971 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6972 .set = gfx_v9_0_set_priv_inst_fault_state, 6973 .process = gfx_v9_0_priv_inst_irq, 6974 }; 6975 6976 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6977 .set = gfx_v9_0_set_cp_ecc_error_state, 6978 .process = amdgpu_gfx_cp_ecc_error_irq, 6979 }; 6980 6981 6982 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6983 { 6984 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6985 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6986 6987 adev->gfx.priv_reg_irq.num_types = 1; 6988 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6989 6990 adev->gfx.priv_inst_irq.num_types = 1; 6991 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6992 6993 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6994 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6995 } 6996 6997 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6998 { 6999 switch (adev->asic_type) { 7000 case CHIP_VEGA10: 7001 case CHIP_VEGA12: 7002 case CHIP_VEGA20: 7003 case CHIP_RAVEN: 7004 case CHIP_ARCTURUS: 7005 case CHIP_RENOIR: 7006 case CHIP_ALDEBARAN: 7007 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7008 break; 7009 default: 7010 break; 7011 } 7012 } 7013 7014 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7015 { 7016 /* init asci gds info */ 7017 switch (adev->asic_type) { 7018 case CHIP_VEGA10: 7019 case CHIP_VEGA12: 7020 case CHIP_VEGA20: 7021 adev->gds.gds_size = 0x10000; 7022 break; 7023 case CHIP_RAVEN: 7024 case CHIP_ARCTURUS: 7025 adev->gds.gds_size = 0x1000; 7026 break; 7027 case CHIP_ALDEBARAN: 7028 /* aldebaran removed all the GDS internal memory, 7029 * only support GWS opcode in kernel, like barrier 7030 * semaphore.etc */ 7031 adev->gds.gds_size = 0; 7032 break; 7033 default: 7034 adev->gds.gds_size = 0x10000; 7035 break; 7036 } 7037 7038 switch (adev->asic_type) { 7039 case CHIP_VEGA10: 7040 case CHIP_VEGA20: 7041 adev->gds.gds_compute_max_wave_id = 0x7ff; 7042 break; 7043 case CHIP_VEGA12: 7044 adev->gds.gds_compute_max_wave_id = 0x27f; 7045 break; 7046 case CHIP_RAVEN: 7047 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7048 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7049 else 7050 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7051 break; 7052 case CHIP_ARCTURUS: 7053 adev->gds.gds_compute_max_wave_id = 0xfff; 7054 break; 7055 case CHIP_ALDEBARAN: 7056 /* deprecated for Aldebaran, no usage at all */ 7057 adev->gds.gds_compute_max_wave_id = 0; 7058 break; 7059 default: 7060 /* this really depends on the chip */ 7061 adev->gds.gds_compute_max_wave_id = 0x7ff; 7062 break; 7063 } 7064 7065 adev->gds.gws_size = 64; 7066 adev->gds.oa_size = 16; 7067 } 7068 7069 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7070 u32 bitmap) 7071 { 7072 u32 data; 7073 7074 if (!bitmap) 7075 return; 7076 7077 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7078 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7079 7080 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7081 } 7082 7083 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7084 { 7085 u32 data, mask; 7086 7087 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7088 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7089 7090 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7091 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7092 7093 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7094 7095 return (~data) & mask; 7096 } 7097 7098 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7099 struct amdgpu_cu_info *cu_info) 7100 { 7101 int i, j, k, counter, active_cu_number = 0; 7102 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7103 unsigned disable_masks[4 * 4]; 7104 7105 if (!adev || !cu_info) 7106 return -EINVAL; 7107 7108 /* 7109 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7110 */ 7111 if (adev->gfx.config.max_shader_engines * 7112 adev->gfx.config.max_sh_per_se > 16) 7113 return -EINVAL; 7114 7115 amdgpu_gfx_parse_disable_cu(disable_masks, 7116 adev->gfx.config.max_shader_engines, 7117 adev->gfx.config.max_sh_per_se); 7118 7119 mutex_lock(&adev->grbm_idx_mutex); 7120 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7121 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7122 mask = 1; 7123 ao_bitmap = 0; 7124 counter = 0; 7125 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 7126 gfx_v9_0_set_user_cu_inactive_bitmap( 7127 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7128 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7129 7130 /* 7131 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7132 * 4x4 size array, and it's usually suitable for Vega 7133 * ASICs which has 4*2 SE/SH layout. 7134 * But for Arcturus, SE/SH layout is changed to 8*1. 7135 * To mostly reduce the impact, we make it compatible 7136 * with current bitmap array as below: 7137 * SE4,SH0 --> bitmap[0][1] 7138 * SE5,SH0 --> bitmap[1][1] 7139 * SE6,SH0 --> bitmap[2][1] 7140 * SE7,SH0 --> bitmap[3][1] 7141 */ 7142 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 7143 7144 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7145 if (bitmap & mask) { 7146 if (counter < adev->gfx.config.max_cu_per_sh) 7147 ao_bitmap |= mask; 7148 counter ++; 7149 } 7150 mask <<= 1; 7151 } 7152 active_cu_number += counter; 7153 if (i < 2 && j < 2) 7154 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7155 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7156 } 7157 } 7158 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7159 mutex_unlock(&adev->grbm_idx_mutex); 7160 7161 cu_info->number = active_cu_number; 7162 cu_info->ao_cu_mask = ao_cu_mask; 7163 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7164 7165 return 0; 7166 } 7167 7168 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7169 { 7170 .type = AMD_IP_BLOCK_TYPE_GFX, 7171 .major = 9, 7172 .minor = 0, 7173 .rev = 0, 7174 .funcs = &gfx_v9_0_ip_funcs, 7175 }; 7176