1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "gfx_v9_4.h" 51 #include "gfx_v9_0.h" 52 53 #include "asic_reg/pwr/pwr_10_0_offset.h" 54 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 55 56 #define GFX9_NUM_GFX_RINGS 1 57 #define GFX9_MEC_HPD_SIZE 4096 58 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 59 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 60 61 #define mmGCEA_PROBE_MAP 0x070c 62 #define mmGCEA_PROBE_MAP_BASE_IDX 0 63 64 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 65 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 66 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 67 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 68 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 69 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 70 71 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 72 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 73 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 75 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 76 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 77 78 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 79 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 80 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 81 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 82 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 83 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 84 85 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 86 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 87 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 88 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 89 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 90 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 91 92 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 93 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 94 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 95 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 96 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 97 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 99 100 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 101 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 102 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 103 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 104 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 105 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 106 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 107 108 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 109 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin"); 110 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 111 112 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 113 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 114 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 115 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 116 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 118 119 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 120 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 122 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 125 126 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 127 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 128 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 129 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 130 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 131 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 132 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 133 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 134 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 135 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 136 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 137 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 138 139 enum ta_ras_gfx_subblock { 140 /*CPC*/ 141 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 142 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 143 TA_RAS_BLOCK__GFX_CPC_UCODE, 144 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 145 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 146 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 147 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 148 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 149 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 150 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 151 /* CPF*/ 152 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 153 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 154 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 155 TA_RAS_BLOCK__GFX_CPF_TAG, 156 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 157 /* CPG*/ 158 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 159 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 160 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 161 TA_RAS_BLOCK__GFX_CPG_TAG, 162 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 163 /* GDS*/ 164 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 165 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 166 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 167 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 168 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 169 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 170 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 171 /* SPI*/ 172 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 173 /* SQ*/ 174 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 175 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 176 TA_RAS_BLOCK__GFX_SQ_LDS_D, 177 TA_RAS_BLOCK__GFX_SQ_LDS_I, 178 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 179 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 180 /* SQC (3 ranges)*/ 181 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 182 /* SQC range 0*/ 183 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 184 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 185 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 186 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 187 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 188 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 189 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 190 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 191 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 192 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 193 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 194 /* SQC range 1*/ 195 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 196 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 197 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 198 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 199 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 200 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 201 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 202 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 203 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 204 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 205 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 206 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 207 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 208 /* SQC range 2*/ 209 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 210 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 211 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 212 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 213 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 214 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 215 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 217 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 218 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 219 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 220 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 221 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 222 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 223 /* TA*/ 224 TA_RAS_BLOCK__GFX_TA_INDEX_START, 225 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 226 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 227 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 228 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 229 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 230 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 231 /* TCA*/ 232 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 233 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 234 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 235 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 236 /* TCC (5 sub-ranges)*/ 237 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 238 /* TCC range 0*/ 239 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 240 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 241 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 242 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 243 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 244 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 245 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 246 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 247 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 248 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 249 /* TCC range 1*/ 250 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 251 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 252 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 253 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 254 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 255 /* TCC range 2*/ 256 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 257 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 258 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 259 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 260 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 261 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 262 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 263 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 264 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 265 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 266 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 267 /* TCC range 3*/ 268 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 269 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 270 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 271 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 272 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 273 /* TCC range 4*/ 274 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 275 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 276 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 277 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 278 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 279 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 280 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 281 /* TCI*/ 282 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 283 /* TCP*/ 284 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 285 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 286 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 287 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 288 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 289 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 290 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 291 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 292 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 293 /* TD*/ 294 TA_RAS_BLOCK__GFX_TD_INDEX_START, 295 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 296 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 297 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 298 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 299 /* EA (3 sub-ranges)*/ 300 TA_RAS_BLOCK__GFX_EA_INDEX_START, 301 /* EA range 0*/ 302 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 303 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 304 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 305 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 306 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 307 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 308 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 309 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 310 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 311 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 312 /* EA range 1*/ 313 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 314 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 315 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 316 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 317 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 318 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 319 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 320 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 321 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 322 /* EA range 2*/ 323 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 324 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 325 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 326 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 327 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 328 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 329 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 330 /* UTC VM L2 bank*/ 331 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 332 /* UTC VM walker*/ 333 TA_RAS_BLOCK__UTC_VML2_WALKER, 334 /* UTC ATC L2 2MB cache*/ 335 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 336 /* UTC ATC L2 4KB cache*/ 337 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 338 TA_RAS_BLOCK__GFX_MAX 339 }; 340 341 struct ras_gfx_subblock { 342 unsigned char *name; 343 int ta_subblock; 344 int hw_supported_error_type; 345 int sw_supported_error_type; 346 }; 347 348 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 349 [AMDGPU_RAS_BLOCK__##subblock] = { \ 350 #subblock, \ 351 TA_RAS_BLOCK__##subblock, \ 352 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 353 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 354 } 355 356 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 357 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 358 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 359 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 360 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 361 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 362 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 363 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 364 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 365 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 366 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 367 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 368 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 369 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 370 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 371 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 372 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 373 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 374 0), 375 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 376 0), 377 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 378 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 379 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 380 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 381 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 382 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 383 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 384 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 385 0, 0), 386 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 387 0), 388 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 389 0, 0), 390 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 391 0), 392 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 393 0, 0), 394 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 395 0), 396 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 397 1), 398 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 399 0, 0, 0), 400 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 401 0), 402 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 403 0), 404 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 405 0), 406 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 407 0), 408 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 409 0), 410 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 411 0, 0), 412 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 413 0), 414 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 415 0), 416 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 417 0, 0, 0), 418 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 419 0), 420 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 421 0), 422 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 423 0), 424 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 425 0), 426 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 427 0), 428 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 429 0, 0), 430 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 431 0), 432 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 433 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 434 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 435 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 436 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 437 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 438 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 439 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 440 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 441 1), 442 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 443 1), 444 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 445 1), 446 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 447 0), 448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 449 0), 450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 459 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 462 0), 463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 464 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 465 0), 466 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 467 0, 0), 468 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 469 0), 470 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 471 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 472 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 473 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 474 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 475 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 476 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 477 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 478 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 479 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 480 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 481 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 497 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 498 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 499 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 500 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 501 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 502 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 503 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 504 }; 505 506 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 507 { 508 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 525 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 526 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 527 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 528 }; 529 530 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 531 { 532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 547 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 548 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 549 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 550 }; 551 552 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 553 { 554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 562 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 563 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 564 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 565 }; 566 567 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 568 { 569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 590 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 591 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 592 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 593 }; 594 595 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 596 { 597 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 600 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 601 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 602 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 603 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 604 }; 605 606 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 607 { 608 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 624 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 625 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 626 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 627 }; 628 629 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 630 { 631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 640 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 641 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 642 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 643 }; 644 645 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 646 { 647 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 648 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 649 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 650 }; 651 652 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 653 { 654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 667 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 668 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 669 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 670 }; 671 672 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 673 { 674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 687 }; 688 689 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 690 { 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 698 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 699 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 700 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 701 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 702 }; 703 704 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 705 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 706 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 707 }; 708 709 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 710 { 711 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 712 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 713 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 714 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 715 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 716 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 717 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 718 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 719 }; 720 721 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 722 { 723 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 724 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 725 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 726 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 727 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 728 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 729 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 730 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 731 }; 732 733 static void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v) 734 { 735 static void *scratch_reg0; 736 static void *scratch_reg1; 737 static void *scratch_reg2; 738 static void *scratch_reg3; 739 static void *spare_int; 740 static uint32_t grbm_cntl; 741 static uint32_t grbm_idx; 742 743 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4; 744 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4; 745 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4; 746 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4; 747 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4; 748 749 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL; 750 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX; 751 752 if (amdgpu_sriov_runtime(adev)) { 753 pr_err("shouldn't call rlcg write register during runtime\n"); 754 return; 755 } 756 757 if (offset == grbm_cntl || offset == grbm_idx) { 758 if (offset == grbm_cntl) 759 writel(v, scratch_reg2); 760 else if (offset == grbm_idx) 761 writel(v, scratch_reg3); 762 763 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4)); 764 } else { 765 uint32_t i = 0; 766 uint32_t retries = 50000; 767 768 writel(v, scratch_reg0); 769 writel(offset | 0x80000000, scratch_reg1); 770 writel(1, spare_int); 771 for (i = 0; i < retries; i++) { 772 u32 tmp; 773 774 tmp = readl(scratch_reg1); 775 if (!(tmp & 0x80000000)) 776 break; 777 778 udelay(10); 779 } 780 if (i >= retries) 781 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset); 782 } 783 784 } 785 786 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 787 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 788 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 789 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 790 791 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 792 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 793 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 794 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 795 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 796 struct amdgpu_cu_info *cu_info); 797 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 798 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring); 799 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 800 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 801 void *ras_error_status); 802 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 803 void *inject_if); 804 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 805 806 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 807 uint64_t queue_mask) 808 { 809 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 810 amdgpu_ring_write(kiq_ring, 811 PACKET3_SET_RESOURCES_VMID_MASK(0) | 812 /* vmid_mask:0* queue_type:0 (KIQ) */ 813 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 814 amdgpu_ring_write(kiq_ring, 815 lower_32_bits(queue_mask)); /* queue mask lo */ 816 amdgpu_ring_write(kiq_ring, 817 upper_32_bits(queue_mask)); /* queue mask hi */ 818 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 819 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 820 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 821 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 822 } 823 824 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 825 struct amdgpu_ring *ring) 826 { 827 struct amdgpu_device *adev = kiq_ring->adev; 828 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 829 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 830 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 831 832 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 833 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 834 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 835 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 836 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 837 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 838 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 839 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 840 /*queue_type: normal compute queue */ 841 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 842 /* alloc format: all_on_one_pipe */ 843 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 844 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 845 /* num_queues: must be 1 */ 846 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 847 amdgpu_ring_write(kiq_ring, 848 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 849 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 850 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 851 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 852 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 853 } 854 855 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 856 struct amdgpu_ring *ring, 857 enum amdgpu_unmap_queues_action action, 858 u64 gpu_addr, u64 seq) 859 { 860 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 861 862 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 863 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 864 PACKET3_UNMAP_QUEUES_ACTION(action) | 865 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 866 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 867 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 868 amdgpu_ring_write(kiq_ring, 869 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 870 871 if (action == PREEMPT_QUEUES_NO_UNMAP) { 872 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 873 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 874 amdgpu_ring_write(kiq_ring, seq); 875 } else { 876 amdgpu_ring_write(kiq_ring, 0); 877 amdgpu_ring_write(kiq_ring, 0); 878 amdgpu_ring_write(kiq_ring, 0); 879 } 880 } 881 882 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 883 struct amdgpu_ring *ring, 884 u64 addr, 885 u64 seq) 886 { 887 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 888 889 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 890 amdgpu_ring_write(kiq_ring, 891 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 892 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 893 PACKET3_QUERY_STATUS_COMMAND(2)); 894 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 895 amdgpu_ring_write(kiq_ring, 896 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 897 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 898 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 899 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 900 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 901 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 902 } 903 904 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 905 uint16_t pasid, uint32_t flush_type, 906 bool all_hub) 907 { 908 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 909 amdgpu_ring_write(kiq_ring, 910 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 911 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 912 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 913 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 914 } 915 916 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 917 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 918 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 919 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 920 .kiq_query_status = gfx_v9_0_kiq_query_status, 921 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 922 .set_resources_size = 8, 923 .map_queues_size = 7, 924 .unmap_queues_size = 6, 925 .query_status_size = 7, 926 .invalidate_tlbs_size = 2, 927 }; 928 929 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 930 { 931 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs; 932 } 933 934 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 935 { 936 switch (adev->asic_type) { 937 case CHIP_VEGA10: 938 soc15_program_register_sequence(adev, 939 golden_settings_gc_9_0, 940 ARRAY_SIZE(golden_settings_gc_9_0)); 941 soc15_program_register_sequence(adev, 942 golden_settings_gc_9_0_vg10, 943 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 944 break; 945 case CHIP_VEGA12: 946 soc15_program_register_sequence(adev, 947 golden_settings_gc_9_2_1, 948 ARRAY_SIZE(golden_settings_gc_9_2_1)); 949 soc15_program_register_sequence(adev, 950 golden_settings_gc_9_2_1_vg12, 951 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 952 break; 953 case CHIP_VEGA20: 954 soc15_program_register_sequence(adev, 955 golden_settings_gc_9_0, 956 ARRAY_SIZE(golden_settings_gc_9_0)); 957 soc15_program_register_sequence(adev, 958 golden_settings_gc_9_0_vg20, 959 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 960 break; 961 case CHIP_ARCTURUS: 962 soc15_program_register_sequence(adev, 963 golden_settings_gc_9_4_1_arct, 964 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 965 break; 966 case CHIP_RAVEN: 967 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 968 ARRAY_SIZE(golden_settings_gc_9_1)); 969 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 970 soc15_program_register_sequence(adev, 971 golden_settings_gc_9_1_rv2, 972 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 973 else 974 soc15_program_register_sequence(adev, 975 golden_settings_gc_9_1_rv1, 976 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 977 break; 978 case CHIP_RENOIR: 979 soc15_program_register_sequence(adev, 980 golden_settings_gc_9_1_rn, 981 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 982 return; /* for renoir, don't need common goldensetting */ 983 default: 984 break; 985 } 986 987 if (adev->asic_type != CHIP_ARCTURUS) 988 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 989 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 990 } 991 992 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev) 993 { 994 adev->gfx.scratch.num_reg = 8; 995 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 996 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1; 997 } 998 999 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1000 bool wc, uint32_t reg, uint32_t val) 1001 { 1002 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1003 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1004 WRITE_DATA_DST_SEL(0) | 1005 (wc ? WR_CONFIRM : 0)); 1006 amdgpu_ring_write(ring, reg); 1007 amdgpu_ring_write(ring, 0); 1008 amdgpu_ring_write(ring, val); 1009 } 1010 1011 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1012 int mem_space, int opt, uint32_t addr0, 1013 uint32_t addr1, uint32_t ref, uint32_t mask, 1014 uint32_t inv) 1015 { 1016 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1017 amdgpu_ring_write(ring, 1018 /* memory (1) or register (0) */ 1019 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1020 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1021 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1022 WAIT_REG_MEM_ENGINE(eng_sel))); 1023 1024 if (mem_space) 1025 BUG_ON(addr0 & 0x3); /* Dword align */ 1026 amdgpu_ring_write(ring, addr0); 1027 amdgpu_ring_write(ring, addr1); 1028 amdgpu_ring_write(ring, ref); 1029 amdgpu_ring_write(ring, mask); 1030 amdgpu_ring_write(ring, inv); /* poll interval */ 1031 } 1032 1033 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1034 { 1035 struct amdgpu_device *adev = ring->adev; 1036 uint32_t scratch; 1037 uint32_t tmp = 0; 1038 unsigned i; 1039 int r; 1040 1041 r = amdgpu_gfx_scratch_get(adev, &scratch); 1042 if (r) 1043 return r; 1044 1045 WREG32(scratch, 0xCAFEDEAD); 1046 r = amdgpu_ring_alloc(ring, 3); 1047 if (r) 1048 goto error_free_scratch; 1049 1050 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1051 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START)); 1052 amdgpu_ring_write(ring, 0xDEADBEEF); 1053 amdgpu_ring_commit(ring); 1054 1055 for (i = 0; i < adev->usec_timeout; i++) { 1056 tmp = RREG32(scratch); 1057 if (tmp == 0xDEADBEEF) 1058 break; 1059 udelay(1); 1060 } 1061 1062 if (i >= adev->usec_timeout) 1063 r = -ETIMEDOUT; 1064 1065 error_free_scratch: 1066 amdgpu_gfx_scratch_free(adev, scratch); 1067 return r; 1068 } 1069 1070 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1071 { 1072 struct amdgpu_device *adev = ring->adev; 1073 struct amdgpu_ib ib; 1074 struct dma_fence *f = NULL; 1075 1076 unsigned index; 1077 uint64_t gpu_addr; 1078 uint32_t tmp; 1079 long r; 1080 1081 r = amdgpu_device_wb_get(adev, &index); 1082 if (r) 1083 return r; 1084 1085 gpu_addr = adev->wb.gpu_addr + (index * 4); 1086 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1087 memset(&ib, 0, sizeof(ib)); 1088 r = amdgpu_ib_get(adev, NULL, 16, 1089 AMDGPU_IB_POOL_DIRECT, &ib); 1090 if (r) 1091 goto err1; 1092 1093 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1094 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1095 ib.ptr[2] = lower_32_bits(gpu_addr); 1096 ib.ptr[3] = upper_32_bits(gpu_addr); 1097 ib.ptr[4] = 0xDEADBEEF; 1098 ib.length_dw = 5; 1099 1100 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1101 if (r) 1102 goto err2; 1103 1104 r = dma_fence_wait_timeout(f, false, timeout); 1105 if (r == 0) { 1106 r = -ETIMEDOUT; 1107 goto err2; 1108 } else if (r < 0) { 1109 goto err2; 1110 } 1111 1112 tmp = adev->wb.wb[index]; 1113 if (tmp == 0xDEADBEEF) 1114 r = 0; 1115 else 1116 r = -EINVAL; 1117 1118 err2: 1119 amdgpu_ib_free(adev, &ib, NULL); 1120 dma_fence_put(f); 1121 err1: 1122 amdgpu_device_wb_free(adev, index); 1123 return r; 1124 } 1125 1126 1127 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1128 { 1129 release_firmware(adev->gfx.pfp_fw); 1130 adev->gfx.pfp_fw = NULL; 1131 release_firmware(adev->gfx.me_fw); 1132 adev->gfx.me_fw = NULL; 1133 release_firmware(adev->gfx.ce_fw); 1134 adev->gfx.ce_fw = NULL; 1135 release_firmware(adev->gfx.rlc_fw); 1136 adev->gfx.rlc_fw = NULL; 1137 release_firmware(adev->gfx.mec_fw); 1138 adev->gfx.mec_fw = NULL; 1139 release_firmware(adev->gfx.mec2_fw); 1140 adev->gfx.mec2_fw = NULL; 1141 1142 kfree(adev->gfx.rlc.register_list_format); 1143 } 1144 1145 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev) 1146 { 1147 const struct rlc_firmware_header_v2_1 *rlc_hdr; 1148 1149 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1150 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver); 1151 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver); 1152 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes); 1153 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes); 1154 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver); 1155 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver); 1156 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes); 1157 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes); 1158 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver); 1159 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver); 1160 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes); 1161 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes); 1162 adev->gfx.rlc.reg_list_format_direct_reg_list_length = 1163 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length); 1164 } 1165 1166 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1167 { 1168 adev->gfx.me_fw_write_wait = false; 1169 adev->gfx.mec_fw_write_wait = false; 1170 1171 if ((adev->asic_type != CHIP_ARCTURUS) && 1172 ((adev->gfx.mec_fw_version < 0x000001a5) || 1173 (adev->gfx.mec_feature_version < 46) || 1174 (adev->gfx.pfp_fw_version < 0x000000b7) || 1175 (adev->gfx.pfp_feature_version < 46))) 1176 DRM_WARN_ONCE("CP firmware version too old, please update!"); 1177 1178 switch (adev->asic_type) { 1179 case CHIP_VEGA10: 1180 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1181 (adev->gfx.me_feature_version >= 42) && 1182 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1183 (adev->gfx.pfp_feature_version >= 42)) 1184 adev->gfx.me_fw_write_wait = true; 1185 1186 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1187 (adev->gfx.mec_feature_version >= 42)) 1188 adev->gfx.mec_fw_write_wait = true; 1189 break; 1190 case CHIP_VEGA12: 1191 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1192 (adev->gfx.me_feature_version >= 44) && 1193 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1194 (adev->gfx.pfp_feature_version >= 44)) 1195 adev->gfx.me_fw_write_wait = true; 1196 1197 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1198 (adev->gfx.mec_feature_version >= 44)) 1199 adev->gfx.mec_fw_write_wait = true; 1200 break; 1201 case CHIP_VEGA20: 1202 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1203 (adev->gfx.me_feature_version >= 44) && 1204 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1205 (adev->gfx.pfp_feature_version >= 44)) 1206 adev->gfx.me_fw_write_wait = true; 1207 1208 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1209 (adev->gfx.mec_feature_version >= 44)) 1210 adev->gfx.mec_fw_write_wait = true; 1211 break; 1212 case CHIP_RAVEN: 1213 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1214 (adev->gfx.me_feature_version >= 42) && 1215 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1216 (adev->gfx.pfp_feature_version >= 42)) 1217 adev->gfx.me_fw_write_wait = true; 1218 1219 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1220 (adev->gfx.mec_feature_version >= 42)) 1221 adev->gfx.mec_fw_write_wait = true; 1222 break; 1223 default: 1224 adev->gfx.me_fw_write_wait = true; 1225 adev->gfx.mec_fw_write_wait = true; 1226 break; 1227 } 1228 } 1229 1230 struct amdgpu_gfxoff_quirk { 1231 u16 chip_vendor; 1232 u16 chip_device; 1233 u16 subsys_vendor; 1234 u16 subsys_device; 1235 u8 revision; 1236 }; 1237 1238 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1239 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1240 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1241 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1242 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1243 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1244 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1245 { 0, 0, 0, 0, 0 }, 1246 }; 1247 1248 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1249 { 1250 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1251 1252 while (p && p->chip_device != 0) { 1253 if (pdev->vendor == p->chip_vendor && 1254 pdev->device == p->chip_device && 1255 pdev->subsystem_vendor == p->subsys_vendor && 1256 pdev->subsystem_device == p->subsys_device && 1257 pdev->revision == p->revision) { 1258 return true; 1259 } 1260 ++p; 1261 } 1262 return false; 1263 } 1264 1265 static bool is_raven_kicker(struct amdgpu_device *adev) 1266 { 1267 if (adev->pm.fw_version >= 0x41e2b) 1268 return true; 1269 else 1270 return false; 1271 } 1272 1273 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1274 { 1275 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1276 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1277 1278 switch (adev->asic_type) { 1279 case CHIP_VEGA10: 1280 case CHIP_VEGA12: 1281 case CHIP_VEGA20: 1282 break; 1283 case CHIP_RAVEN: 1284 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1285 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1286 ((!is_raven_kicker(adev) && 1287 adev->gfx.rlc_fw_version < 531) || 1288 (adev->gfx.rlc_feature_version < 1) || 1289 !adev->gfx.rlc.is_rlc_v2_1)) 1290 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1291 1292 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1293 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1294 AMD_PG_SUPPORT_CP | 1295 AMD_PG_SUPPORT_RLC_SMU_HS; 1296 break; 1297 case CHIP_RENOIR: 1298 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1299 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1300 AMD_PG_SUPPORT_CP | 1301 AMD_PG_SUPPORT_RLC_SMU_HS; 1302 break; 1303 default: 1304 break; 1305 } 1306 } 1307 1308 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1309 const char *chip_name) 1310 { 1311 char fw_name[30]; 1312 int err; 1313 struct amdgpu_firmware_info *info = NULL; 1314 const struct common_firmware_header *header = NULL; 1315 const struct gfx_firmware_header_v1_0 *cp_hdr; 1316 1317 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name); 1318 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev); 1319 if (err) 1320 goto out; 1321 err = amdgpu_ucode_validate(adev->gfx.pfp_fw); 1322 if (err) 1323 goto out; 1324 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data; 1325 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1326 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1327 1328 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name); 1329 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev); 1330 if (err) 1331 goto out; 1332 err = amdgpu_ucode_validate(adev->gfx.me_fw); 1333 if (err) 1334 goto out; 1335 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data; 1336 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1337 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1338 1339 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name); 1340 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev); 1341 if (err) 1342 goto out; 1343 err = amdgpu_ucode_validate(adev->gfx.ce_fw); 1344 if (err) 1345 goto out; 1346 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data; 1347 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1348 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1349 1350 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1351 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP]; 1352 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP; 1353 info->fw = adev->gfx.pfp_fw; 1354 header = (const struct common_firmware_header *)info->fw->data; 1355 adev->firmware.fw_size += 1356 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1357 1358 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME]; 1359 info->ucode_id = AMDGPU_UCODE_ID_CP_ME; 1360 info->fw = adev->gfx.me_fw; 1361 header = (const struct common_firmware_header *)info->fw->data; 1362 adev->firmware.fw_size += 1363 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1364 1365 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE]; 1366 info->ucode_id = AMDGPU_UCODE_ID_CP_CE; 1367 info->fw = adev->gfx.ce_fw; 1368 header = (const struct common_firmware_header *)info->fw->data; 1369 adev->firmware.fw_size += 1370 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1371 } 1372 1373 out: 1374 if (err) { 1375 dev_err(adev->dev, 1376 "gfx9: Failed to load firmware \"%s\"\n", 1377 fw_name); 1378 release_firmware(adev->gfx.pfp_fw); 1379 adev->gfx.pfp_fw = NULL; 1380 release_firmware(adev->gfx.me_fw); 1381 adev->gfx.me_fw = NULL; 1382 release_firmware(adev->gfx.ce_fw); 1383 adev->gfx.ce_fw = NULL; 1384 } 1385 return err; 1386 } 1387 1388 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1389 const char *chip_name) 1390 { 1391 char fw_name[30]; 1392 int err; 1393 struct amdgpu_firmware_info *info = NULL; 1394 const struct common_firmware_header *header = NULL; 1395 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1396 unsigned int *tmp = NULL; 1397 unsigned int i = 0; 1398 uint16_t version_major; 1399 uint16_t version_minor; 1400 uint32_t smu_version; 1401 1402 /* 1403 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1404 * instead of picasso_rlc.bin. 1405 * Judgment method: 1406 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1407 * or revision >= 0xD8 && revision <= 0xDF 1408 * otherwise is PCO FP5 1409 */ 1410 if (!strcmp(chip_name, "picasso") && 1411 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1412 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1413 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name); 1414 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1415 (smu_version >= 0x41e2b)) 1416 /** 1417 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1418 */ 1419 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name); 1420 else 1421 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name); 1422 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev); 1423 if (err) 1424 goto out; 1425 err = amdgpu_ucode_validate(adev->gfx.rlc_fw); 1426 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1427 1428 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1429 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1430 if (version_major == 2 && version_minor == 1) 1431 adev->gfx.rlc.is_rlc_v2_1 = true; 1432 1433 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version); 1434 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version); 1435 adev->gfx.rlc.save_and_restore_offset = 1436 le32_to_cpu(rlc_hdr->save_and_restore_offset); 1437 adev->gfx.rlc.clear_state_descriptor_offset = 1438 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset); 1439 adev->gfx.rlc.avail_scratch_ram_locations = 1440 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations); 1441 adev->gfx.rlc.reg_restore_list_size = 1442 le32_to_cpu(rlc_hdr->reg_restore_list_size); 1443 adev->gfx.rlc.reg_list_format_start = 1444 le32_to_cpu(rlc_hdr->reg_list_format_start); 1445 adev->gfx.rlc.reg_list_format_separate_start = 1446 le32_to_cpu(rlc_hdr->reg_list_format_separate_start); 1447 adev->gfx.rlc.starting_offsets_start = 1448 le32_to_cpu(rlc_hdr->starting_offsets_start); 1449 adev->gfx.rlc.reg_list_format_size_bytes = 1450 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes); 1451 adev->gfx.rlc.reg_list_size_bytes = 1452 le32_to_cpu(rlc_hdr->reg_list_size_bytes); 1453 adev->gfx.rlc.register_list_format = 1454 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes + 1455 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL); 1456 if (!adev->gfx.rlc.register_list_format) { 1457 err = -ENOMEM; 1458 goto out; 1459 } 1460 1461 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1462 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes)); 1463 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++) 1464 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]); 1465 1466 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i; 1467 1468 tmp = (unsigned int *)((uintptr_t)rlc_hdr + 1469 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes)); 1470 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++) 1471 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]); 1472 1473 if (adev->gfx.rlc.is_rlc_v2_1) 1474 gfx_v9_0_init_rlc_ext_microcode(adev); 1475 1476 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1477 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G]; 1478 info->ucode_id = AMDGPU_UCODE_ID_RLC_G; 1479 info->fw = adev->gfx.rlc_fw; 1480 header = (const struct common_firmware_header *)info->fw->data; 1481 adev->firmware.fw_size += 1482 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE); 1483 1484 if (adev->gfx.rlc.is_rlc_v2_1 && 1485 adev->gfx.rlc.save_restore_list_cntl_size_bytes && 1486 adev->gfx.rlc.save_restore_list_gpm_size_bytes && 1487 adev->gfx.rlc.save_restore_list_srm_size_bytes) { 1488 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL]; 1489 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL; 1490 info->fw = adev->gfx.rlc_fw; 1491 adev->firmware.fw_size += 1492 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE); 1493 1494 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM]; 1495 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM; 1496 info->fw = adev->gfx.rlc_fw; 1497 adev->firmware.fw_size += 1498 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE); 1499 1500 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM]; 1501 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM; 1502 info->fw = adev->gfx.rlc_fw; 1503 adev->firmware.fw_size += 1504 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE); 1505 } 1506 } 1507 1508 out: 1509 if (err) { 1510 dev_err(adev->dev, 1511 "gfx9: Failed to load firmware \"%s\"\n", 1512 fw_name); 1513 release_firmware(adev->gfx.rlc_fw); 1514 adev->gfx.rlc_fw = NULL; 1515 } 1516 return err; 1517 } 1518 1519 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1520 const char *chip_name) 1521 { 1522 char fw_name[30]; 1523 int err; 1524 struct amdgpu_firmware_info *info = NULL; 1525 const struct common_firmware_header *header = NULL; 1526 const struct gfx_firmware_header_v1_0 *cp_hdr; 1527 1528 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name); 1529 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev); 1530 if (err) 1531 goto out; 1532 err = amdgpu_ucode_validate(adev->gfx.mec_fw); 1533 if (err) 1534 goto out; 1535 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1536 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version); 1537 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version); 1538 1539 1540 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name); 1541 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev); 1542 if (!err) { 1543 err = amdgpu_ucode_validate(adev->gfx.mec2_fw); 1544 if (err) 1545 goto out; 1546 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1547 adev->gfx.mec2_fw->data; 1548 adev->gfx.mec2_fw_version = 1549 le32_to_cpu(cp_hdr->header.ucode_version); 1550 adev->gfx.mec2_feature_version = 1551 le32_to_cpu(cp_hdr->ucode_feature_version); 1552 } else { 1553 err = 0; 1554 adev->gfx.mec2_fw = NULL; 1555 } 1556 1557 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1558 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1]; 1559 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1; 1560 info->fw = adev->gfx.mec_fw; 1561 header = (const struct common_firmware_header *)info->fw->data; 1562 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1563 adev->firmware.fw_size += 1564 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1565 1566 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT]; 1567 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT; 1568 info->fw = adev->gfx.mec_fw; 1569 adev->firmware.fw_size += 1570 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1571 1572 if (adev->gfx.mec2_fw) { 1573 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2]; 1574 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2; 1575 info->fw = adev->gfx.mec2_fw; 1576 header = (const struct common_firmware_header *)info->fw->data; 1577 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data; 1578 adev->firmware.fw_size += 1579 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE); 1580 1581 /* TODO: Determine if MEC2 JT FW loading can be removed 1582 for all GFX V9 asic and above */ 1583 if (adev->asic_type != CHIP_ARCTURUS && 1584 adev->asic_type != CHIP_RENOIR) { 1585 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT]; 1586 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT; 1587 info->fw = adev->gfx.mec2_fw; 1588 adev->firmware.fw_size += 1589 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, 1590 PAGE_SIZE); 1591 } 1592 } 1593 } 1594 1595 out: 1596 gfx_v9_0_check_if_need_gfxoff(adev); 1597 gfx_v9_0_check_fw_write_wait(adev); 1598 if (err) { 1599 dev_err(adev->dev, 1600 "gfx9: Failed to load firmware \"%s\"\n", 1601 fw_name); 1602 release_firmware(adev->gfx.mec_fw); 1603 adev->gfx.mec_fw = NULL; 1604 release_firmware(adev->gfx.mec2_fw); 1605 adev->gfx.mec2_fw = NULL; 1606 } 1607 return err; 1608 } 1609 1610 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1611 { 1612 const char *chip_name; 1613 int r; 1614 1615 DRM_DEBUG("\n"); 1616 1617 switch (adev->asic_type) { 1618 case CHIP_VEGA10: 1619 chip_name = "vega10"; 1620 break; 1621 case CHIP_VEGA12: 1622 chip_name = "vega12"; 1623 break; 1624 case CHIP_VEGA20: 1625 chip_name = "vega20"; 1626 break; 1627 case CHIP_RAVEN: 1628 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1629 chip_name = "raven2"; 1630 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1631 chip_name = "picasso"; 1632 else 1633 chip_name = "raven"; 1634 break; 1635 case CHIP_ARCTURUS: 1636 chip_name = "arcturus"; 1637 break; 1638 case CHIP_RENOIR: 1639 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1640 chip_name = "renoir"; 1641 else 1642 chip_name = "green_sardine"; 1643 break; 1644 default: 1645 BUG(); 1646 } 1647 1648 /* No CPG in Arcturus */ 1649 if (adev->gfx.num_gfx_rings) { 1650 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name); 1651 if (r) 1652 return r; 1653 } 1654 1655 r = gfx_v9_0_init_rlc_microcode(adev, chip_name); 1656 if (r) 1657 return r; 1658 1659 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name); 1660 if (r) 1661 return r; 1662 1663 return r; 1664 } 1665 1666 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1667 { 1668 u32 count = 0; 1669 const struct cs_section_def *sect = NULL; 1670 const struct cs_extent_def *ext = NULL; 1671 1672 /* begin clear state */ 1673 count += 2; 1674 /* context control state */ 1675 count += 3; 1676 1677 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1678 for (ext = sect->section; ext->extent != NULL; ++ext) { 1679 if (sect->id == SECT_CONTEXT) 1680 count += 2 + ext->reg_count; 1681 else 1682 return 0; 1683 } 1684 } 1685 1686 /* end clear state */ 1687 count += 2; 1688 /* clear state */ 1689 count += 2; 1690 1691 return count; 1692 } 1693 1694 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, 1695 volatile u32 *buffer) 1696 { 1697 u32 count = 0, i; 1698 const struct cs_section_def *sect = NULL; 1699 const struct cs_extent_def *ext = NULL; 1700 1701 if (adev->gfx.rlc.cs_data == NULL) 1702 return; 1703 if (buffer == NULL) 1704 return; 1705 1706 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1707 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 1708 1709 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 1710 buffer[count++] = cpu_to_le32(0x80000000); 1711 buffer[count++] = cpu_to_le32(0x80000000); 1712 1713 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 1714 for (ext = sect->section; ext->extent != NULL; ++ext) { 1715 if (sect->id == SECT_CONTEXT) { 1716 buffer[count++] = 1717 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count)); 1718 buffer[count++] = cpu_to_le32(ext->reg_index - 1719 PACKET3_SET_CONTEXT_REG_START); 1720 for (i = 0; i < ext->reg_count; i++) 1721 buffer[count++] = cpu_to_le32(ext->extent[i]); 1722 } else { 1723 return; 1724 } 1725 } 1726 } 1727 1728 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 1729 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE); 1730 1731 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0)); 1732 buffer[count++] = cpu_to_le32(0); 1733 } 1734 1735 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1736 { 1737 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1738 uint32_t pg_always_on_cu_num = 2; 1739 uint32_t always_on_cu_num; 1740 uint32_t i, j, k; 1741 uint32_t mask, cu_bitmap, counter; 1742 1743 if (adev->flags & AMD_IS_APU) 1744 always_on_cu_num = 4; 1745 else if (adev->asic_type == CHIP_VEGA12) 1746 always_on_cu_num = 8; 1747 else 1748 always_on_cu_num = 12; 1749 1750 mutex_lock(&adev->grbm_idx_mutex); 1751 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1752 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1753 mask = 1; 1754 cu_bitmap = 0; 1755 counter = 0; 1756 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 1757 1758 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1759 if (cu_info->bitmap[i][j] & mask) { 1760 if (counter == pg_always_on_cu_num) 1761 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1762 if (counter < always_on_cu_num) 1763 cu_bitmap |= mask; 1764 else 1765 break; 1766 counter++; 1767 } 1768 mask <<= 1; 1769 } 1770 1771 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1772 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1773 } 1774 } 1775 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1776 mutex_unlock(&adev->grbm_idx_mutex); 1777 } 1778 1779 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1780 { 1781 uint32_t data; 1782 1783 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1784 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1785 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1786 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1787 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1788 1789 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1790 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1791 1792 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1793 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1794 1795 mutex_lock(&adev->grbm_idx_mutex); 1796 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1797 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1798 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1799 1800 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1801 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1802 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1803 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1804 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1805 1806 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1807 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1808 data &= 0x0000FFFF; 1809 data |= 0x00C00000; 1810 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1811 1812 /* 1813 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1814 * programmed in gfx_v9_0_init_always_on_cu_mask() 1815 */ 1816 1817 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1818 * but used for RLC_LB_CNTL configuration */ 1819 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1820 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1821 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1822 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1823 mutex_unlock(&adev->grbm_idx_mutex); 1824 1825 gfx_v9_0_init_always_on_cu_mask(adev); 1826 } 1827 1828 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1829 { 1830 uint32_t data; 1831 1832 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1833 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1834 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1835 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1836 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1837 1838 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1839 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1840 1841 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1842 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1843 1844 mutex_lock(&adev->grbm_idx_mutex); 1845 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1846 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 1847 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1848 1849 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1850 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1851 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1852 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1853 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1854 1855 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1856 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1857 data &= 0x0000FFFF; 1858 data |= 0x00C00000; 1859 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1860 1861 /* 1862 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1863 * programmed in gfx_v9_0_init_always_on_cu_mask() 1864 */ 1865 1866 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1867 * but used for RLC_LB_CNTL configuration */ 1868 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1869 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1870 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1871 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1872 mutex_unlock(&adev->grbm_idx_mutex); 1873 1874 gfx_v9_0_init_always_on_cu_mask(adev); 1875 } 1876 1877 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1878 { 1879 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1880 } 1881 1882 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1883 { 1884 return 5; 1885 } 1886 1887 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1888 { 1889 const struct cs_section_def *cs_data; 1890 int r; 1891 1892 adev->gfx.rlc.cs_data = gfx9_cs_data; 1893 1894 cs_data = adev->gfx.rlc.cs_data; 1895 1896 if (cs_data) { 1897 /* init clear state block */ 1898 r = amdgpu_gfx_rlc_init_csb(adev); 1899 if (r) 1900 return r; 1901 } 1902 1903 if (adev->flags & AMD_IS_APU) { 1904 /* TODO: double check the cp_table_size for RV */ 1905 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1906 r = amdgpu_gfx_rlc_init_cpt(adev); 1907 if (r) 1908 return r; 1909 } 1910 1911 switch (adev->asic_type) { 1912 case CHIP_RAVEN: 1913 gfx_v9_0_init_lbpw(adev); 1914 break; 1915 case CHIP_VEGA20: 1916 gfx_v9_4_init_lbpw(adev); 1917 break; 1918 default: 1919 break; 1920 } 1921 1922 /* init spm vmid with 0xf */ 1923 if (adev->gfx.rlc.funcs->update_spm_vmid) 1924 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf); 1925 1926 return 0; 1927 } 1928 1929 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1930 { 1931 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1932 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1933 } 1934 1935 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1936 { 1937 int r; 1938 u32 *hpd; 1939 const __le32 *fw_data; 1940 unsigned fw_size; 1941 u32 *fw; 1942 size_t mec_hpd_size; 1943 1944 const struct gfx_firmware_header_v1_0 *mec_hdr; 1945 1946 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1947 1948 /* take ownership of the relevant compute queues */ 1949 amdgpu_gfx_compute_queue_acquire(adev); 1950 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1951 if (mec_hpd_size) { 1952 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1953 AMDGPU_GEM_DOMAIN_VRAM, 1954 &adev->gfx.mec.hpd_eop_obj, 1955 &adev->gfx.mec.hpd_eop_gpu_addr, 1956 (void **)&hpd); 1957 if (r) { 1958 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1959 gfx_v9_0_mec_fini(adev); 1960 return r; 1961 } 1962 1963 memset(hpd, 0, mec_hpd_size); 1964 1965 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1966 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1967 } 1968 1969 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1970 1971 fw_data = (const __le32 *) 1972 (adev->gfx.mec_fw->data + 1973 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1974 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1975 1976 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1977 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1978 &adev->gfx.mec.mec_fw_obj, 1979 &adev->gfx.mec.mec_fw_gpu_addr, 1980 (void **)&fw); 1981 if (r) { 1982 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1983 gfx_v9_0_mec_fini(adev); 1984 return r; 1985 } 1986 1987 memcpy(fw, fw_data, fw_size); 1988 1989 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1990 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1991 1992 return 0; 1993 } 1994 1995 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1996 { 1997 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1998 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1999 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2000 (address << SQ_IND_INDEX__INDEX__SHIFT) | 2001 (SQ_IND_INDEX__FORCE_READ_MASK)); 2002 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2003 } 2004 2005 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 2006 uint32_t wave, uint32_t thread, 2007 uint32_t regno, uint32_t num, uint32_t *out) 2008 { 2009 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 2010 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 2011 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 2012 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 2013 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 2014 (SQ_IND_INDEX__FORCE_READ_MASK) | 2015 (SQ_IND_INDEX__AUTO_INCR_MASK)); 2016 while (num--) 2017 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 2018 } 2019 2020 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 2021 { 2022 /* type 1 wave data */ 2023 dst[(*no_fields)++] = 1; 2024 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 2025 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 2026 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 2027 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 2028 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 2029 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 2030 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 2031 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 2032 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 2033 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 2034 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 2035 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 2036 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 2037 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 2038 } 2039 2040 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd, 2041 uint32_t wave, uint32_t start, 2042 uint32_t size, uint32_t *dst) 2043 { 2044 wave_read_regs( 2045 adev, simd, wave, 0, 2046 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 2047 } 2048 2049 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd, 2050 uint32_t wave, uint32_t thread, 2051 uint32_t start, uint32_t size, 2052 uint32_t *dst) 2053 { 2054 wave_read_regs( 2055 adev, simd, wave, thread, 2056 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 2057 } 2058 2059 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 2060 u32 me, u32 pipe, u32 q, u32 vm) 2061 { 2062 soc15_grbm_select(adev, me, pipe, q, vm); 2063 } 2064 2065 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2066 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2067 .select_se_sh = &gfx_v9_0_select_se_sh, 2068 .read_wave_data = &gfx_v9_0_read_wave_data, 2069 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2070 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2071 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2072 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2073 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2074 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2075 }; 2076 2077 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = { 2078 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2079 .select_se_sh = &gfx_v9_0_select_se_sh, 2080 .read_wave_data = &gfx_v9_0_read_wave_data, 2081 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2082 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2083 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2084 .ras_error_inject = &gfx_v9_4_ras_error_inject, 2085 .query_ras_error_count = &gfx_v9_4_query_ras_error_count, 2086 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count, 2087 .query_ras_error_status = &gfx_v9_4_query_ras_error_status, 2088 }; 2089 2090 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2091 { 2092 u32 gb_addr_config; 2093 int err; 2094 2095 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 2096 2097 switch (adev->asic_type) { 2098 case CHIP_VEGA10: 2099 adev->gfx.config.max_hw_contexts = 8; 2100 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2101 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2102 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2103 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2104 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2105 break; 2106 case CHIP_VEGA12: 2107 adev->gfx.config.max_hw_contexts = 8; 2108 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2109 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2110 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2111 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2112 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2113 DRM_INFO("fix gfx.config for vega12\n"); 2114 break; 2115 case CHIP_VEGA20: 2116 adev->gfx.config.max_hw_contexts = 8; 2117 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2118 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2119 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2120 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2121 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2122 gb_addr_config &= ~0xf3e777ff; 2123 gb_addr_config |= 0x22014042; 2124 /* check vbios table if gpu info is not available */ 2125 err = amdgpu_atomfirmware_get_gfx_info(adev); 2126 if (err) 2127 return err; 2128 break; 2129 case CHIP_RAVEN: 2130 adev->gfx.config.max_hw_contexts = 8; 2131 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2132 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2133 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2134 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2135 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2136 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2137 else 2138 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2139 break; 2140 case CHIP_ARCTURUS: 2141 adev->gfx.funcs = &gfx_v9_4_gfx_funcs; 2142 adev->gfx.config.max_hw_contexts = 8; 2143 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2144 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2145 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2146 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2147 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2148 gb_addr_config &= ~0xf3e777ff; 2149 gb_addr_config |= 0x22014042; 2150 break; 2151 case CHIP_RENOIR: 2152 adev->gfx.config.max_hw_contexts = 8; 2153 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2154 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2155 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2156 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2157 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2158 gb_addr_config &= ~0xf3e777ff; 2159 gb_addr_config |= 0x22010042; 2160 break; 2161 default: 2162 BUG(); 2163 break; 2164 } 2165 2166 adev->gfx.config.gb_addr_config = gb_addr_config; 2167 2168 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2169 REG_GET_FIELD( 2170 adev->gfx.config.gb_addr_config, 2171 GB_ADDR_CONFIG, 2172 NUM_PIPES); 2173 2174 adev->gfx.config.max_tile_pipes = 2175 adev->gfx.config.gb_addr_config_fields.num_pipes; 2176 2177 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2178 REG_GET_FIELD( 2179 adev->gfx.config.gb_addr_config, 2180 GB_ADDR_CONFIG, 2181 NUM_BANKS); 2182 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2183 REG_GET_FIELD( 2184 adev->gfx.config.gb_addr_config, 2185 GB_ADDR_CONFIG, 2186 MAX_COMPRESSED_FRAGS); 2187 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2188 REG_GET_FIELD( 2189 adev->gfx.config.gb_addr_config, 2190 GB_ADDR_CONFIG, 2191 NUM_RB_PER_SE); 2192 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2193 REG_GET_FIELD( 2194 adev->gfx.config.gb_addr_config, 2195 GB_ADDR_CONFIG, 2196 NUM_SHADER_ENGINES); 2197 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2198 REG_GET_FIELD( 2199 adev->gfx.config.gb_addr_config, 2200 GB_ADDR_CONFIG, 2201 PIPE_INTERLEAVE_SIZE)); 2202 2203 return 0; 2204 } 2205 2206 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2207 int mec, int pipe, int queue) 2208 { 2209 unsigned irq_type; 2210 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2211 unsigned int hw_prio; 2212 2213 ring = &adev->gfx.compute_ring[ring_id]; 2214 2215 /* mec0 is me1 */ 2216 ring->me = mec + 1; 2217 ring->pipe = pipe; 2218 ring->queue = queue; 2219 2220 ring->ring_obj = NULL; 2221 ring->use_doorbell = true; 2222 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2223 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2224 + (ring_id * GFX9_MEC_HPD_SIZE); 2225 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2226 2227 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2228 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2229 + ring->pipe; 2230 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring->pipe, 2231 ring->queue) ? 2232 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 2233 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2234 return amdgpu_ring_init(adev, ring, 1024, 2235 &adev->gfx.eop_irq, irq_type, hw_prio); 2236 } 2237 2238 static int gfx_v9_0_sw_init(void *handle) 2239 { 2240 int i, j, k, r, ring_id; 2241 struct amdgpu_ring *ring; 2242 struct amdgpu_kiq *kiq; 2243 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2244 2245 switch (adev->asic_type) { 2246 case CHIP_VEGA10: 2247 case CHIP_VEGA12: 2248 case CHIP_VEGA20: 2249 case CHIP_RAVEN: 2250 case CHIP_ARCTURUS: 2251 case CHIP_RENOIR: 2252 adev->gfx.mec.num_mec = 2; 2253 break; 2254 default: 2255 adev->gfx.mec.num_mec = 1; 2256 break; 2257 } 2258 2259 adev->gfx.mec.num_pipe_per_mec = 4; 2260 adev->gfx.mec.num_queue_per_pipe = 8; 2261 2262 /* EOP Event */ 2263 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2264 if (r) 2265 return r; 2266 2267 /* Privileged reg */ 2268 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2269 &adev->gfx.priv_reg_irq); 2270 if (r) 2271 return r; 2272 2273 /* Privileged inst */ 2274 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2275 &adev->gfx.priv_inst_irq); 2276 if (r) 2277 return r; 2278 2279 /* ECC error */ 2280 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2281 &adev->gfx.cp_ecc_error_irq); 2282 if (r) 2283 return r; 2284 2285 /* FUE error */ 2286 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2287 &adev->gfx.cp_ecc_error_irq); 2288 if (r) 2289 return r; 2290 2291 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2292 2293 gfx_v9_0_scratch_init(adev); 2294 2295 r = gfx_v9_0_init_microcode(adev); 2296 if (r) { 2297 DRM_ERROR("Failed to load gfx firmware!\n"); 2298 return r; 2299 } 2300 2301 r = adev->gfx.rlc.funcs->init(adev); 2302 if (r) { 2303 DRM_ERROR("Failed to init rlc BOs!\n"); 2304 return r; 2305 } 2306 2307 r = gfx_v9_0_mec_init(adev); 2308 if (r) { 2309 DRM_ERROR("Failed to init MEC BOs!\n"); 2310 return r; 2311 } 2312 2313 /* set up the gfx ring */ 2314 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2315 ring = &adev->gfx.gfx_ring[i]; 2316 ring->ring_obj = NULL; 2317 if (!i) 2318 sprintf(ring->name, "gfx"); 2319 else 2320 sprintf(ring->name, "gfx_%d", i); 2321 ring->use_doorbell = true; 2322 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2323 r = amdgpu_ring_init(adev, ring, 1024, 2324 &adev->gfx.eop_irq, 2325 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2326 AMDGPU_RING_PRIO_DEFAULT); 2327 if (r) 2328 return r; 2329 } 2330 2331 /* set up the compute queues - allocate horizontally across pipes */ 2332 ring_id = 0; 2333 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2334 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2335 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2336 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j)) 2337 continue; 2338 2339 r = gfx_v9_0_compute_ring_init(adev, 2340 ring_id, 2341 i, k, j); 2342 if (r) 2343 return r; 2344 2345 ring_id++; 2346 } 2347 } 2348 } 2349 2350 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE); 2351 if (r) { 2352 DRM_ERROR("Failed to init KIQ BOs!\n"); 2353 return r; 2354 } 2355 2356 kiq = &adev->gfx.kiq; 2357 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq); 2358 if (r) 2359 return r; 2360 2361 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2362 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation)); 2363 if (r) 2364 return r; 2365 2366 adev->gfx.ce_ram_size = 0x8000; 2367 2368 r = gfx_v9_0_gpu_early_init(adev); 2369 if (r) 2370 return r; 2371 2372 return 0; 2373 } 2374 2375 2376 static int gfx_v9_0_sw_fini(void *handle) 2377 { 2378 int i; 2379 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2380 2381 amdgpu_gfx_ras_fini(adev); 2382 2383 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2384 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2385 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2386 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2387 2388 amdgpu_gfx_mqd_sw_fini(adev); 2389 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring); 2390 amdgpu_gfx_kiq_fini(adev); 2391 2392 gfx_v9_0_mec_fini(adev); 2393 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj); 2394 if (adev->flags & AMD_IS_APU) { 2395 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2396 &adev->gfx.rlc.cp_table_gpu_addr, 2397 (void **)&adev->gfx.rlc.cp_table_ptr); 2398 } 2399 gfx_v9_0_free_microcode(adev); 2400 2401 return 0; 2402 } 2403 2404 2405 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2406 { 2407 /* TODO */ 2408 } 2409 2410 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2411 u32 instance) 2412 { 2413 u32 data; 2414 2415 if (instance == 0xffffffff) 2416 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2417 else 2418 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2419 2420 if (se_num == 0xffffffff) 2421 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2422 else 2423 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2424 2425 if (sh_num == 0xffffffff) 2426 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2427 else 2428 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2429 2430 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2431 } 2432 2433 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2434 { 2435 u32 data, mask; 2436 2437 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2438 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2439 2440 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2441 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2442 2443 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2444 adev->gfx.config.max_sh_per_se); 2445 2446 return (~data) & mask; 2447 } 2448 2449 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2450 { 2451 int i, j; 2452 u32 data; 2453 u32 active_rbs = 0; 2454 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2455 adev->gfx.config.max_sh_per_se; 2456 2457 mutex_lock(&adev->grbm_idx_mutex); 2458 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2459 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2460 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2461 data = gfx_v9_0_get_rb_active_bitmap(adev); 2462 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2463 rb_bitmap_width_per_sh); 2464 } 2465 } 2466 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2467 mutex_unlock(&adev->grbm_idx_mutex); 2468 2469 adev->gfx.config.backend_enable_mask = active_rbs; 2470 adev->gfx.config.num_rbs = hweight32(active_rbs); 2471 } 2472 2473 #define DEFAULT_SH_MEM_BASES (0x6000) 2474 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2475 { 2476 int i; 2477 uint32_t sh_mem_config; 2478 uint32_t sh_mem_bases; 2479 2480 /* 2481 * Configure apertures: 2482 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2483 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2484 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2485 */ 2486 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2487 2488 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2489 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2490 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2491 2492 mutex_lock(&adev->srbm_mutex); 2493 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2494 soc15_grbm_select(adev, 0, 0, 0, i); 2495 /* CP and shaders */ 2496 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2497 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2498 } 2499 soc15_grbm_select(adev, 0, 0, 0, 0); 2500 mutex_unlock(&adev->srbm_mutex); 2501 2502 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2503 acccess. These should be enabled by FW for target VMIDs. */ 2504 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2505 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2506 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2507 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2508 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2509 } 2510 } 2511 2512 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2513 { 2514 int vmid; 2515 2516 /* 2517 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2518 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2519 * the driver can enable them for graphics. VMID0 should maintain 2520 * access so that HWS firmware can save/restore entries. 2521 */ 2522 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2523 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2524 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2525 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2526 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2527 } 2528 } 2529 2530 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2531 { 2532 uint32_t tmp; 2533 2534 switch (adev->asic_type) { 2535 case CHIP_ARCTURUS: 2536 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2537 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, 2538 DISABLE_BARRIER_WAITCNT, 1); 2539 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2540 break; 2541 default: 2542 break; 2543 } 2544 } 2545 2546 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2547 { 2548 u32 tmp; 2549 int i; 2550 2551 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2552 2553 gfx_v9_0_tiling_mode_table_init(adev); 2554 2555 gfx_v9_0_setup_rb(adev); 2556 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2557 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2558 2559 /* XXX SH_MEM regs */ 2560 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2561 mutex_lock(&adev->srbm_mutex); 2562 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) { 2563 soc15_grbm_select(adev, 0, 0, 0, i); 2564 /* CP and shaders */ 2565 if (i == 0) { 2566 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2567 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2568 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2569 !!adev->gmc.noretry); 2570 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2571 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2572 } else { 2573 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2574 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2575 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2576 !!adev->gmc.noretry); 2577 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2578 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2579 (adev->gmc.private_aperture_start >> 48)); 2580 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2581 (adev->gmc.shared_aperture_start >> 48)); 2582 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2583 } 2584 } 2585 soc15_grbm_select(adev, 0, 0, 0, 0); 2586 2587 mutex_unlock(&adev->srbm_mutex); 2588 2589 gfx_v9_0_init_compute_vmid(adev); 2590 gfx_v9_0_init_gds_vmid(adev); 2591 gfx_v9_0_init_sq_config(adev); 2592 } 2593 2594 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2595 { 2596 u32 i, j, k; 2597 u32 mask; 2598 2599 mutex_lock(&adev->grbm_idx_mutex); 2600 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2601 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2602 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 2603 for (k = 0; k < adev->usec_timeout; k++) { 2604 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2605 break; 2606 udelay(1); 2607 } 2608 if (k == adev->usec_timeout) { 2609 gfx_v9_0_select_se_sh(adev, 0xffffffff, 2610 0xffffffff, 0xffffffff); 2611 mutex_unlock(&adev->grbm_idx_mutex); 2612 DRM_INFO("Timeout wait for RLC serdes %u,%u\n", 2613 i, j); 2614 return; 2615 } 2616 } 2617 } 2618 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 2619 mutex_unlock(&adev->grbm_idx_mutex); 2620 2621 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2622 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2623 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2624 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2625 for (k = 0; k < adev->usec_timeout; k++) { 2626 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2627 break; 2628 udelay(1); 2629 } 2630 } 2631 2632 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2633 bool enable) 2634 { 2635 u32 tmp; 2636 2637 /* don't toggle interrupts that are only applicable 2638 * to me0 pipe0 on AISCs that have me0 removed */ 2639 if (!adev->gfx.num_gfx_rings) 2640 return; 2641 2642 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2643 2644 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2645 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2646 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2647 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2648 2649 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2650 } 2651 2652 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2653 { 2654 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2655 /* csib */ 2656 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2657 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2658 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2659 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2660 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2661 adev->gfx.rlc.clear_state_size); 2662 } 2663 2664 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2665 int indirect_offset, 2666 int list_size, 2667 int *unique_indirect_regs, 2668 int unique_indirect_reg_count, 2669 int *indirect_start_offsets, 2670 int *indirect_start_offsets_count, 2671 int max_start_offsets_count) 2672 { 2673 int idx; 2674 2675 for (; indirect_offset < list_size; indirect_offset++) { 2676 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2677 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2678 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2679 2680 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2681 indirect_offset += 2; 2682 2683 /* look for the matching indice */ 2684 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2685 if (unique_indirect_regs[idx] == 2686 register_list_format[indirect_offset] || 2687 !unique_indirect_regs[idx]) 2688 break; 2689 } 2690 2691 BUG_ON(idx >= unique_indirect_reg_count); 2692 2693 if (!unique_indirect_regs[idx]) 2694 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2695 2696 indirect_offset++; 2697 } 2698 } 2699 } 2700 2701 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2702 { 2703 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2704 int unique_indirect_reg_count = 0; 2705 2706 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2707 int indirect_start_offsets_count = 0; 2708 2709 int list_size = 0; 2710 int i = 0, j = 0; 2711 u32 tmp = 0; 2712 2713 u32 *register_list_format = 2714 kmemdup(adev->gfx.rlc.register_list_format, 2715 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2716 if (!register_list_format) 2717 return -ENOMEM; 2718 2719 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2720 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2721 gfx_v9_1_parse_ind_reg_list(register_list_format, 2722 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2723 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2724 unique_indirect_regs, 2725 unique_indirect_reg_count, 2726 indirect_start_offsets, 2727 &indirect_start_offsets_count, 2728 ARRAY_SIZE(indirect_start_offsets)); 2729 2730 /* enable auto inc in case it is disabled */ 2731 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2732 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2733 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2734 2735 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2736 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2737 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2738 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2739 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2740 adev->gfx.rlc.register_restore[i]); 2741 2742 /* load indirect register */ 2743 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2744 adev->gfx.rlc.reg_list_format_start); 2745 2746 /* direct register portion */ 2747 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2748 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2749 register_list_format[i]); 2750 2751 /* indirect register portion */ 2752 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2753 if (register_list_format[i] == 0xFFFFFFFF) { 2754 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2755 continue; 2756 } 2757 2758 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2759 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2760 2761 for (j = 0; j < unique_indirect_reg_count; j++) { 2762 if (register_list_format[i] == unique_indirect_regs[j]) { 2763 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2764 break; 2765 } 2766 } 2767 2768 BUG_ON(j >= unique_indirect_reg_count); 2769 2770 i++; 2771 } 2772 2773 /* set save/restore list size */ 2774 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2775 list_size = list_size >> 1; 2776 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2777 adev->gfx.rlc.reg_restore_list_size); 2778 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2779 2780 /* write the starting offsets to RLC scratch ram */ 2781 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2782 adev->gfx.rlc.starting_offsets_start); 2783 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2784 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2785 indirect_start_offsets[i]); 2786 2787 /* load unique indirect regs*/ 2788 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2789 if (unique_indirect_regs[i] != 0) { 2790 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2791 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2792 unique_indirect_regs[i] & 0x3FFFF); 2793 2794 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2795 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2796 unique_indirect_regs[i] >> 20); 2797 } 2798 } 2799 2800 kfree(register_list_format); 2801 return 0; 2802 } 2803 2804 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2805 { 2806 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2807 } 2808 2809 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2810 bool enable) 2811 { 2812 uint32_t data = 0; 2813 uint32_t default_data = 0; 2814 2815 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2816 if (enable) { 2817 /* enable GFXIP control over CGPG */ 2818 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2819 if(default_data != data) 2820 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2821 2822 /* update status */ 2823 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2824 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2825 if(default_data != data) 2826 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2827 } else { 2828 /* restore GFXIP control over GCPG */ 2829 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2830 if(default_data != data) 2831 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2832 } 2833 } 2834 2835 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2836 { 2837 uint32_t data = 0; 2838 2839 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2840 AMD_PG_SUPPORT_GFX_SMG | 2841 AMD_PG_SUPPORT_GFX_DMG)) { 2842 /* init IDLE_POLL_COUNT = 60 */ 2843 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2844 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2845 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2846 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2847 2848 /* init RLC PG Delay */ 2849 data = 0; 2850 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2851 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2852 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2853 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2854 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2855 2856 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2857 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2858 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2859 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2860 2861 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2862 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2863 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2864 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2865 2866 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2867 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2868 2869 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2870 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2871 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2872 if (adev->asic_type != CHIP_RENOIR) 2873 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2874 } 2875 } 2876 2877 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2878 bool enable) 2879 { 2880 uint32_t data = 0; 2881 uint32_t default_data = 0; 2882 2883 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2884 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2885 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 2886 enable ? 1 : 0); 2887 if (default_data != data) 2888 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2889 } 2890 2891 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 2892 bool enable) 2893 { 2894 uint32_t data = 0; 2895 uint32_t default_data = 0; 2896 2897 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2898 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2899 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 2900 enable ? 1 : 0); 2901 if(default_data != data) 2902 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2903 } 2904 2905 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 2906 bool enable) 2907 { 2908 uint32_t data = 0; 2909 uint32_t default_data = 0; 2910 2911 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2912 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2913 CP_PG_DISABLE, 2914 enable ? 0 : 1); 2915 if(default_data != data) 2916 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2917 } 2918 2919 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 2920 bool enable) 2921 { 2922 uint32_t data, default_data; 2923 2924 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2925 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2926 GFX_POWER_GATING_ENABLE, 2927 enable ? 1 : 0); 2928 if(default_data != data) 2929 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2930 } 2931 2932 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 2933 bool enable) 2934 { 2935 uint32_t data, default_data; 2936 2937 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2938 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2939 GFX_PIPELINE_PG_ENABLE, 2940 enable ? 1 : 0); 2941 if(default_data != data) 2942 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2943 2944 if (!enable) 2945 /* read any GFX register to wake up GFX */ 2946 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 2947 } 2948 2949 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 2950 bool enable) 2951 { 2952 uint32_t data, default_data; 2953 2954 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2955 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2956 STATIC_PER_CU_PG_ENABLE, 2957 enable ? 1 : 0); 2958 if(default_data != data) 2959 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2960 } 2961 2962 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 2963 bool enable) 2964 { 2965 uint32_t data, default_data; 2966 2967 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 2968 data = REG_SET_FIELD(data, RLC_PG_CNTL, 2969 DYN_PER_CU_PG_ENABLE, 2970 enable ? 1 : 0); 2971 if(default_data != data) 2972 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 2973 } 2974 2975 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 2976 { 2977 gfx_v9_0_init_csb(adev); 2978 2979 /* 2980 * Rlc save restore list is workable since v2_1. 2981 * And it's needed by gfxoff feature. 2982 */ 2983 if (adev->gfx.rlc.is_rlc_v2_1) { 2984 if (adev->asic_type == CHIP_VEGA12 || 2985 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 2986 gfx_v9_1_init_rlc_save_restore_list(adev); 2987 gfx_v9_0_enable_save_restore_machine(adev); 2988 } 2989 2990 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2991 AMD_PG_SUPPORT_GFX_SMG | 2992 AMD_PG_SUPPORT_GFX_DMG | 2993 AMD_PG_SUPPORT_CP | 2994 AMD_PG_SUPPORT_GDS | 2995 AMD_PG_SUPPORT_RLC_SMU_HS)) { 2996 WREG32(mmRLC_JUMP_TABLE_RESTORE, 2997 adev->gfx.rlc.cp_table_gpu_addr >> 8); 2998 gfx_v9_0_init_gfx_power_gating(adev); 2999 } 3000 } 3001 3002 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3003 { 3004 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3005 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3006 gfx_v9_0_wait_for_rlc_serdes(adev); 3007 } 3008 3009 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3010 { 3011 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3012 udelay(50); 3013 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3014 udelay(50); 3015 } 3016 3017 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3018 { 3019 #ifdef AMDGPU_RLC_DEBUG_RETRY 3020 u32 rlc_ucode_ver; 3021 #endif 3022 3023 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3024 udelay(50); 3025 3026 /* carrizo do enable cp interrupt after cp inited */ 3027 if (!(adev->flags & AMD_IS_APU)) { 3028 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3029 udelay(50); 3030 } 3031 3032 #ifdef AMDGPU_RLC_DEBUG_RETRY 3033 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3034 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3035 if(rlc_ucode_ver == 0x108) { 3036 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n", 3037 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3038 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3039 * default is 0x9C4 to create a 100us interval */ 3040 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3041 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3042 * to disable the page fault retry interrupts, default is 3043 * 0x100 (256) */ 3044 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3045 } 3046 #endif 3047 } 3048 3049 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3050 { 3051 const struct rlc_firmware_header_v2_0 *hdr; 3052 const __le32 *fw_data; 3053 unsigned i, fw_size; 3054 3055 if (!adev->gfx.rlc_fw) 3056 return -EINVAL; 3057 3058 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3059 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3060 3061 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3062 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3063 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3064 3065 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3066 RLCG_UCODE_LOADING_START_ADDRESS); 3067 for (i = 0; i < fw_size; i++) 3068 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3069 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3070 3071 return 0; 3072 } 3073 3074 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3075 { 3076 int r; 3077 3078 if (amdgpu_sriov_vf(adev)) { 3079 gfx_v9_0_init_csb(adev); 3080 return 0; 3081 } 3082 3083 adev->gfx.rlc.funcs->stop(adev); 3084 3085 /* disable CG */ 3086 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3087 3088 gfx_v9_0_init_pg(adev); 3089 3090 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3091 /* legacy rlc firmware loading */ 3092 r = gfx_v9_0_rlc_load_microcode(adev); 3093 if (r) 3094 return r; 3095 } 3096 3097 switch (adev->asic_type) { 3098 case CHIP_RAVEN: 3099 if (amdgpu_lbpw == 0) 3100 gfx_v9_0_enable_lbpw(adev, false); 3101 else 3102 gfx_v9_0_enable_lbpw(adev, true); 3103 break; 3104 case CHIP_VEGA20: 3105 if (amdgpu_lbpw > 0) 3106 gfx_v9_0_enable_lbpw(adev, true); 3107 else 3108 gfx_v9_0_enable_lbpw(adev, false); 3109 break; 3110 default: 3111 break; 3112 } 3113 3114 adev->gfx.rlc.funcs->start(adev); 3115 3116 return 0; 3117 } 3118 3119 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3120 { 3121 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3122 3123 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3124 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3125 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3126 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3127 udelay(50); 3128 } 3129 3130 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3131 { 3132 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3133 const struct gfx_firmware_header_v1_0 *ce_hdr; 3134 const struct gfx_firmware_header_v1_0 *me_hdr; 3135 const __le32 *fw_data; 3136 unsigned i, fw_size; 3137 3138 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3139 return -EINVAL; 3140 3141 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3142 adev->gfx.pfp_fw->data; 3143 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3144 adev->gfx.ce_fw->data; 3145 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3146 adev->gfx.me_fw->data; 3147 3148 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3149 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3150 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3151 3152 gfx_v9_0_cp_gfx_enable(adev, false); 3153 3154 /* PFP */ 3155 fw_data = (const __le32 *) 3156 (adev->gfx.pfp_fw->data + 3157 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3158 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3159 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3160 for (i = 0; i < fw_size; i++) 3161 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3162 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3163 3164 /* CE */ 3165 fw_data = (const __le32 *) 3166 (adev->gfx.ce_fw->data + 3167 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3168 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3169 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3170 for (i = 0; i < fw_size; i++) 3171 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3172 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3173 3174 /* ME */ 3175 fw_data = (const __le32 *) 3176 (adev->gfx.me_fw->data + 3177 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3178 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3179 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3180 for (i = 0; i < fw_size; i++) 3181 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3182 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3183 3184 return 0; 3185 } 3186 3187 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3188 { 3189 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3190 const struct cs_section_def *sect = NULL; 3191 const struct cs_extent_def *ext = NULL; 3192 int r, i, tmp; 3193 3194 /* init the CP */ 3195 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3196 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3197 3198 gfx_v9_0_cp_gfx_enable(adev, true); 3199 3200 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3201 if (r) { 3202 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r); 3203 return r; 3204 } 3205 3206 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3207 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3208 3209 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3210 amdgpu_ring_write(ring, 0x80000000); 3211 amdgpu_ring_write(ring, 0x80000000); 3212 3213 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3214 for (ext = sect->section; ext->extent != NULL; ++ext) { 3215 if (sect->id == SECT_CONTEXT) { 3216 amdgpu_ring_write(ring, 3217 PACKET3(PACKET3_SET_CONTEXT_REG, 3218 ext->reg_count)); 3219 amdgpu_ring_write(ring, 3220 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3221 for (i = 0; i < ext->reg_count; i++) 3222 amdgpu_ring_write(ring, ext->extent[i]); 3223 } 3224 } 3225 } 3226 3227 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3228 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3229 3230 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3231 amdgpu_ring_write(ring, 0); 3232 3233 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3234 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3235 amdgpu_ring_write(ring, 0x8000); 3236 amdgpu_ring_write(ring, 0x8000); 3237 3238 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3239 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3240 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3241 amdgpu_ring_write(ring, tmp); 3242 amdgpu_ring_write(ring, 0); 3243 3244 amdgpu_ring_commit(ring); 3245 3246 return 0; 3247 } 3248 3249 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3250 { 3251 struct amdgpu_ring *ring; 3252 u32 tmp; 3253 u32 rb_bufsz; 3254 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3255 3256 /* Set the write pointer delay */ 3257 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3258 3259 /* set the RB to use vmid 0 */ 3260 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3261 3262 /* Set ring buffer size */ 3263 ring = &adev->gfx.gfx_ring[0]; 3264 rb_bufsz = order_base_2(ring->ring_size / 8); 3265 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3266 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3267 #ifdef __BIG_ENDIAN 3268 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3269 #endif 3270 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3271 3272 /* Initialize the ring buffer's write pointers */ 3273 ring->wptr = 0; 3274 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3275 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3276 3277 /* set the wb address wether it's enabled or not */ 3278 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3279 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3280 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3281 3282 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3283 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3284 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3285 3286 mdelay(1); 3287 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3288 3289 rb_addr = ring->gpu_addr >> 8; 3290 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3291 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3292 3293 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3294 if (ring->use_doorbell) { 3295 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3296 DOORBELL_OFFSET, ring->doorbell_index); 3297 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3298 DOORBELL_EN, 1); 3299 } else { 3300 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3301 } 3302 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3303 3304 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3305 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3306 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3307 3308 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3309 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3310 3311 3312 /* start the ring */ 3313 gfx_v9_0_cp_gfx_start(adev); 3314 ring->sched.ready = true; 3315 3316 return 0; 3317 } 3318 3319 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3320 { 3321 if (enable) { 3322 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3323 } else { 3324 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3325 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3326 adev->gfx.kiq.ring.sched.ready = false; 3327 } 3328 udelay(50); 3329 } 3330 3331 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3332 { 3333 const struct gfx_firmware_header_v1_0 *mec_hdr; 3334 const __le32 *fw_data; 3335 unsigned i; 3336 u32 tmp; 3337 3338 if (!adev->gfx.mec_fw) 3339 return -EINVAL; 3340 3341 gfx_v9_0_cp_compute_enable(adev, false); 3342 3343 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3344 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3345 3346 fw_data = (const __le32 *) 3347 (adev->gfx.mec_fw->data + 3348 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3349 tmp = 0; 3350 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3351 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3352 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3353 3354 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3355 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3356 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3357 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3358 3359 /* MEC1 */ 3360 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3361 mec_hdr->jt_offset); 3362 for (i = 0; i < mec_hdr->jt_size; i++) 3363 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3364 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3365 3366 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3367 adev->gfx.mec_fw_version); 3368 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3369 3370 return 0; 3371 } 3372 3373 /* KIQ functions */ 3374 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3375 { 3376 uint32_t tmp; 3377 struct amdgpu_device *adev = ring->adev; 3378 3379 /* tell RLC which is KIQ queue */ 3380 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3381 tmp &= 0xffffff00; 3382 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3383 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3384 tmp |= 0x80; 3385 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp); 3386 } 3387 3388 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3389 { 3390 struct amdgpu_device *adev = ring->adev; 3391 3392 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3393 if (amdgpu_gfx_is_high_priority_compute_queue(adev, 3394 ring->pipe, 3395 ring->queue)) { 3396 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3397 mqd->cp_hqd_queue_priority = 3398 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3399 } 3400 } 3401 } 3402 3403 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3404 { 3405 struct amdgpu_device *adev = ring->adev; 3406 struct v9_mqd *mqd = ring->mqd_ptr; 3407 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3408 uint32_t tmp; 3409 3410 mqd->header = 0xC0310800; 3411 mqd->compute_pipelinestat_enable = 0x00000001; 3412 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3413 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3414 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3415 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3416 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3417 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3418 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3419 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3420 mqd->compute_misc_reserved = 0x00000003; 3421 3422 mqd->dynamic_cu_mask_addr_lo = 3423 lower_32_bits(ring->mqd_gpu_addr 3424 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3425 mqd->dynamic_cu_mask_addr_hi = 3426 upper_32_bits(ring->mqd_gpu_addr 3427 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3428 3429 eop_base_addr = ring->eop_gpu_addr >> 8; 3430 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3431 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3432 3433 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3434 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3435 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3436 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3437 3438 mqd->cp_hqd_eop_control = tmp; 3439 3440 /* enable doorbell? */ 3441 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3442 3443 if (ring->use_doorbell) { 3444 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3445 DOORBELL_OFFSET, ring->doorbell_index); 3446 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3447 DOORBELL_EN, 1); 3448 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3449 DOORBELL_SOURCE, 0); 3450 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3451 DOORBELL_HIT, 0); 3452 } else { 3453 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3454 DOORBELL_EN, 0); 3455 } 3456 3457 mqd->cp_hqd_pq_doorbell_control = tmp; 3458 3459 /* disable the queue if it's active */ 3460 ring->wptr = 0; 3461 mqd->cp_hqd_dequeue_request = 0; 3462 mqd->cp_hqd_pq_rptr = 0; 3463 mqd->cp_hqd_pq_wptr_lo = 0; 3464 mqd->cp_hqd_pq_wptr_hi = 0; 3465 3466 /* set the pointer to the MQD */ 3467 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3468 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3469 3470 /* set MQD vmid to 0 */ 3471 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3472 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3473 mqd->cp_mqd_control = tmp; 3474 3475 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3476 hqd_gpu_addr = ring->gpu_addr >> 8; 3477 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3478 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3479 3480 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3481 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3482 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3483 (order_base_2(ring->ring_size / 4) - 1)); 3484 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3485 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8)); 3486 #ifdef __BIG_ENDIAN 3487 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3488 #endif 3489 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3490 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3491 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3492 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3493 mqd->cp_hqd_pq_control = tmp; 3494 3495 /* set the wb address whether it's enabled or not */ 3496 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4); 3497 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3498 mqd->cp_hqd_pq_rptr_report_addr_hi = 3499 upper_32_bits(wb_gpu_addr) & 0xffff; 3500 3501 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3502 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4); 3503 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3504 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3505 3506 tmp = 0; 3507 /* enable the doorbell if requested */ 3508 if (ring->use_doorbell) { 3509 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3510 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3511 DOORBELL_OFFSET, ring->doorbell_index); 3512 3513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3514 DOORBELL_EN, 1); 3515 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3516 DOORBELL_SOURCE, 0); 3517 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3518 DOORBELL_HIT, 0); 3519 } 3520 3521 mqd->cp_hqd_pq_doorbell_control = tmp; 3522 3523 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3524 ring->wptr = 0; 3525 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3526 3527 /* set the vmid for the queue */ 3528 mqd->cp_hqd_vmid = 0; 3529 3530 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3531 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3532 mqd->cp_hqd_persistent_state = tmp; 3533 3534 /* set MIN_IB_AVAIL_SIZE */ 3535 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3536 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3537 mqd->cp_hqd_ib_control = tmp; 3538 3539 /* set static priority for a queue/ring */ 3540 gfx_v9_0_mqd_set_priority(ring, mqd); 3541 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM); 3542 3543 /* map_queues packet doesn't need activate the queue, 3544 * so only kiq need set this field. 3545 */ 3546 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3547 mqd->cp_hqd_active = 1; 3548 3549 return 0; 3550 } 3551 3552 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3553 { 3554 struct amdgpu_device *adev = ring->adev; 3555 struct v9_mqd *mqd = ring->mqd_ptr; 3556 int j; 3557 3558 /* disable wptr polling */ 3559 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3560 3561 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3562 mqd->cp_hqd_eop_base_addr_lo); 3563 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3564 mqd->cp_hqd_eop_base_addr_hi); 3565 3566 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3567 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3568 mqd->cp_hqd_eop_control); 3569 3570 /* enable doorbell? */ 3571 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3572 mqd->cp_hqd_pq_doorbell_control); 3573 3574 /* disable the queue if it's active */ 3575 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3576 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3577 for (j = 0; j < adev->usec_timeout; j++) { 3578 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3579 break; 3580 udelay(1); 3581 } 3582 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3583 mqd->cp_hqd_dequeue_request); 3584 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3585 mqd->cp_hqd_pq_rptr); 3586 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3587 mqd->cp_hqd_pq_wptr_lo); 3588 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3589 mqd->cp_hqd_pq_wptr_hi); 3590 } 3591 3592 /* set the pointer to the MQD */ 3593 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3594 mqd->cp_mqd_base_addr_lo); 3595 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3596 mqd->cp_mqd_base_addr_hi); 3597 3598 /* set MQD vmid to 0 */ 3599 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3600 mqd->cp_mqd_control); 3601 3602 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3603 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3604 mqd->cp_hqd_pq_base_lo); 3605 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3606 mqd->cp_hqd_pq_base_hi); 3607 3608 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3609 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3610 mqd->cp_hqd_pq_control); 3611 3612 /* set the wb address whether it's enabled or not */ 3613 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3614 mqd->cp_hqd_pq_rptr_report_addr_lo); 3615 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3616 mqd->cp_hqd_pq_rptr_report_addr_hi); 3617 3618 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3619 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3620 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3621 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3622 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3623 3624 /* enable the doorbell if requested */ 3625 if (ring->use_doorbell) { 3626 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3627 (adev->doorbell_index.kiq * 2) << 2); 3628 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3629 (adev->doorbell_index.userqueue_end * 2) << 2); 3630 } 3631 3632 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3633 mqd->cp_hqd_pq_doorbell_control); 3634 3635 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3637 mqd->cp_hqd_pq_wptr_lo); 3638 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3639 mqd->cp_hqd_pq_wptr_hi); 3640 3641 /* set the vmid for the queue */ 3642 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3643 3644 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3645 mqd->cp_hqd_persistent_state); 3646 3647 /* activate the queue */ 3648 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3649 mqd->cp_hqd_active); 3650 3651 if (ring->use_doorbell) 3652 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3653 3654 return 0; 3655 } 3656 3657 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3658 { 3659 struct amdgpu_device *adev = ring->adev; 3660 int j; 3661 3662 /* disable the queue if it's active */ 3663 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3664 3665 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3666 3667 for (j = 0; j < adev->usec_timeout; j++) { 3668 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3669 break; 3670 udelay(1); 3671 } 3672 3673 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3674 DRM_DEBUG("KIQ dequeue request failed.\n"); 3675 3676 /* Manual disable if dequeue request times out */ 3677 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3678 } 3679 3680 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3681 0); 3682 } 3683 3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3685 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3686 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3687 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3688 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3689 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3690 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3691 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3692 3693 return 0; 3694 } 3695 3696 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3697 { 3698 struct amdgpu_device *adev = ring->adev; 3699 struct v9_mqd *mqd = ring->mqd_ptr; 3700 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS; 3701 3702 gfx_v9_0_kiq_setting(ring); 3703 3704 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3705 /* reset MQD to a clean status */ 3706 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3707 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3708 3709 /* reset ring buffer */ 3710 ring->wptr = 0; 3711 amdgpu_ring_clear_ring(ring); 3712 3713 mutex_lock(&adev->srbm_mutex); 3714 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3715 gfx_v9_0_kiq_init_register(ring); 3716 soc15_grbm_select(adev, 0, 0, 0, 0); 3717 mutex_unlock(&adev->srbm_mutex); 3718 } else { 3719 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3720 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3721 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3722 mutex_lock(&adev->srbm_mutex); 3723 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3724 gfx_v9_0_mqd_init(ring); 3725 gfx_v9_0_kiq_init_register(ring); 3726 soc15_grbm_select(adev, 0, 0, 0, 0); 3727 mutex_unlock(&adev->srbm_mutex); 3728 3729 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3730 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3731 } 3732 3733 return 0; 3734 } 3735 3736 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring) 3737 { 3738 struct amdgpu_device *adev = ring->adev; 3739 struct v9_mqd *mqd = ring->mqd_ptr; 3740 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3741 3742 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3743 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3744 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3745 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3746 mutex_lock(&adev->srbm_mutex); 3747 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 3748 gfx_v9_0_mqd_init(ring); 3749 soc15_grbm_select(adev, 0, 0, 0, 0); 3750 mutex_unlock(&adev->srbm_mutex); 3751 3752 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3753 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3754 } else if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 3755 /* reset MQD to a clean status */ 3756 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3757 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3758 3759 /* reset ring buffer */ 3760 ring->wptr = 0; 3761 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0); 3762 amdgpu_ring_clear_ring(ring); 3763 } else { 3764 amdgpu_ring_clear_ring(ring); 3765 } 3766 3767 return 0; 3768 } 3769 3770 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3771 { 3772 struct amdgpu_ring *ring; 3773 int r; 3774 3775 ring = &adev->gfx.kiq.ring; 3776 3777 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3778 if (unlikely(r != 0)) 3779 return r; 3780 3781 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3782 if (unlikely(r != 0)) 3783 return r; 3784 3785 gfx_v9_0_kiq_init_queue(ring); 3786 amdgpu_bo_kunmap(ring->mqd_obj); 3787 ring->mqd_ptr = NULL; 3788 amdgpu_bo_unreserve(ring->mqd_obj); 3789 ring->sched.ready = true; 3790 return 0; 3791 } 3792 3793 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3794 { 3795 struct amdgpu_ring *ring = NULL; 3796 int r = 0, i; 3797 3798 gfx_v9_0_cp_compute_enable(adev, true); 3799 3800 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3801 ring = &adev->gfx.compute_ring[i]; 3802 3803 r = amdgpu_bo_reserve(ring->mqd_obj, false); 3804 if (unlikely(r != 0)) 3805 goto done; 3806 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 3807 if (!r) { 3808 r = gfx_v9_0_kcq_init_queue(ring); 3809 amdgpu_bo_kunmap(ring->mqd_obj); 3810 ring->mqd_ptr = NULL; 3811 } 3812 amdgpu_bo_unreserve(ring->mqd_obj); 3813 if (r) 3814 goto done; 3815 } 3816 3817 r = amdgpu_gfx_enable_kcq(adev); 3818 done: 3819 return r; 3820 } 3821 3822 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3823 { 3824 int r, i; 3825 struct amdgpu_ring *ring; 3826 3827 if (!(adev->flags & AMD_IS_APU)) 3828 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3829 3830 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3831 if (adev->gfx.num_gfx_rings) { 3832 /* legacy firmware loading */ 3833 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3834 if (r) 3835 return r; 3836 } 3837 3838 r = gfx_v9_0_cp_compute_load_microcode(adev); 3839 if (r) 3840 return r; 3841 } 3842 3843 r = gfx_v9_0_kiq_resume(adev); 3844 if (r) 3845 return r; 3846 3847 if (adev->gfx.num_gfx_rings) { 3848 r = gfx_v9_0_cp_gfx_resume(adev); 3849 if (r) 3850 return r; 3851 } 3852 3853 r = gfx_v9_0_kcq_resume(adev); 3854 if (r) 3855 return r; 3856 3857 if (adev->gfx.num_gfx_rings) { 3858 ring = &adev->gfx.gfx_ring[0]; 3859 r = amdgpu_ring_test_helper(ring); 3860 if (r) 3861 return r; 3862 } 3863 3864 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3865 ring = &adev->gfx.compute_ring[i]; 3866 amdgpu_ring_test_helper(ring); 3867 } 3868 3869 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3870 3871 return 0; 3872 } 3873 3874 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 3875 { 3876 u32 tmp; 3877 3878 if (adev->asic_type != CHIP_ARCTURUS) 3879 return; 3880 3881 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 3882 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 3883 adev->df.hash_status.hash_64k); 3884 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 3885 adev->df.hash_status.hash_2m); 3886 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 3887 adev->df.hash_status.hash_1g); 3888 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 3889 } 3890 3891 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 3892 { 3893 if (adev->gfx.num_gfx_rings) 3894 gfx_v9_0_cp_gfx_enable(adev, enable); 3895 gfx_v9_0_cp_compute_enable(adev, enable); 3896 } 3897 3898 static int gfx_v9_0_hw_init(void *handle) 3899 { 3900 int r; 3901 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3902 3903 if (!amdgpu_sriov_vf(adev)) 3904 gfx_v9_0_init_golden_registers(adev); 3905 3906 gfx_v9_0_constants_init(adev); 3907 3908 gfx_v9_0_init_tcp_config(adev); 3909 3910 r = adev->gfx.rlc.funcs->resume(adev); 3911 if (r) 3912 return r; 3913 3914 r = gfx_v9_0_cp_resume(adev); 3915 if (r) 3916 return r; 3917 3918 return r; 3919 } 3920 3921 static int gfx_v9_0_hw_fini(void *handle) 3922 { 3923 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3924 3925 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0); 3926 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3927 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3928 3929 /* DF freeze and kcq disable will fail */ 3930 if (!amdgpu_ras_intr_triggered()) 3931 /* disable KCQ to avoid CPC touch memory not valid anymore */ 3932 amdgpu_gfx_disable_kcq(adev); 3933 3934 if (amdgpu_sriov_vf(adev)) { 3935 gfx_v9_0_cp_gfx_enable(adev, false); 3936 /* must disable polling for SRIOV when hw finished, otherwise 3937 * CPC engine may still keep fetching WB address which is already 3938 * invalid after sw finished and trigger DMAR reading error in 3939 * hypervisor side. 3940 */ 3941 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3942 return 0; 3943 } 3944 3945 /* Use deinitialize sequence from CAIL when unbinding device from driver, 3946 * otherwise KIQ is hanging when binding back 3947 */ 3948 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 3949 mutex_lock(&adev->srbm_mutex); 3950 soc15_grbm_select(adev, adev->gfx.kiq.ring.me, 3951 adev->gfx.kiq.ring.pipe, 3952 adev->gfx.kiq.ring.queue, 0); 3953 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring); 3954 soc15_grbm_select(adev, 0, 0, 0, 0); 3955 mutex_unlock(&adev->srbm_mutex); 3956 } 3957 3958 gfx_v9_0_cp_enable(adev, false); 3959 adev->gfx.rlc.funcs->stop(adev); 3960 3961 return 0; 3962 } 3963 3964 static int gfx_v9_0_suspend(void *handle) 3965 { 3966 return gfx_v9_0_hw_fini(handle); 3967 } 3968 3969 static int gfx_v9_0_resume(void *handle) 3970 { 3971 return gfx_v9_0_hw_init(handle); 3972 } 3973 3974 static bool gfx_v9_0_is_idle(void *handle) 3975 { 3976 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3977 3978 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 3979 GRBM_STATUS, GUI_ACTIVE)) 3980 return false; 3981 else 3982 return true; 3983 } 3984 3985 static int gfx_v9_0_wait_for_idle(void *handle) 3986 { 3987 unsigned i; 3988 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 3989 3990 for (i = 0; i < adev->usec_timeout; i++) { 3991 if (gfx_v9_0_is_idle(handle)) 3992 return 0; 3993 udelay(1); 3994 } 3995 return -ETIMEDOUT; 3996 } 3997 3998 static int gfx_v9_0_soft_reset(void *handle) 3999 { 4000 u32 grbm_soft_reset = 0; 4001 u32 tmp; 4002 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4003 4004 /* GRBM_STATUS */ 4005 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4006 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4007 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4008 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4009 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4010 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4011 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4012 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4013 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4014 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4015 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4016 } 4017 4018 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4019 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4020 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4021 } 4022 4023 /* GRBM_STATUS2 */ 4024 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4025 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4026 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4027 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4028 4029 4030 if (grbm_soft_reset) { 4031 /* stop the rlc */ 4032 adev->gfx.rlc.funcs->stop(adev); 4033 4034 if (adev->gfx.num_gfx_rings) 4035 /* Disable GFX parsing/prefetching */ 4036 gfx_v9_0_cp_gfx_enable(adev, false); 4037 4038 /* Disable MEC parsing/prefetching */ 4039 gfx_v9_0_cp_compute_enable(adev, false); 4040 4041 if (grbm_soft_reset) { 4042 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4043 tmp |= grbm_soft_reset; 4044 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4045 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4046 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4047 4048 udelay(50); 4049 4050 tmp &= ~grbm_soft_reset; 4051 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4052 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4053 } 4054 4055 /* Wait a little for things to settle down */ 4056 udelay(50); 4057 } 4058 return 0; 4059 } 4060 4061 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4062 { 4063 signed long r, cnt = 0; 4064 unsigned long flags; 4065 uint32_t seq, reg_val_offs = 0; 4066 uint64_t value = 0; 4067 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 4068 struct amdgpu_ring *ring = &kiq->ring; 4069 4070 BUG_ON(!ring->funcs->emit_rreg); 4071 4072 spin_lock_irqsave(&kiq->ring_lock, flags); 4073 if (amdgpu_device_wb_get(adev, ®_val_offs)) { 4074 pr_err("critical bug! too many kiq readers\n"); 4075 goto failed_unlock; 4076 } 4077 amdgpu_ring_alloc(ring, 32); 4078 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4079 amdgpu_ring_write(ring, 9 | /* src: register*/ 4080 (5 << 8) | /* dst: memory */ 4081 (1 << 16) | /* count sel */ 4082 (1 << 20)); /* write confirm */ 4083 amdgpu_ring_write(ring, 0); 4084 amdgpu_ring_write(ring, 0); 4085 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4086 reg_val_offs * 4)); 4087 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4088 reg_val_offs * 4)); 4089 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4090 if (r) 4091 goto failed_undo; 4092 4093 amdgpu_ring_commit(ring); 4094 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4095 4096 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4097 4098 /* don't wait anymore for gpu reset case because this way may 4099 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4100 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4101 * never return if we keep waiting in virt_kiq_rreg, which cause 4102 * gpu_recover() hang there. 4103 * 4104 * also don't wait anymore for IRQ context 4105 * */ 4106 if (r < 1 && (amdgpu_in_reset(adev) || in_interrupt())) 4107 goto failed_kiq_read; 4108 4109 might_sleep(); 4110 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4111 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4112 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4113 } 4114 4115 if (cnt > MAX_KIQ_REG_TRY) 4116 goto failed_kiq_read; 4117 4118 mb(); 4119 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4120 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4121 amdgpu_device_wb_free(adev, reg_val_offs); 4122 return value; 4123 4124 failed_undo: 4125 amdgpu_ring_undo(ring); 4126 failed_unlock: 4127 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4128 failed_kiq_read: 4129 if (reg_val_offs) 4130 amdgpu_device_wb_free(adev, reg_val_offs); 4131 pr_err("failed to read gpu clock\n"); 4132 return ~0; 4133 } 4134 4135 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4136 { 4137 uint64_t clock; 4138 4139 amdgpu_gfx_off_ctrl(adev, false); 4140 mutex_lock(&adev->gfx.gpu_clock_mutex); 4141 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) { 4142 clock = gfx_v9_0_kiq_read_clock(adev); 4143 } else { 4144 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4145 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4146 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4147 } 4148 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4149 amdgpu_gfx_off_ctrl(adev, true); 4150 return clock; 4151 } 4152 4153 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4154 uint32_t vmid, 4155 uint32_t gds_base, uint32_t gds_size, 4156 uint32_t gws_base, uint32_t gws_size, 4157 uint32_t oa_base, uint32_t oa_size) 4158 { 4159 struct amdgpu_device *adev = ring->adev; 4160 4161 /* GDS Base */ 4162 gfx_v9_0_write_data_to_reg(ring, 0, false, 4163 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4164 gds_base); 4165 4166 /* GDS Size */ 4167 gfx_v9_0_write_data_to_reg(ring, 0, false, 4168 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4169 gds_size); 4170 4171 /* GWS */ 4172 gfx_v9_0_write_data_to_reg(ring, 0, false, 4173 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4174 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4175 4176 /* OA */ 4177 gfx_v9_0_write_data_to_reg(ring, 0, false, 4178 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4179 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4180 } 4181 4182 static const u32 vgpr_init_compute_shader[] = 4183 { 4184 0xb07c0000, 0xbe8000ff, 4185 0x000000f8, 0xbf110800, 4186 0x7e000280, 0x7e020280, 4187 0x7e040280, 0x7e060280, 4188 0x7e080280, 0x7e0a0280, 4189 0x7e0c0280, 0x7e0e0280, 4190 0x80808800, 0xbe803200, 4191 0xbf84fff5, 0xbf9c0000, 4192 0xd28c0001, 0x0001007f, 4193 0xd28d0001, 0x0002027e, 4194 0x10020288, 0xb8810904, 4195 0xb7814000, 0xd1196a01, 4196 0x00000301, 0xbe800087, 4197 0xbefc00c1, 0xd89c4000, 4198 0x00020201, 0xd89cc080, 4199 0x00040401, 0x320202ff, 4200 0x00000800, 0x80808100, 4201 0xbf84fff8, 0x7e020280, 4202 0xbf810000, 0x00000000, 4203 }; 4204 4205 static const u32 sgpr_init_compute_shader[] = 4206 { 4207 0xb07c0000, 0xbe8000ff, 4208 0x0000005f, 0xbee50080, 4209 0xbe812c65, 0xbe822c65, 4210 0xbe832c65, 0xbe842c65, 4211 0xbe852c65, 0xb77c0005, 4212 0x80808500, 0xbf84fff8, 4213 0xbe800080, 0xbf810000, 4214 }; 4215 4216 static const u32 vgpr_init_compute_shader_arcturus[] = { 4217 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4218 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4219 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4220 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4221 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4222 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4223 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4224 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4225 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4226 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4227 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4228 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4229 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4230 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4231 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4232 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4233 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4234 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4235 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4236 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4237 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4238 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4239 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4240 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4241 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4242 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4243 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4244 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4245 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4246 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4247 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4248 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4249 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4250 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4251 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4252 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4253 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4254 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4255 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4256 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4257 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4258 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4259 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4260 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4261 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4262 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4263 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4264 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4265 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4266 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4267 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4268 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4269 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4270 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4271 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4272 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4273 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4274 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4275 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4276 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4277 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4278 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4279 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4280 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4281 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4282 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4283 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4284 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4285 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4286 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4287 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4288 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4289 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4290 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4291 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4292 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4293 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4294 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4295 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4296 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4297 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4298 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4299 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4300 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4301 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4302 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4303 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4304 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4305 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4306 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4307 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4308 0xbf84fff8, 0xbf810000, 4309 }; 4310 4311 /* When below register arrays changed, please update gpr_reg_size, 4312 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4313 to cover all gfx9 ASICs */ 4314 static const struct soc15_reg_entry vgpr_init_regs[] = { 4315 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4316 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4317 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4318 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4319 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4320 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4321 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4322 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4323 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4324 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4325 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4326 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4327 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4328 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4329 }; 4330 4331 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4332 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4333 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4334 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4335 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4336 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4337 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4338 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4339 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4340 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4341 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4342 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4343 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4344 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4345 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4346 }; 4347 4348 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4349 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4350 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4351 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4352 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4353 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4354 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4355 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4356 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4357 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4358 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4359 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4360 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4361 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4362 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4363 }; 4364 4365 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4366 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4367 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4368 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4369 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4370 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4371 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4372 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4373 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4374 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4375 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4376 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4377 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4378 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4379 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4380 }; 4381 4382 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4383 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4384 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4385 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4386 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4387 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4388 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4389 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4390 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4391 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4392 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4393 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4394 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4395 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4396 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4397 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4398 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4399 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4400 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4401 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4402 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4403 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4404 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4405 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4406 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4407 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4408 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4409 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4410 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4411 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4412 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4413 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4414 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4415 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4416 }; 4417 4418 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4419 { 4420 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4421 int i, r; 4422 4423 /* only support when RAS is enabled */ 4424 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4425 return 0; 4426 4427 r = amdgpu_ring_alloc(ring, 7); 4428 if (r) { 4429 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n", 4430 ring->name, r); 4431 return r; 4432 } 4433 4434 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4435 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4436 4437 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4438 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4439 PACKET3_DMA_DATA_DST_SEL(1) | 4440 PACKET3_DMA_DATA_SRC_SEL(2) | 4441 PACKET3_DMA_DATA_ENGINE(0))); 4442 amdgpu_ring_write(ring, 0); 4443 amdgpu_ring_write(ring, 0); 4444 amdgpu_ring_write(ring, 0); 4445 amdgpu_ring_write(ring, 0); 4446 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4447 adev->gds.gds_size); 4448 4449 amdgpu_ring_commit(ring); 4450 4451 for (i = 0; i < adev->usec_timeout; i++) { 4452 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4453 break; 4454 udelay(1); 4455 } 4456 4457 if (i >= adev->usec_timeout) 4458 r = -ETIMEDOUT; 4459 4460 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4461 4462 return r; 4463 } 4464 4465 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4466 { 4467 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4468 struct amdgpu_ib ib; 4469 struct dma_fence *f = NULL; 4470 int r, i; 4471 unsigned total_size, vgpr_offset, sgpr_offset; 4472 u64 gpu_addr; 4473 4474 int compute_dim_x = adev->gfx.config.max_shader_engines * 4475 adev->gfx.config.max_cu_per_sh * 4476 adev->gfx.config.max_sh_per_se; 4477 int sgpr_work_group_size = 5; 4478 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4479 int vgpr_init_shader_size; 4480 const u32 *vgpr_init_shader_ptr; 4481 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4482 4483 /* only support when RAS is enabled */ 4484 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4485 return 0; 4486 4487 /* bail if the compute ring is not ready */ 4488 if (!ring->sched.ready) 4489 return 0; 4490 4491 if (adev->asic_type == CHIP_ARCTURUS) { 4492 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4493 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4494 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4495 } else { 4496 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4497 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4498 vgpr_init_regs_ptr = vgpr_init_regs; 4499 } 4500 4501 total_size = 4502 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4503 total_size += 4504 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4505 total_size += 4506 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4507 total_size = ALIGN(total_size, 256); 4508 vgpr_offset = total_size; 4509 total_size += ALIGN(vgpr_init_shader_size, 256); 4510 sgpr_offset = total_size; 4511 total_size += sizeof(sgpr_init_compute_shader); 4512 4513 /* allocate an indirect buffer to put the commands in */ 4514 memset(&ib, 0, sizeof(ib)); 4515 r = amdgpu_ib_get(adev, NULL, total_size, 4516 AMDGPU_IB_POOL_DIRECT, &ib); 4517 if (r) { 4518 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r); 4519 return r; 4520 } 4521 4522 /* load the compute shaders */ 4523 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4524 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4525 4526 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4527 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4528 4529 /* init the ib length to 0 */ 4530 ib.length_dw = 0; 4531 4532 /* VGPR */ 4533 /* write the register state for the compute dispatch */ 4534 for (i = 0; i < gpr_reg_size; i++) { 4535 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4536 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4537 - PACKET3_SET_SH_REG_START; 4538 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4539 } 4540 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4541 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4542 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4543 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4544 - PACKET3_SET_SH_REG_START; 4545 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4546 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4547 4548 /* write dispatch packet */ 4549 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4550 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4551 ib.ptr[ib.length_dw++] = 1; /* y */ 4552 ib.ptr[ib.length_dw++] = 1; /* z */ 4553 ib.ptr[ib.length_dw++] = 4554 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4555 4556 /* write CS partial flush packet */ 4557 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4558 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4559 4560 /* SGPR1 */ 4561 /* write the register state for the compute dispatch */ 4562 for (i = 0; i < gpr_reg_size; i++) { 4563 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4564 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4565 - PACKET3_SET_SH_REG_START; 4566 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4567 } 4568 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4569 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4570 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4571 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4572 - PACKET3_SET_SH_REG_START; 4573 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4574 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4575 4576 /* write dispatch packet */ 4577 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4578 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4579 ib.ptr[ib.length_dw++] = 1; /* y */ 4580 ib.ptr[ib.length_dw++] = 1; /* z */ 4581 ib.ptr[ib.length_dw++] = 4582 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4583 4584 /* write CS partial flush packet */ 4585 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4586 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4587 4588 /* SGPR2 */ 4589 /* write the register state for the compute dispatch */ 4590 for (i = 0; i < gpr_reg_size; i++) { 4591 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4592 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4593 - PACKET3_SET_SH_REG_START; 4594 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4595 } 4596 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4597 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4598 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4599 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4600 - PACKET3_SET_SH_REG_START; 4601 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4602 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4603 4604 /* write dispatch packet */ 4605 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4606 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4607 ib.ptr[ib.length_dw++] = 1; /* y */ 4608 ib.ptr[ib.length_dw++] = 1; /* z */ 4609 ib.ptr[ib.length_dw++] = 4610 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4611 4612 /* write CS partial flush packet */ 4613 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4614 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4615 4616 /* shedule the ib on the ring */ 4617 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4618 if (r) { 4619 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r); 4620 goto fail; 4621 } 4622 4623 /* wait for the GPU to finish processing the IB */ 4624 r = dma_fence_wait(f, false); 4625 if (r) { 4626 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r); 4627 goto fail; 4628 } 4629 4630 fail: 4631 amdgpu_ib_free(adev, &ib, NULL); 4632 dma_fence_put(f); 4633 4634 return r; 4635 } 4636 4637 static int gfx_v9_0_early_init(void *handle) 4638 { 4639 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4640 4641 if (adev->asic_type == CHIP_ARCTURUS) 4642 adev->gfx.num_gfx_rings = 0; 4643 else 4644 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4645 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4646 AMDGPU_MAX_COMPUTE_RINGS); 4647 gfx_v9_0_set_kiq_pm4_funcs(adev); 4648 gfx_v9_0_set_ring_funcs(adev); 4649 gfx_v9_0_set_irq_funcs(adev); 4650 gfx_v9_0_set_gds_init(adev); 4651 gfx_v9_0_set_rlc_funcs(adev); 4652 4653 return 0; 4654 } 4655 4656 static int gfx_v9_0_ecc_late_init(void *handle) 4657 { 4658 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4659 int r; 4660 4661 /* 4662 * Temp workaround to fix the issue that CP firmware fails to 4663 * update read pointer when CPDMA is writing clearing operation 4664 * to GDS in suspend/resume sequence on several cards. So just 4665 * limit this operation in cold boot sequence. 4666 */ 4667 if (!adev->in_suspend) { 4668 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4669 if (r) 4670 return r; 4671 } 4672 4673 /* requires IBs so do in late init after IB pool is initialized */ 4674 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4675 if (r) 4676 return r; 4677 4678 if (adev->gfx.funcs && 4679 adev->gfx.funcs->reset_ras_error_count) 4680 adev->gfx.funcs->reset_ras_error_count(adev); 4681 4682 r = amdgpu_gfx_ras_late_init(adev); 4683 if (r) 4684 return r; 4685 4686 return 0; 4687 } 4688 4689 static int gfx_v9_0_late_init(void *handle) 4690 { 4691 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4692 int r; 4693 4694 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4695 if (r) 4696 return r; 4697 4698 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4699 if (r) 4700 return r; 4701 4702 r = gfx_v9_0_ecc_late_init(handle); 4703 if (r) 4704 return r; 4705 4706 return 0; 4707 } 4708 4709 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4710 { 4711 uint32_t rlc_setting; 4712 4713 /* if RLC is not enabled, do nothing */ 4714 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4715 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4716 return false; 4717 4718 return true; 4719 } 4720 4721 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev) 4722 { 4723 uint32_t data; 4724 unsigned i; 4725 4726 data = RLC_SAFE_MODE__CMD_MASK; 4727 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4728 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4729 4730 /* wait for RLC_SAFE_MODE */ 4731 for (i = 0; i < adev->usec_timeout; i++) { 4732 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4733 break; 4734 udelay(1); 4735 } 4736 } 4737 4738 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev) 4739 { 4740 uint32_t data; 4741 4742 data = RLC_SAFE_MODE__CMD_MASK; 4743 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4744 } 4745 4746 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4747 bool enable) 4748 { 4749 amdgpu_gfx_rlc_enter_safe_mode(adev); 4750 4751 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4752 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4753 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4754 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4755 } else { 4756 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4757 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4758 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4759 } 4760 4761 amdgpu_gfx_rlc_exit_safe_mode(adev); 4762 } 4763 4764 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4765 bool enable) 4766 { 4767 /* TODO: double check if we need to perform under safe mode */ 4768 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4769 4770 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4771 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 4772 else 4773 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 4774 4775 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 4776 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 4777 else 4778 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 4779 4780 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 4781 } 4782 4783 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4784 bool enable) 4785 { 4786 uint32_t data, def; 4787 4788 amdgpu_gfx_rlc_enter_safe_mode(adev); 4789 4790 /* It is disabled by HW by default */ 4791 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 4792 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 4793 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4794 4795 if (adev->asic_type != CHIP_VEGA12) 4796 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4797 4798 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4799 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4800 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4801 4802 /* only for Vega10 & Raven1 */ 4803 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 4804 4805 if (def != data) 4806 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4807 4808 /* MGLS is a global flag to control all MGLS in GFX */ 4809 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 4810 /* 2 - RLC memory Light sleep */ 4811 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 4812 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4813 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4814 if (def != data) 4815 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4816 } 4817 /* 3 - CP memory Light sleep */ 4818 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 4819 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4820 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4821 if (def != data) 4822 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4823 } 4824 } 4825 } else { 4826 /* 1 - MGCG_OVERRIDE */ 4827 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4828 4829 if (adev->asic_type != CHIP_VEGA12) 4830 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 4831 4832 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 4833 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 4834 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 4835 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 4836 4837 if (def != data) 4838 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4839 4840 /* 2 - disable MGLS in RLC */ 4841 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 4842 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 4843 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 4844 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 4845 } 4846 4847 /* 3 - disable MGLS in CP */ 4848 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 4849 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 4850 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 4851 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 4852 } 4853 } 4854 4855 amdgpu_gfx_rlc_exit_safe_mode(adev); 4856 } 4857 4858 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 4859 bool enable) 4860 { 4861 uint32_t data, def; 4862 4863 if (adev->asic_type == CHIP_ARCTURUS) 4864 return; 4865 4866 amdgpu_gfx_rlc_enter_safe_mode(adev); 4867 4868 /* Enable 3D CGCG/CGLS */ 4869 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) { 4870 /* write cmd to clear cgcg/cgls ov */ 4871 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4872 /* unset CGCG override */ 4873 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 4874 /* update CGCG and CGLS override bits */ 4875 if (def != data) 4876 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4877 4878 /* enable 3Dcgcg FSM(0x0000363f) */ 4879 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4880 4881 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4882 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 4883 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 4884 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4885 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 4886 if (def != data) 4887 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4888 4889 /* set IDLE_POLL_COUNT(0x00900100) */ 4890 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4891 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4892 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4893 if (def != data) 4894 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4895 } else { 4896 /* Disable CGCG/CGLS */ 4897 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 4898 /* disable cgcg, cgls should be disabled */ 4899 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 4900 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 4901 /* disable cgcg and cgls in FSM */ 4902 if (def != data) 4903 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 4904 } 4905 4906 amdgpu_gfx_rlc_exit_safe_mode(adev); 4907 } 4908 4909 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 4910 bool enable) 4911 { 4912 uint32_t def, data; 4913 4914 amdgpu_gfx_rlc_enter_safe_mode(adev); 4915 4916 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 4917 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 4918 /* unset CGCG override */ 4919 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 4920 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4921 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4922 else 4923 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 4924 /* update CGCG and CGLS override bits */ 4925 if (def != data) 4926 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 4927 4928 /* enable cgcg FSM(0x0000363F) */ 4929 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4930 4931 if (adev->asic_type == CHIP_ARCTURUS) 4932 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4933 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4934 else 4935 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 4936 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 4937 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 4938 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 4939 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 4940 if (def != data) 4941 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4942 4943 /* set IDLE_POLL_COUNT(0x00900100) */ 4944 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 4945 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 4946 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 4947 if (def != data) 4948 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 4949 } else { 4950 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 4951 /* reset CGCG/CGLS bits */ 4952 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 4953 /* disable cgcg and cgls in FSM */ 4954 if (def != data) 4955 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 4956 } 4957 4958 amdgpu_gfx_rlc_exit_safe_mode(adev); 4959 } 4960 4961 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 4962 bool enable) 4963 { 4964 if (enable) { 4965 /* CGCG/CGLS should be enabled after MGCG/MGLS 4966 * === MGCG + MGLS === 4967 */ 4968 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4969 /* === CGCG /CGLS for GFX 3D Only === */ 4970 gfx_v9_0_update_3d_clock_gating(adev, enable); 4971 /* === CGCG + CGLS === */ 4972 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4973 } else { 4974 /* CGCG/CGLS should be disabled before MGCG/MGLS 4975 * === CGCG + CGLS === 4976 */ 4977 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 4978 /* === CGCG /CGLS for GFX 3D Only === */ 4979 gfx_v9_0_update_3d_clock_gating(adev, enable); 4980 /* === MGCG + MGLS === */ 4981 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 4982 } 4983 return 0; 4984 } 4985 4986 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid) 4987 { 4988 u32 reg, data; 4989 4990 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 4991 if (amdgpu_sriov_is_pp_one_vf(adev)) 4992 data = RREG32_NO_KIQ(reg); 4993 else 4994 data = RREG32(reg); 4995 4996 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 4997 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 4998 4999 if (amdgpu_sriov_is_pp_one_vf(adev)) 5000 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5001 else 5002 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5003 } 5004 5005 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5006 uint32_t offset, 5007 struct soc15_reg_rlcg *entries, int arr_size) 5008 { 5009 int i; 5010 uint32_t reg; 5011 5012 if (!entries) 5013 return false; 5014 5015 for (i = 0; i < arr_size; i++) { 5016 const struct soc15_reg_rlcg *entry; 5017 5018 entry = &entries[i]; 5019 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5020 if (offset == reg) 5021 return true; 5022 } 5023 5024 return false; 5025 } 5026 5027 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5028 { 5029 return gfx_v9_0_check_rlcg_range(adev, offset, 5030 (void *)rlcg_access_gc_9_0, 5031 ARRAY_SIZE(rlcg_access_gc_9_0)); 5032 } 5033 5034 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5035 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5036 .set_safe_mode = gfx_v9_0_set_safe_mode, 5037 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5038 .init = gfx_v9_0_rlc_init, 5039 .get_csb_size = gfx_v9_0_get_csb_size, 5040 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5041 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5042 .resume = gfx_v9_0_rlc_resume, 5043 .stop = gfx_v9_0_rlc_stop, 5044 .reset = gfx_v9_0_rlc_reset, 5045 .start = gfx_v9_0_rlc_start, 5046 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5047 .rlcg_wreg = gfx_v9_0_rlcg_wreg, 5048 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5049 }; 5050 5051 static int gfx_v9_0_set_powergating_state(void *handle, 5052 enum amd_powergating_state state) 5053 { 5054 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5055 bool enable = (state == AMD_PG_STATE_GATE); 5056 5057 switch (adev->asic_type) { 5058 case CHIP_RAVEN: 5059 case CHIP_RENOIR: 5060 if (!enable) 5061 amdgpu_gfx_off_ctrl(adev, false); 5062 5063 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5064 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5065 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5066 } else { 5067 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5068 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5069 } 5070 5071 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5072 gfx_v9_0_enable_cp_power_gating(adev, true); 5073 else 5074 gfx_v9_0_enable_cp_power_gating(adev, false); 5075 5076 /* update gfx cgpg state */ 5077 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5078 5079 /* update mgcg state */ 5080 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5081 5082 if (enable) 5083 amdgpu_gfx_off_ctrl(adev, true); 5084 break; 5085 case CHIP_VEGA12: 5086 amdgpu_gfx_off_ctrl(adev, enable); 5087 break; 5088 default: 5089 break; 5090 } 5091 5092 return 0; 5093 } 5094 5095 static int gfx_v9_0_set_clockgating_state(void *handle, 5096 enum amd_clockgating_state state) 5097 { 5098 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5099 5100 if (amdgpu_sriov_vf(adev)) 5101 return 0; 5102 5103 switch (adev->asic_type) { 5104 case CHIP_VEGA10: 5105 case CHIP_VEGA12: 5106 case CHIP_VEGA20: 5107 case CHIP_RAVEN: 5108 case CHIP_ARCTURUS: 5109 case CHIP_RENOIR: 5110 gfx_v9_0_update_gfx_clock_gating(adev, 5111 state == AMD_CG_STATE_GATE); 5112 break; 5113 default: 5114 break; 5115 } 5116 return 0; 5117 } 5118 5119 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags) 5120 { 5121 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 5122 int data; 5123 5124 if (amdgpu_sriov_vf(adev)) 5125 *flags = 0; 5126 5127 /* AMD_CG_SUPPORT_GFX_MGCG */ 5128 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5129 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5130 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5131 5132 /* AMD_CG_SUPPORT_GFX_CGCG */ 5133 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5134 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5135 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5136 5137 /* AMD_CG_SUPPORT_GFX_CGLS */ 5138 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5139 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5140 5141 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5142 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5143 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5144 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5145 5146 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5147 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5148 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5149 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5150 5151 if (adev->asic_type != CHIP_ARCTURUS) { 5152 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5153 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5154 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5155 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5156 5157 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5158 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5159 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5160 } 5161 } 5162 5163 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5164 { 5165 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/ 5166 } 5167 5168 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5169 { 5170 struct amdgpu_device *adev = ring->adev; 5171 u64 wptr; 5172 5173 /* XXX check if swapping is necessary on BE */ 5174 if (ring->use_doorbell) { 5175 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]); 5176 } else { 5177 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5178 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5179 } 5180 5181 return wptr; 5182 } 5183 5184 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5185 { 5186 struct amdgpu_device *adev = ring->adev; 5187 5188 if (ring->use_doorbell) { 5189 /* XXX check if swapping is necessary on BE */ 5190 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5191 WDOORBELL64(ring->doorbell_index, ring->wptr); 5192 } else { 5193 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5194 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5195 } 5196 } 5197 5198 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5199 { 5200 struct amdgpu_device *adev = ring->adev; 5201 u32 ref_and_mask, reg_mem_engine; 5202 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg; 5203 5204 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 5205 switch (ring->me) { 5206 case 1: 5207 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe; 5208 break; 5209 case 2: 5210 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe; 5211 break; 5212 default: 5213 return; 5214 } 5215 reg_mem_engine = 0; 5216 } else { 5217 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0; 5218 reg_mem_engine = 1; /* pfp */ 5219 } 5220 5221 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5222 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5223 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5224 ref_and_mask, ref_and_mask, 0x20); 5225 } 5226 5227 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5228 struct amdgpu_job *job, 5229 struct amdgpu_ib *ib, 5230 uint32_t flags) 5231 { 5232 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5233 u32 header, control = 0; 5234 5235 if (ib->flags & AMDGPU_IB_FLAG_CE) 5236 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5237 else 5238 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5239 5240 control |= ib->length_dw | (vmid << 24); 5241 5242 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5243 control |= INDIRECT_BUFFER_PRE_ENB(1); 5244 5245 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5246 gfx_v9_0_ring_emit_de_meta(ring); 5247 } 5248 5249 amdgpu_ring_write(ring, header); 5250 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5251 amdgpu_ring_write(ring, 5252 #ifdef __BIG_ENDIAN 5253 (2 << 0) | 5254 #endif 5255 lower_32_bits(ib->gpu_addr)); 5256 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5257 amdgpu_ring_write(ring, control); 5258 } 5259 5260 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5261 struct amdgpu_job *job, 5262 struct amdgpu_ib *ib, 5263 uint32_t flags) 5264 { 5265 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5266 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5267 5268 /* Currently, there is a high possibility to get wave ID mismatch 5269 * between ME and GDS, leading to a hw deadlock, because ME generates 5270 * different wave IDs than the GDS expects. This situation happens 5271 * randomly when at least 5 compute pipes use GDS ordered append. 5272 * The wave IDs generated by ME are also wrong after suspend/resume. 5273 * Those are probably bugs somewhere else in the kernel driver. 5274 * 5275 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5276 * GDS to 0 for this ring (me/pipe). 5277 */ 5278 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5279 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5280 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5281 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5282 } 5283 5284 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5285 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */ 5286 amdgpu_ring_write(ring, 5287 #ifdef __BIG_ENDIAN 5288 (2 << 0) | 5289 #endif 5290 lower_32_bits(ib->gpu_addr)); 5291 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5292 amdgpu_ring_write(ring, control); 5293 } 5294 5295 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5296 u64 seq, unsigned flags) 5297 { 5298 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5299 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5300 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5301 5302 /* RELEASE_MEM - flush caches, send int */ 5303 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5304 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN | 5305 EOP_TC_NC_ACTION_EN) : 5306 (EOP_TCL1_ACTION_EN | 5307 EOP_TC_ACTION_EN | 5308 EOP_TC_WB_ACTION_EN | 5309 EOP_TC_MD_ACTION_EN)) | 5310 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5311 EVENT_INDEX(5))); 5312 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5313 5314 /* 5315 * the address should be Qword aligned if 64bit write, Dword 5316 * aligned if only send 32bit data low (discard data high) 5317 */ 5318 if (write64bit) 5319 BUG_ON(addr & 0x7); 5320 else 5321 BUG_ON(addr & 0x3); 5322 amdgpu_ring_write(ring, lower_32_bits(addr)); 5323 amdgpu_ring_write(ring, upper_32_bits(addr)); 5324 amdgpu_ring_write(ring, lower_32_bits(seq)); 5325 amdgpu_ring_write(ring, upper_32_bits(seq)); 5326 amdgpu_ring_write(ring, 0); 5327 } 5328 5329 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5330 { 5331 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5332 uint32_t seq = ring->fence_drv.sync_seq; 5333 uint64_t addr = ring->fence_drv.gpu_addr; 5334 5335 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0, 5336 lower_32_bits(addr), upper_32_bits(addr), 5337 seq, 0xffffffff, 4); 5338 } 5339 5340 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5341 unsigned vmid, uint64_t pd_addr) 5342 { 5343 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5344 5345 /* compute doesn't have PFP */ 5346 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5347 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5348 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5349 amdgpu_ring_write(ring, 0x0); 5350 } 5351 } 5352 5353 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5354 { 5355 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */ 5356 } 5357 5358 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5359 { 5360 u64 wptr; 5361 5362 /* XXX check if swapping is necessary on BE */ 5363 if (ring->use_doorbell) 5364 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]); 5365 else 5366 BUG(); 5367 return wptr; 5368 } 5369 5370 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5371 { 5372 struct amdgpu_device *adev = ring->adev; 5373 5374 /* XXX check if swapping is necessary on BE */ 5375 if (ring->use_doorbell) { 5376 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], ring->wptr); 5377 WDOORBELL64(ring->doorbell_index, ring->wptr); 5378 } else{ 5379 BUG(); /* only DOORBELL method supported on gfx9 now */ 5380 } 5381 } 5382 5383 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5384 u64 seq, unsigned int flags) 5385 { 5386 struct amdgpu_device *adev = ring->adev; 5387 5388 /* we only allocate 32bit for each seq wb address */ 5389 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT); 5390 5391 /* write fence seq to the "addr" */ 5392 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5393 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5394 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5395 amdgpu_ring_write(ring, lower_32_bits(addr)); 5396 amdgpu_ring_write(ring, upper_32_bits(addr)); 5397 amdgpu_ring_write(ring, lower_32_bits(seq)); 5398 5399 if (flags & AMDGPU_FENCE_FLAG_INT) { 5400 /* set register to trigger INT */ 5401 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5402 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5403 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5404 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5405 amdgpu_ring_write(ring, 0); 5406 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5407 } 5408 } 5409 5410 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5411 { 5412 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5413 amdgpu_ring_write(ring, 0); 5414 } 5415 5416 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring) 5417 { 5418 struct v9_ce_ib_state ce_payload = {0}; 5419 uint64_t csa_addr; 5420 int cnt; 5421 5422 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5423 csa_addr = amdgpu_csa_vaddr(ring->adev); 5424 5425 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5426 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5427 WRITE_DATA_DST_SEL(8) | 5428 WR_CONFIRM) | 5429 WRITE_DATA_CACHE_POLICY(0)); 5430 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5431 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload))); 5432 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2); 5433 } 5434 5435 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring) 5436 { 5437 struct v9_de_ib_state de_payload = {0}; 5438 uint64_t csa_addr, gds_addr; 5439 int cnt; 5440 5441 csa_addr = amdgpu_csa_vaddr(ring->adev); 5442 gds_addr = csa_addr + 4096; 5443 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5444 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5445 5446 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5447 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5448 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5449 WRITE_DATA_DST_SEL(8) | 5450 WR_CONFIRM) | 5451 WRITE_DATA_CACHE_POLICY(0)); 5452 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5453 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload))); 5454 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2); 5455 } 5456 5457 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5458 bool secure) 5459 { 5460 uint32_t v = secure ? FRAME_TMZ : 0; 5461 5462 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5463 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5464 } 5465 5466 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5467 { 5468 uint32_t dw2 = 0; 5469 5470 if (amdgpu_sriov_vf(ring->adev)) 5471 gfx_v9_0_ring_emit_ce_meta(ring); 5472 5473 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5474 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5475 /* set load_global_config & load_global_uconfig */ 5476 dw2 |= 0x8001; 5477 /* set load_cs_sh_regs */ 5478 dw2 |= 0x01000000; 5479 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5480 dw2 |= 0x10002; 5481 5482 /* set load_ce_ram if preamble presented */ 5483 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5484 dw2 |= 0x10000000; 5485 } else { 5486 /* still load_ce_ram if this is the first time preamble presented 5487 * although there is no context switch happens. 5488 */ 5489 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5490 dw2 |= 0x10000000; 5491 } 5492 5493 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5494 amdgpu_ring_write(ring, dw2); 5495 amdgpu_ring_write(ring, 0); 5496 } 5497 5498 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring) 5499 { 5500 unsigned ret; 5501 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5502 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr)); 5503 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr)); 5504 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */ 5505 ret = ring->wptr & ring->buf_mask; 5506 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */ 5507 return ret; 5508 } 5509 5510 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset) 5511 { 5512 unsigned cur; 5513 BUG_ON(offset > ring->buf_mask); 5514 BUG_ON(ring->ring[offset] != 0x55aa55aa); 5515 5516 cur = (ring->wptr & ring->buf_mask) - 1; 5517 if (likely(cur > offset)) 5518 ring->ring[offset] = cur - offset; 5519 else 5520 ring->ring[offset] = (ring->ring_size>>2) - offset + cur; 5521 } 5522 5523 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5524 uint32_t reg_val_offs) 5525 { 5526 struct amdgpu_device *adev = ring->adev; 5527 5528 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5529 amdgpu_ring_write(ring, 0 | /* src: register*/ 5530 (5 << 8) | /* dst: memory */ 5531 (1 << 20)); /* write confirm */ 5532 amdgpu_ring_write(ring, reg); 5533 amdgpu_ring_write(ring, 0); 5534 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5535 reg_val_offs * 4)); 5536 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5537 reg_val_offs * 4)); 5538 } 5539 5540 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5541 uint32_t val) 5542 { 5543 uint32_t cmd = 0; 5544 5545 switch (ring->funcs->type) { 5546 case AMDGPU_RING_TYPE_GFX: 5547 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5548 break; 5549 case AMDGPU_RING_TYPE_KIQ: 5550 cmd = (1 << 16); /* no inc addr */ 5551 break; 5552 default: 5553 cmd = WR_CONFIRM; 5554 break; 5555 } 5556 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5557 amdgpu_ring_write(ring, cmd); 5558 amdgpu_ring_write(ring, reg); 5559 amdgpu_ring_write(ring, 0); 5560 amdgpu_ring_write(ring, val); 5561 } 5562 5563 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5564 uint32_t val, uint32_t mask) 5565 { 5566 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5567 } 5568 5569 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5570 uint32_t reg0, uint32_t reg1, 5571 uint32_t ref, uint32_t mask) 5572 { 5573 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5574 struct amdgpu_device *adev = ring->adev; 5575 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5576 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5577 5578 if (fw_version_ok) 5579 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5580 ref, mask, 0x20); 5581 else 5582 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5583 ref, mask); 5584 } 5585 5586 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5587 { 5588 struct amdgpu_device *adev = ring->adev; 5589 uint32_t value = 0; 5590 5591 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5592 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5593 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5594 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5595 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 5596 } 5597 5598 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 5599 enum amdgpu_interrupt_state state) 5600 { 5601 switch (state) { 5602 case AMDGPU_IRQ_STATE_DISABLE: 5603 case AMDGPU_IRQ_STATE_ENABLE: 5604 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5605 TIME_STAMP_INT_ENABLE, 5606 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5607 break; 5608 default: 5609 break; 5610 } 5611 } 5612 5613 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 5614 int me, int pipe, 5615 enum amdgpu_interrupt_state state) 5616 { 5617 u32 mec_int_cntl, mec_int_cntl_reg; 5618 5619 /* 5620 * amdgpu controls only the first MEC. That's why this function only 5621 * handles the setting of interrupts for this specific MEC. All other 5622 * pipes' interrupts are set by amdkfd. 5623 */ 5624 5625 if (me == 1) { 5626 switch (pipe) { 5627 case 0: 5628 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 5629 break; 5630 case 1: 5631 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 5632 break; 5633 case 2: 5634 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 5635 break; 5636 case 3: 5637 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 5638 break; 5639 default: 5640 DRM_DEBUG("invalid pipe %d\n", pipe); 5641 return; 5642 } 5643 } else { 5644 DRM_DEBUG("invalid me %d\n", me); 5645 return; 5646 } 5647 5648 switch (state) { 5649 case AMDGPU_IRQ_STATE_DISABLE: 5650 mec_int_cntl = RREG32(mec_int_cntl_reg); 5651 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5652 TIME_STAMP_INT_ENABLE, 0); 5653 WREG32(mec_int_cntl_reg, mec_int_cntl); 5654 break; 5655 case AMDGPU_IRQ_STATE_ENABLE: 5656 mec_int_cntl = RREG32(mec_int_cntl_reg); 5657 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 5658 TIME_STAMP_INT_ENABLE, 1); 5659 WREG32(mec_int_cntl_reg, mec_int_cntl); 5660 break; 5661 default: 5662 break; 5663 } 5664 } 5665 5666 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 5667 struct amdgpu_irq_src *source, 5668 unsigned type, 5669 enum amdgpu_interrupt_state state) 5670 { 5671 switch (state) { 5672 case AMDGPU_IRQ_STATE_DISABLE: 5673 case AMDGPU_IRQ_STATE_ENABLE: 5674 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5675 PRIV_REG_INT_ENABLE, 5676 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5677 break; 5678 default: 5679 break; 5680 } 5681 5682 return 0; 5683 } 5684 5685 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 5686 struct amdgpu_irq_src *source, 5687 unsigned type, 5688 enum amdgpu_interrupt_state state) 5689 { 5690 switch (state) { 5691 case AMDGPU_IRQ_STATE_DISABLE: 5692 case AMDGPU_IRQ_STATE_ENABLE: 5693 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5694 PRIV_INSTR_INT_ENABLE, 5695 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 5696 break; 5697 default: 5698 break; 5699 } 5700 5701 return 0; 5702 } 5703 5704 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 5705 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5706 CP_ECC_ERROR_INT_ENABLE, 1) 5707 5708 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 5709 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 5710 CP_ECC_ERROR_INT_ENABLE, 0) 5711 5712 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 5713 struct amdgpu_irq_src *source, 5714 unsigned type, 5715 enum amdgpu_interrupt_state state) 5716 { 5717 switch (state) { 5718 case AMDGPU_IRQ_STATE_DISABLE: 5719 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5720 CP_ECC_ERROR_INT_ENABLE, 0); 5721 DISABLE_ECC_ON_ME_PIPE(1, 0); 5722 DISABLE_ECC_ON_ME_PIPE(1, 1); 5723 DISABLE_ECC_ON_ME_PIPE(1, 2); 5724 DISABLE_ECC_ON_ME_PIPE(1, 3); 5725 break; 5726 5727 case AMDGPU_IRQ_STATE_ENABLE: 5728 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 5729 CP_ECC_ERROR_INT_ENABLE, 1); 5730 ENABLE_ECC_ON_ME_PIPE(1, 0); 5731 ENABLE_ECC_ON_ME_PIPE(1, 1); 5732 ENABLE_ECC_ON_ME_PIPE(1, 2); 5733 ENABLE_ECC_ON_ME_PIPE(1, 3); 5734 break; 5735 default: 5736 break; 5737 } 5738 5739 return 0; 5740 } 5741 5742 5743 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 5744 struct amdgpu_irq_src *src, 5745 unsigned type, 5746 enum amdgpu_interrupt_state state) 5747 { 5748 switch (type) { 5749 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 5750 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 5751 break; 5752 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 5753 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 5754 break; 5755 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 5756 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 5757 break; 5758 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 5759 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 5760 break; 5761 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 5762 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 5763 break; 5764 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 5765 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 5766 break; 5767 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 5768 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 5769 break; 5770 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 5771 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 5772 break; 5773 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 5774 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 5775 break; 5776 default: 5777 break; 5778 } 5779 return 0; 5780 } 5781 5782 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 5783 struct amdgpu_irq_src *source, 5784 struct amdgpu_iv_entry *entry) 5785 { 5786 int i; 5787 u8 me_id, pipe_id, queue_id; 5788 struct amdgpu_ring *ring; 5789 5790 DRM_DEBUG("IH: CP EOP\n"); 5791 me_id = (entry->ring_id & 0x0c) >> 2; 5792 pipe_id = (entry->ring_id & 0x03) >> 0; 5793 queue_id = (entry->ring_id & 0x70) >> 4; 5794 5795 switch (me_id) { 5796 case 0: 5797 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 5798 break; 5799 case 1: 5800 case 2: 5801 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5802 ring = &adev->gfx.compute_ring[i]; 5803 /* Per-queue interrupt is supported for MEC starting from VI. 5804 * The interrupt can only be enabled/disabled per pipe instead of per queue. 5805 */ 5806 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 5807 amdgpu_fence_process(ring); 5808 } 5809 break; 5810 } 5811 return 0; 5812 } 5813 5814 static void gfx_v9_0_fault(struct amdgpu_device *adev, 5815 struct amdgpu_iv_entry *entry) 5816 { 5817 u8 me_id, pipe_id, queue_id; 5818 struct amdgpu_ring *ring; 5819 int i; 5820 5821 me_id = (entry->ring_id & 0x0c) >> 2; 5822 pipe_id = (entry->ring_id & 0x03) >> 0; 5823 queue_id = (entry->ring_id & 0x70) >> 4; 5824 5825 switch (me_id) { 5826 case 0: 5827 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 5828 break; 5829 case 1: 5830 case 2: 5831 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5832 ring = &adev->gfx.compute_ring[i]; 5833 if (ring->me == me_id && ring->pipe == pipe_id && 5834 ring->queue == queue_id) 5835 drm_sched_fault(&ring->sched); 5836 } 5837 break; 5838 } 5839 } 5840 5841 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 5842 struct amdgpu_irq_src *source, 5843 struct amdgpu_iv_entry *entry) 5844 { 5845 DRM_ERROR("Illegal register access in command stream\n"); 5846 gfx_v9_0_fault(adev, entry); 5847 return 0; 5848 } 5849 5850 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 5851 struct amdgpu_irq_src *source, 5852 struct amdgpu_iv_entry *entry) 5853 { 5854 DRM_ERROR("Illegal instruction in command stream\n"); 5855 gfx_v9_0_fault(adev, entry); 5856 return 0; 5857 } 5858 5859 5860 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 5861 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 5862 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 5863 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 5864 }, 5865 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 5866 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 5867 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 5868 }, 5869 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5870 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 5871 0, 0 5872 }, 5873 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 5874 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 5875 0, 0 5876 }, 5877 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 5878 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 5879 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 5880 }, 5881 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5882 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 5883 0, 0 5884 }, 5885 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 5886 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 5887 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 5888 }, 5889 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 5890 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 5891 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 5892 }, 5893 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 5894 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 5895 0, 0 5896 }, 5897 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 5898 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 5899 0, 0 5900 }, 5901 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 5902 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 5903 0, 0 5904 }, 5905 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5906 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 5907 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 5908 }, 5909 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 5910 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 5911 0, 0 5912 }, 5913 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5914 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 5915 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 5916 }, 5917 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 5918 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5919 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 5920 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 5921 }, 5922 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 5923 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 5924 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 5925 0, 0 5926 }, 5927 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 5928 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5929 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 5930 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 5931 }, 5932 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 5933 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5934 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 5935 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 5936 }, 5937 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 5938 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5939 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 5940 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 5941 }, 5942 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 5943 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 5944 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 5945 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 5946 }, 5947 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 5948 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 5949 0, 0 5950 }, 5951 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5952 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 5953 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 5954 }, 5955 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5956 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 5957 0, 0 5958 }, 5959 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5960 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 5961 0, 0 5962 }, 5963 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5964 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 5965 0, 0 5966 }, 5967 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 5968 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 5969 0, 0 5970 }, 5971 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5972 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 5973 0, 0 5974 }, 5975 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 5976 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 5977 0, 0 5978 }, 5979 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5980 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 5981 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 5982 }, 5983 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5984 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 5985 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 5986 }, 5987 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5988 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 5989 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 5990 }, 5991 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5992 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 5993 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 5994 }, 5995 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 5996 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 5997 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 5998 }, 5999 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6000 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6001 0, 0 6002 }, 6003 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6004 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6005 0, 0 6006 }, 6007 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6008 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6009 0, 0 6010 }, 6011 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6012 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6013 0, 0 6014 }, 6015 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6016 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6017 0, 0 6018 }, 6019 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6020 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6021 0, 0 6022 }, 6023 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6024 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6025 0, 0 6026 }, 6027 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6028 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6029 0, 0 6030 }, 6031 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6032 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6033 0, 0 6034 }, 6035 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6036 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6037 0, 0 6038 }, 6039 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6040 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6041 0, 0 6042 }, 6043 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6044 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6045 0, 0 6046 }, 6047 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6048 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6049 0, 0 6050 }, 6051 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6052 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6053 0, 0 6054 }, 6055 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6056 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6057 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6058 }, 6059 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6060 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6061 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6062 }, 6063 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6064 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6065 0, 0 6066 }, 6067 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6068 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6069 0, 0 6070 }, 6071 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6072 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6073 0, 0 6074 }, 6075 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6076 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6077 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6078 }, 6079 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6080 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6081 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6082 }, 6083 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6084 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6085 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6086 }, 6087 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6088 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6089 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6090 }, 6091 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6092 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6093 0, 0 6094 }, 6095 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6096 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6097 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6098 }, 6099 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6100 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6101 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6102 }, 6103 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6104 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6105 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6106 }, 6107 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6108 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6109 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6110 }, 6111 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6112 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6113 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6114 }, 6115 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6116 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6117 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6118 }, 6119 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6120 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6121 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6122 }, 6123 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6124 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6125 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6126 }, 6127 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6128 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6129 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6130 }, 6131 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6132 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6133 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6134 }, 6135 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6136 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6137 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6138 }, 6139 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6140 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6141 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6142 }, 6143 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6144 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6145 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6146 }, 6147 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6148 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6149 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6150 }, 6151 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6152 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6153 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6154 }, 6155 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6156 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6157 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6158 }, 6159 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6160 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6161 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6162 }, 6163 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6164 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6165 0, 0 6166 }, 6167 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6168 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6169 0, 0 6170 }, 6171 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6172 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6173 0, 0 6174 }, 6175 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6176 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6177 0, 0 6178 }, 6179 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6180 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6181 0, 0 6182 }, 6183 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6184 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6185 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6186 }, 6187 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6188 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6189 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6190 }, 6191 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6192 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6193 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6194 }, 6195 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6196 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6197 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6198 }, 6199 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6200 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6201 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6202 }, 6203 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6204 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6205 0, 0 6206 }, 6207 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6208 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6209 0, 0 6210 }, 6211 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6212 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6213 0, 0 6214 }, 6215 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6216 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6217 0, 0 6218 }, 6219 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6220 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6221 0, 0 6222 }, 6223 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6224 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6225 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6226 }, 6227 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6228 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6229 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6230 }, 6231 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6232 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6233 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6234 }, 6235 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6236 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6237 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6238 }, 6239 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6240 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6241 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6242 }, 6243 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6244 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6245 0, 0 6246 }, 6247 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6248 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6249 0, 0 6250 }, 6251 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6252 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6253 0, 0 6254 }, 6255 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6256 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6257 0, 0 6258 }, 6259 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6260 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6261 0, 0 6262 }, 6263 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6264 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6265 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6266 }, 6267 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6268 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6269 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6270 }, 6271 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6272 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6273 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6274 }, 6275 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6276 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6277 0, 0 6278 }, 6279 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6280 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6281 0, 0 6282 }, 6283 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6284 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6285 0, 0 6286 }, 6287 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6288 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6289 0, 0 6290 }, 6291 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6292 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6293 0, 0 6294 }, 6295 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6296 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6297 0, 0 6298 } 6299 }; 6300 6301 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6302 void *inject_if) 6303 { 6304 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6305 int ret; 6306 struct ta_ras_trigger_error_input block_info = { 0 }; 6307 6308 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6309 return -EINVAL; 6310 6311 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6312 return -EINVAL; 6313 6314 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6315 return -EPERM; 6316 6317 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6318 info->head.type)) { 6319 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6320 ras_gfx_subblocks[info->head.sub_block_index].name, 6321 info->head.type); 6322 return -EPERM; 6323 } 6324 6325 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6326 info->head.type)) { 6327 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6328 ras_gfx_subblocks[info->head.sub_block_index].name, 6329 info->head.type); 6330 return -EPERM; 6331 } 6332 6333 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6334 block_info.sub_block_index = 6335 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6336 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6337 block_info.address = info->address; 6338 block_info.value = info->value; 6339 6340 mutex_lock(&adev->grbm_idx_mutex); 6341 ret = psp_ras_trigger_error(&adev->psp, &block_info); 6342 mutex_unlock(&adev->grbm_idx_mutex); 6343 6344 return ret; 6345 } 6346 6347 static const char *vml2_mems[] = { 6348 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6349 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6350 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6351 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6352 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6353 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6354 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6355 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6356 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6357 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6358 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6359 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6360 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6361 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6362 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6363 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6364 }; 6365 6366 static const char *vml2_walker_mems[] = { 6367 "UTC_VML2_CACHE_PDE0_MEM0", 6368 "UTC_VML2_CACHE_PDE0_MEM1", 6369 "UTC_VML2_CACHE_PDE1_MEM0", 6370 "UTC_VML2_CACHE_PDE1_MEM1", 6371 "UTC_VML2_CACHE_PDE2_MEM0", 6372 "UTC_VML2_CACHE_PDE2_MEM1", 6373 "UTC_VML2_RDIF_LOG_FIFO", 6374 }; 6375 6376 static const char *atc_l2_cache_2m_mems[] = { 6377 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6378 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6379 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6380 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6381 }; 6382 6383 static const char *atc_l2_cache_4k_mems[] = { 6384 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6385 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6386 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6387 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6388 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6389 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6390 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6391 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6392 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6393 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6394 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6395 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6396 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6397 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6398 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6399 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6400 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6401 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6402 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6403 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6404 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6405 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6406 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6407 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6408 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6409 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6410 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6411 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6412 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6413 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6414 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6415 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6416 }; 6417 6418 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6419 struct ras_err_data *err_data) 6420 { 6421 uint32_t i, data; 6422 uint32_t sec_count, ded_count; 6423 6424 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6425 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6426 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6427 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6428 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6429 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6430 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6431 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6432 6433 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6434 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6435 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6436 6437 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6438 if (sec_count) { 6439 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6440 "SEC %d\n", i, vml2_mems[i], sec_count); 6441 err_data->ce_count += sec_count; 6442 } 6443 6444 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6445 if (ded_count) { 6446 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6447 "DED %d\n", i, vml2_mems[i], ded_count); 6448 err_data->ue_count += ded_count; 6449 } 6450 } 6451 6452 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6453 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6454 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6455 6456 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6457 SEC_COUNT); 6458 if (sec_count) { 6459 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6460 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6461 err_data->ce_count += sec_count; 6462 } 6463 6464 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6465 DED_COUNT); 6466 if (ded_count) { 6467 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6468 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6469 err_data->ue_count += ded_count; 6470 } 6471 } 6472 6473 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6474 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6475 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6476 6477 sec_count = (data & 0x00006000L) >> 0xd; 6478 if (sec_count) { 6479 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6480 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6481 sec_count); 6482 err_data->ce_count += sec_count; 6483 } 6484 } 6485 6486 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6487 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6488 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6489 6490 sec_count = (data & 0x00006000L) >> 0xd; 6491 if (sec_count) { 6492 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6493 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6494 sec_count); 6495 err_data->ce_count += sec_count; 6496 } 6497 6498 ded_count = (data & 0x00018000L) >> 0xf; 6499 if (ded_count) { 6500 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6501 "DED %d\n", i, atc_l2_cache_4k_mems[i], 6502 ded_count); 6503 err_data->ue_count += ded_count; 6504 } 6505 } 6506 6507 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6508 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6509 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6510 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6511 6512 return 0; 6513 } 6514 6515 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 6516 const struct soc15_reg_entry *reg, 6517 uint32_t se_id, uint32_t inst_id, uint32_t value, 6518 uint32_t *sec_count, uint32_t *ded_count) 6519 { 6520 uint32_t i; 6521 uint32_t sec_cnt, ded_cnt; 6522 6523 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 6524 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 6525 gfx_v9_0_ras_fields[i].seg != reg->seg || 6526 gfx_v9_0_ras_fields[i].inst != reg->inst) 6527 continue; 6528 6529 sec_cnt = (value & 6530 gfx_v9_0_ras_fields[i].sec_count_mask) >> 6531 gfx_v9_0_ras_fields[i].sec_count_shift; 6532 if (sec_cnt) { 6533 dev_info(adev->dev, "GFX SubBlock %s, " 6534 "Instance[%d][%d], SEC %d\n", 6535 gfx_v9_0_ras_fields[i].name, 6536 se_id, inst_id, 6537 sec_cnt); 6538 *sec_count += sec_cnt; 6539 } 6540 6541 ded_cnt = (value & 6542 gfx_v9_0_ras_fields[i].ded_count_mask) >> 6543 gfx_v9_0_ras_fields[i].ded_count_shift; 6544 if (ded_cnt) { 6545 dev_info(adev->dev, "GFX SubBlock %s, " 6546 "Instance[%d][%d], DED %d\n", 6547 gfx_v9_0_ras_fields[i].name, 6548 se_id, inst_id, 6549 ded_cnt); 6550 *ded_count += ded_cnt; 6551 } 6552 } 6553 6554 return 0; 6555 } 6556 6557 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 6558 { 6559 int i, j, k; 6560 6561 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6562 return; 6563 6564 /* read back registers to clear the counters */ 6565 mutex_lock(&adev->grbm_idx_mutex); 6566 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6567 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6568 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6569 gfx_v9_0_select_se_sh(adev, j, 0x0, k); 6570 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6571 } 6572 } 6573 } 6574 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 6575 mutex_unlock(&adev->grbm_idx_mutex); 6576 6577 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6578 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6579 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6580 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6581 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6582 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6583 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6584 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6585 6586 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6587 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6588 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6589 } 6590 6591 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6592 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6593 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6594 } 6595 6596 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6597 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6598 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6599 } 6600 6601 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6602 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6603 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6604 } 6605 6606 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6607 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6608 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6609 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6610 } 6611 6612 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 6613 void *ras_error_status) 6614 { 6615 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 6616 uint32_t sec_count = 0, ded_count = 0; 6617 uint32_t i, j, k; 6618 uint32_t reg_value; 6619 6620 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6621 return -EINVAL; 6622 6623 err_data->ue_count = 0; 6624 err_data->ce_count = 0; 6625 6626 mutex_lock(&adev->grbm_idx_mutex); 6627 6628 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 6629 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 6630 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 6631 gfx_v9_0_select_se_sh(adev, j, 0, k); 6632 reg_value = 6633 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 6634 if (reg_value) 6635 gfx_v9_0_ras_error_count(adev, 6636 &gfx_v9_0_edc_counter_regs[i], 6637 j, k, reg_value, 6638 &sec_count, &ded_count); 6639 } 6640 } 6641 } 6642 6643 err_data->ce_count += sec_count; 6644 err_data->ue_count += ded_count; 6645 6646 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 6647 mutex_unlock(&adev->grbm_idx_mutex); 6648 6649 gfx_v9_0_query_utc_edc_status(adev, err_data); 6650 6651 return 0; 6652 } 6653 6654 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 6655 { 6656 const unsigned int cp_coher_cntl = 6657 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 6658 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 6659 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 6660 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 6661 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 6662 6663 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 6664 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 6665 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 6666 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6667 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6668 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6669 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6670 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6671 } 6672 6673 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 6674 .name = "gfx_v9_0", 6675 .early_init = gfx_v9_0_early_init, 6676 .late_init = gfx_v9_0_late_init, 6677 .sw_init = gfx_v9_0_sw_init, 6678 .sw_fini = gfx_v9_0_sw_fini, 6679 .hw_init = gfx_v9_0_hw_init, 6680 .hw_fini = gfx_v9_0_hw_fini, 6681 .suspend = gfx_v9_0_suspend, 6682 .resume = gfx_v9_0_resume, 6683 .is_idle = gfx_v9_0_is_idle, 6684 .wait_for_idle = gfx_v9_0_wait_for_idle, 6685 .soft_reset = gfx_v9_0_soft_reset, 6686 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 6687 .set_powergating_state = gfx_v9_0_set_powergating_state, 6688 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 6689 }; 6690 6691 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 6692 .type = AMDGPU_RING_TYPE_GFX, 6693 .align_mask = 0xff, 6694 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6695 .support_64bit_ptrs = true, 6696 .vmhub = AMDGPU_GFXHUB_0, 6697 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 6698 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 6699 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 6700 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 6701 5 + /* COND_EXEC */ 6702 7 + /* PIPELINE_SYNC */ 6703 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6704 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6705 2 + /* VM_FLUSH */ 6706 8 + /* FENCE for VM_FLUSH */ 6707 20 + /* GDS switch */ 6708 4 + /* double SWITCH_BUFFER, 6709 the first COND_EXEC jump to the place just 6710 prior to this double SWITCH_BUFFER */ 6711 5 + /* COND_EXEC */ 6712 7 + /* HDP_flush */ 6713 4 + /* VGT_flush */ 6714 14 + /* CE_META */ 6715 31 + /* DE_META */ 6716 3 + /* CNTX_CTRL */ 6717 5 + /* HDP_INVL */ 6718 8 + 8 + /* FENCE x2 */ 6719 2 + /* SWITCH_BUFFER */ 6720 7, /* gfx_v9_0_emit_mem_sync */ 6721 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 6722 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 6723 .emit_fence = gfx_v9_0_ring_emit_fence, 6724 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6725 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6726 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6727 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6728 .test_ring = gfx_v9_0_ring_test_ring, 6729 .test_ib = gfx_v9_0_ring_test_ib, 6730 .insert_nop = amdgpu_ring_insert_nop, 6731 .pad_ib = amdgpu_ring_generic_pad_ib, 6732 .emit_switch_buffer = gfx_v9_ring_emit_sb, 6733 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 6734 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 6735 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec, 6736 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 6737 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6738 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6739 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6740 .soft_recovery = gfx_v9_0_ring_soft_recovery, 6741 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6742 }; 6743 6744 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 6745 .type = AMDGPU_RING_TYPE_COMPUTE, 6746 .align_mask = 0xff, 6747 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6748 .support_64bit_ptrs = true, 6749 .vmhub = AMDGPU_GFXHUB_0, 6750 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6751 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6752 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6753 .emit_frame_size = 6754 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6755 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6756 5 + /* hdp invalidate */ 6757 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6758 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6759 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6760 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6761 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 6762 7, /* gfx_v9_0_emit_mem_sync */ 6763 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6764 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 6765 .emit_fence = gfx_v9_0_ring_emit_fence, 6766 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 6767 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 6768 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 6769 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 6770 .test_ring = gfx_v9_0_ring_test_ring, 6771 .test_ib = gfx_v9_0_ring_test_ib, 6772 .insert_nop = amdgpu_ring_insert_nop, 6773 .pad_ib = amdgpu_ring_generic_pad_ib, 6774 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6775 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6776 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6777 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 6778 }; 6779 6780 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 6781 .type = AMDGPU_RING_TYPE_KIQ, 6782 .align_mask = 0xff, 6783 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 6784 .support_64bit_ptrs = true, 6785 .vmhub = AMDGPU_GFXHUB_0, 6786 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 6787 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 6788 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 6789 .emit_frame_size = 6790 20 + /* gfx_v9_0_ring_emit_gds_switch */ 6791 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 6792 5 + /* hdp invalidate */ 6793 7 + /* gfx_v9_0_ring_emit_pipeline_sync */ 6794 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 6795 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 6796 2 + /* gfx_v9_0_ring_emit_vm_flush */ 6797 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 6798 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 6799 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 6800 .test_ring = gfx_v9_0_ring_test_ring, 6801 .insert_nop = amdgpu_ring_insert_nop, 6802 .pad_ib = amdgpu_ring_generic_pad_ib, 6803 .emit_rreg = gfx_v9_0_ring_emit_rreg, 6804 .emit_wreg = gfx_v9_0_ring_emit_wreg, 6805 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 6806 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 6807 }; 6808 6809 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 6810 { 6811 int i; 6812 6813 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq; 6814 6815 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 6816 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 6817 6818 for (i = 0; i < adev->gfx.num_compute_rings; i++) 6819 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 6820 } 6821 6822 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 6823 .set = gfx_v9_0_set_eop_interrupt_state, 6824 .process = gfx_v9_0_eop_irq, 6825 }; 6826 6827 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 6828 .set = gfx_v9_0_set_priv_reg_fault_state, 6829 .process = gfx_v9_0_priv_reg_irq, 6830 }; 6831 6832 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 6833 .set = gfx_v9_0_set_priv_inst_fault_state, 6834 .process = gfx_v9_0_priv_inst_irq, 6835 }; 6836 6837 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 6838 .set = gfx_v9_0_set_cp_ecc_error_state, 6839 .process = amdgpu_gfx_cp_ecc_error_irq, 6840 }; 6841 6842 6843 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 6844 { 6845 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 6846 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 6847 6848 adev->gfx.priv_reg_irq.num_types = 1; 6849 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 6850 6851 adev->gfx.priv_inst_irq.num_types = 1; 6852 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 6853 6854 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 6855 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 6856 } 6857 6858 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 6859 { 6860 switch (adev->asic_type) { 6861 case CHIP_VEGA10: 6862 case CHIP_VEGA12: 6863 case CHIP_VEGA20: 6864 case CHIP_RAVEN: 6865 case CHIP_ARCTURUS: 6866 case CHIP_RENOIR: 6867 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 6868 break; 6869 default: 6870 break; 6871 } 6872 } 6873 6874 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 6875 { 6876 /* init asci gds info */ 6877 switch (adev->asic_type) { 6878 case CHIP_VEGA10: 6879 case CHIP_VEGA12: 6880 case CHIP_VEGA20: 6881 adev->gds.gds_size = 0x10000; 6882 break; 6883 case CHIP_RAVEN: 6884 case CHIP_ARCTURUS: 6885 adev->gds.gds_size = 0x1000; 6886 break; 6887 default: 6888 adev->gds.gds_size = 0x10000; 6889 break; 6890 } 6891 6892 switch (adev->asic_type) { 6893 case CHIP_VEGA10: 6894 case CHIP_VEGA20: 6895 adev->gds.gds_compute_max_wave_id = 0x7ff; 6896 break; 6897 case CHIP_VEGA12: 6898 adev->gds.gds_compute_max_wave_id = 0x27f; 6899 break; 6900 case CHIP_RAVEN: 6901 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 6902 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 6903 else 6904 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 6905 break; 6906 case CHIP_ARCTURUS: 6907 adev->gds.gds_compute_max_wave_id = 0xfff; 6908 break; 6909 default: 6910 /* this really depends on the chip */ 6911 adev->gds.gds_compute_max_wave_id = 0x7ff; 6912 break; 6913 } 6914 6915 adev->gds.gws_size = 64; 6916 adev->gds.oa_size = 16; 6917 } 6918 6919 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 6920 u32 bitmap) 6921 { 6922 u32 data; 6923 6924 if (!bitmap) 6925 return; 6926 6927 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6928 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6929 6930 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 6931 } 6932 6933 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 6934 { 6935 u32 data, mask; 6936 6937 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 6938 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 6939 6940 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 6941 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 6942 6943 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 6944 6945 return (~data) & mask; 6946 } 6947 6948 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 6949 struct amdgpu_cu_info *cu_info) 6950 { 6951 int i, j, k, counter, active_cu_number = 0; 6952 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 6953 unsigned disable_masks[4 * 4]; 6954 6955 if (!adev || !cu_info) 6956 return -EINVAL; 6957 6958 /* 6959 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 6960 */ 6961 if (adev->gfx.config.max_shader_engines * 6962 adev->gfx.config.max_sh_per_se > 16) 6963 return -EINVAL; 6964 6965 amdgpu_gfx_parse_disable_cu(disable_masks, 6966 adev->gfx.config.max_shader_engines, 6967 adev->gfx.config.max_sh_per_se); 6968 6969 mutex_lock(&adev->grbm_idx_mutex); 6970 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 6971 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 6972 mask = 1; 6973 ao_bitmap = 0; 6974 counter = 0; 6975 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff); 6976 gfx_v9_0_set_user_cu_inactive_bitmap( 6977 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 6978 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 6979 6980 /* 6981 * The bitmap(and ao_cu_bitmap) in cu_info structure is 6982 * 4x4 size array, and it's usually suitable for Vega 6983 * ASICs which has 4*2 SE/SH layout. 6984 * But for Arcturus, SE/SH layout is changed to 8*1. 6985 * To mostly reduce the impact, we make it compatible 6986 * with current bitmap array as below: 6987 * SE4,SH0 --> bitmap[0][1] 6988 * SE5,SH0 --> bitmap[1][1] 6989 * SE6,SH0 --> bitmap[2][1] 6990 * SE7,SH0 --> bitmap[3][1] 6991 */ 6992 cu_info->bitmap[i % 4][j + i / 4] = bitmap; 6993 6994 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 6995 if (bitmap & mask) { 6996 if (counter < adev->gfx.config.max_cu_per_sh) 6997 ao_bitmap |= mask; 6998 counter ++; 6999 } 7000 mask <<= 1; 7001 } 7002 active_cu_number += counter; 7003 if (i < 2 && j < 2) 7004 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7005 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7006 } 7007 } 7008 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff); 7009 mutex_unlock(&adev->grbm_idx_mutex); 7010 7011 cu_info->number = active_cu_number; 7012 cu_info->ao_cu_mask = ao_cu_mask; 7013 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7014 7015 return 0; 7016 } 7017 7018 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7019 { 7020 .type = AMD_IP_BLOCK_TYPE_GFX, 7021 .major = 9, 7022 .minor = 0, 7023 .rev = 0, 7024 .funcs = &gfx_v9_0_ip_funcs, 7025 }; 7026