1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/delay.h> 25 #include <linux/kernel.h> 26 #include <linux/firmware.h> 27 #include <linux/module.h> 28 #include <linux/pci.h> 29 30 #include "amdgpu.h" 31 #include "amdgpu_gfx.h" 32 #include "soc15.h" 33 #include "soc15d.h" 34 #include "amdgpu_atomfirmware.h" 35 #include "amdgpu_pm.h" 36 37 #include "gc/gc_9_0_offset.h" 38 #include "gc/gc_9_0_sh_mask.h" 39 40 #include "vega10_enum.h" 41 42 #include "soc15_common.h" 43 #include "clearstate_gfx9.h" 44 #include "v9_structs.h" 45 46 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h" 47 48 #include "amdgpu_ras.h" 49 50 #include "amdgpu_ring_mux.h" 51 #include "gfx_v9_4.h" 52 #include "gfx_v9_0.h" 53 #include "gfx_v9_0_cleaner_shader.h" 54 #include "gfx_v9_4_2.h" 55 56 #include "asic_reg/pwr/pwr_10_0_offset.h" 57 #include "asic_reg/pwr/pwr_10_0_sh_mask.h" 58 #include "asic_reg/gc/gc_9_0_default.h" 59 60 #define GFX9_NUM_GFX_RINGS 1 61 #define GFX9_NUM_SW_GFX_RINGS 2 62 #define GFX9_MEC_HPD_SIZE 4096 63 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 64 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L 65 66 #define mmGCEA_PROBE_MAP 0x070c 67 #define mmGCEA_PROBE_MAP_BASE_IDX 0 68 69 MODULE_FIRMWARE("amdgpu/vega10_ce.bin"); 70 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin"); 71 MODULE_FIRMWARE("amdgpu/vega10_me.bin"); 72 MODULE_FIRMWARE("amdgpu/vega10_mec.bin"); 73 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin"); 74 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin"); 75 76 MODULE_FIRMWARE("amdgpu/vega12_ce.bin"); 77 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_me.bin"); 79 MODULE_FIRMWARE("amdgpu/vega12_mec.bin"); 80 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin"); 81 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin"); 82 83 MODULE_FIRMWARE("amdgpu/vega20_ce.bin"); 84 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin"); 85 MODULE_FIRMWARE("amdgpu/vega20_me.bin"); 86 MODULE_FIRMWARE("amdgpu/vega20_mec.bin"); 87 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin"); 88 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin"); 89 90 MODULE_FIRMWARE("amdgpu/raven_ce.bin"); 91 MODULE_FIRMWARE("amdgpu/raven_pfp.bin"); 92 MODULE_FIRMWARE("amdgpu/raven_me.bin"); 93 MODULE_FIRMWARE("amdgpu/raven_mec.bin"); 94 MODULE_FIRMWARE("amdgpu/raven_mec2.bin"); 95 MODULE_FIRMWARE("amdgpu/raven_rlc.bin"); 96 97 MODULE_FIRMWARE("amdgpu/picasso_ce.bin"); 98 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin"); 99 MODULE_FIRMWARE("amdgpu/picasso_me.bin"); 100 MODULE_FIRMWARE("amdgpu/picasso_mec.bin"); 101 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin"); 102 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin"); 103 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin"); 104 105 MODULE_FIRMWARE("amdgpu/raven2_ce.bin"); 106 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin"); 107 MODULE_FIRMWARE("amdgpu/raven2_me.bin"); 108 MODULE_FIRMWARE("amdgpu/raven2_mec.bin"); 109 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin"); 110 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin"); 111 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin"); 112 113 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin"); 114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin"); 115 116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin"); 117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin"); 118 MODULE_FIRMWARE("amdgpu/renoir_me.bin"); 119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin"); 120 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin"); 121 122 MODULE_FIRMWARE("amdgpu/green_sardine_ce.bin"); 123 MODULE_FIRMWARE("amdgpu/green_sardine_pfp.bin"); 124 MODULE_FIRMWARE("amdgpu/green_sardine_me.bin"); 125 MODULE_FIRMWARE("amdgpu/green_sardine_mec.bin"); 126 MODULE_FIRMWARE("amdgpu/green_sardine_mec2.bin"); 127 MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin"); 128 129 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin"); 130 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin"); 131 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin"); 132 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin"); 133 MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin"); 134 135 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03 136 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0 137 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04 138 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0 139 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09 140 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0 141 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a 142 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0 143 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b 144 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0 145 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c 146 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0 147 148 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir 0x0025 149 #define mmGOLDEN_TSC_COUNT_UPPER_Renoir_BASE_IDX 1 150 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir 0x0026 151 #define mmGOLDEN_TSC_COUNT_LOWER_Renoir_BASE_IDX 1 152 153 static const struct amdgpu_hwip_reg_entry gc_reg_list_9[] = { 154 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS), 155 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS2), 156 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT1), 157 SOC15_REG_ENTRY_STR(GC, 0, mmCP_STALLED_STAT2), 158 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STALLED_STAT1), 159 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STALLED_STAT1), 160 SOC15_REG_ENTRY_STR(GC, 0, mmCP_BUSY_STAT), 161 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_BUSY_STAT), 162 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_BUSY_STAT), 163 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPF_STATUS), 164 SOC15_REG_ENTRY_STR(GC, 0, mmCP_GFX_ERROR), 165 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_BASE), 166 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_RPTR), 167 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB_WPTR), 168 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_BASE), 169 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_RPTR), 170 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB0_WPTR), 171 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_BASE), 172 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_RPTR), 173 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB1_WPTR), 174 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_BASE), 175 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR), 176 SOC15_REG_ENTRY_STR(GC, 0, mmCP_RB2_WPTR), 177 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_CMD_BUFSZ), 178 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_CMD_BUFSZ), 179 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_CMD_BUFSZ), 180 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_CMD_BUFSZ), 181 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_LO), 182 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BASE_HI), 183 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB1_BUFSZ), 184 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_LO), 185 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BASE_HI), 186 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_IB2_BUFSZ), 187 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_LO), 188 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BASE_HI), 189 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB1_BUFSZ), 190 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_LO), 191 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BASE_HI), 192 SOC15_REG_ENTRY_STR(GC, 0, mmCP_IB2_BUFSZ), 193 SOC15_REG_ENTRY_STR(GC, 0, mmCPF_UTCL1_STATUS), 194 SOC15_REG_ENTRY_STR(GC, 0, mmCPC_UTCL1_STATUS), 195 SOC15_REG_ENTRY_STR(GC, 0, mmCPG_UTCL1_STATUS), 196 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_PROTECTION_FAULT), 197 SOC15_REG_ENTRY_STR(GC, 0, mmGDS_VM_PROTECTION_FAULT), 198 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_STATUS), 199 SOC15_REG_ENTRY_STR(GC, 0, mmIA_UTCL1_CNTL), 200 SOC15_REG_ENTRY_STR(GC, 0, mmPA_CL_CNTL_STATUS), 201 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_UTCL1_STATUS), 202 SOC15_REG_ENTRY_STR(GC, 0, mmRMI_UTCL1_STATUS), 203 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_DCACHE_UTCL1_STATUS), 204 SOC15_REG_ENTRY_STR(GC, 0, mmSQC_ICACHE_UTCL1_STATUS), 205 SOC15_REG_ENTRY_STR(GC, 0, mmSQ_UTCL1_STATUS), 206 SOC15_REG_ENTRY_STR(GC, 0, mmTCP_UTCL1_STATUS), 207 SOC15_REG_ENTRY_STR(GC, 0, mmWD_UTCL1_STATUS), 208 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_CNTL), 209 SOC15_REG_ENTRY_STR(GC, 0, mmVM_L2_PROTECTION_FAULT_STATUS), 210 SOC15_REG_ENTRY_STR(GC, 0, mmCP_DEBUG), 211 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_CNTL), 212 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_INSTR_PNTR), 213 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC1_INSTR_PNTR), 214 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC2_INSTR_PNTR), 215 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_INSTR_PNTR), 216 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_INSTR_PNTR), 217 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CPC_STATUS), 218 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_STAT), 219 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_COMMAND), 220 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_MESSAGE), 221 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_1), 222 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_ARGUMENT_2), 223 SOC15_REG_ENTRY_STR(GC, 0, mmSMU_RLC_RESPONSE), 224 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SAFE_MODE), 225 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_SMU_SAFE_MODE), 226 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_INT_STAT), 227 SOC15_REG_ENTRY_STR(GC, 0, mmRLC_GPM_GENERAL_6), 228 /* SE status registers */ 229 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE0), 230 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE1), 231 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE2), 232 SOC15_REG_ENTRY_STR(GC, 0, mmGRBM_STATUS_SE3), 233 /* packet headers */ 234 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 235 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 236 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 237 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 238 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 239 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 240 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 241 SOC15_REG_ENTRY_STR(GC, 0, mmCP_CE_HEADER_DUMP), 242 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 243 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 244 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 245 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 246 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 247 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 248 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 249 SOC15_REG_ENTRY_STR(GC, 0, mmCP_PFP_HEADER_DUMP), 250 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 251 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 252 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 253 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 254 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 255 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 256 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP), 257 SOC15_REG_ENTRY_STR(GC, 0, mmCP_ME_HEADER_DUMP) 258 }; 259 260 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_9[] = { 261 /* compute queue registers */ 262 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_VMID), 263 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ACTIVE), 264 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PERSISTENT_STATE), 265 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PIPE_PRIORITY), 266 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUEUE_PRIORITY), 267 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_QUANTUM), 268 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE), 269 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_BASE_HI), 270 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_RPTR), 271 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR), 272 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI), 273 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL), 274 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_CONTROL), 275 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR), 276 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_BASE_ADDR_HI), 277 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_RPTR), 278 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_IB_CONTROL), 279 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_DEQUEUE_REQUEST), 280 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR), 281 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI), 282 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_CONTROL), 283 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_RPTR), 284 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR), 285 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_EVENTS), 286 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_LO), 287 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_BASE_ADDR_HI), 288 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_CONTROL), 289 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_OFFSET), 290 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CNTL_STACK_SIZE), 291 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_WG_STATE_OFFSET), 292 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_CTX_SAVE_SIZE), 293 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GDS_RESOURCE_STATE), 294 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_ERROR), 295 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_EOP_WPTR_MEM), 296 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_LO), 297 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_PQ_WPTR_HI), 298 SOC15_REG_ENTRY_STR(GC, 0, mmCP_HQD_GFX_STATUS), 299 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 300 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 301 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 302 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 303 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 304 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 305 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP), 306 SOC15_REG_ENTRY_STR(GC, 0, mmCP_MEC_ME1_HEADER_DUMP) 307 }; 308 309 enum ta_ras_gfx_subblock { 310 /*CPC*/ 311 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0, 312 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START, 313 TA_RAS_BLOCK__GFX_CPC_UCODE, 314 TA_RAS_BLOCK__GFX_DC_STATE_ME1, 315 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1, 316 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1, 317 TA_RAS_BLOCK__GFX_DC_STATE_ME2, 318 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2, 319 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 320 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2, 321 /* CPF*/ 322 TA_RAS_BLOCK__GFX_CPF_INDEX_START, 323 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START, 324 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1, 325 TA_RAS_BLOCK__GFX_CPF_TAG, 326 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG, 327 /* CPG*/ 328 TA_RAS_BLOCK__GFX_CPG_INDEX_START, 329 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START, 330 TA_RAS_BLOCK__GFX_CPG_DMA_TAG, 331 TA_RAS_BLOCK__GFX_CPG_TAG, 332 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG, 333 /* GDS*/ 334 TA_RAS_BLOCK__GFX_GDS_INDEX_START, 335 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START, 336 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE, 337 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM, 338 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM, 339 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 340 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM, 341 /* SPI*/ 342 TA_RAS_BLOCK__GFX_SPI_SR_MEM, 343 /* SQ*/ 344 TA_RAS_BLOCK__GFX_SQ_INDEX_START, 345 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START, 346 TA_RAS_BLOCK__GFX_SQ_LDS_D, 347 TA_RAS_BLOCK__GFX_SQ_LDS_I, 348 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/ 349 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR, 350 /* SQC (3 ranges)*/ 351 TA_RAS_BLOCK__GFX_SQC_INDEX_START, 352 /* SQC range 0*/ 353 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START, 354 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO = 355 TA_RAS_BLOCK__GFX_SQC_INDEX0_START, 356 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 357 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO, 358 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 359 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO, 360 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 361 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 362 TA_RAS_BLOCK__GFX_SQC_INDEX0_END = 363 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO, 364 /* SQC range 1*/ 365 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 366 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM = 367 TA_RAS_BLOCK__GFX_SQC_INDEX1_START, 368 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 369 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO, 370 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM, 371 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM, 372 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO, 373 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO, 374 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 375 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 376 TA_RAS_BLOCK__GFX_SQC_INDEX1_END = 377 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM, 378 /* SQC range 2*/ 379 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 380 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM = 381 TA_RAS_BLOCK__GFX_SQC_INDEX2_START, 382 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 383 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO, 384 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM, 385 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM, 386 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO, 387 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO, 388 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 389 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 390 TA_RAS_BLOCK__GFX_SQC_INDEX2_END = 391 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM, 392 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END, 393 /* TA*/ 394 TA_RAS_BLOCK__GFX_TA_INDEX_START, 395 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START, 396 TA_RAS_BLOCK__GFX_TA_FS_AFIFO, 397 TA_RAS_BLOCK__GFX_TA_FL_LFIFO, 398 TA_RAS_BLOCK__GFX_TA_FX_LFIFO, 399 TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 400 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO, 401 /* TCA*/ 402 TA_RAS_BLOCK__GFX_TCA_INDEX_START, 403 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START, 404 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 405 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO, 406 /* TCC (5 sub-ranges)*/ 407 TA_RAS_BLOCK__GFX_TCC_INDEX_START, 408 /* TCC range 0*/ 409 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START, 410 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START, 411 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1, 412 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0, 413 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1, 414 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0, 415 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1, 416 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG, 417 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 418 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG, 419 /* TCC range 1*/ 420 TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 421 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START, 422 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 423 TA_RAS_BLOCK__GFX_TCC_INDEX1_END = 424 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER, 425 /* TCC range 2*/ 426 TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 427 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START, 428 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL, 429 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO, 430 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN, 431 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ, 432 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO, 433 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM, 434 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 435 TA_RAS_BLOCK__GFX_TCC_INDEX2_END = 436 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO, 437 /* TCC range 3*/ 438 TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 439 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START, 440 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 441 TA_RAS_BLOCK__GFX_TCC_INDEX3_END = 442 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM, 443 /* TCC range 4*/ 444 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 445 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN = 446 TA_RAS_BLOCK__GFX_TCC_INDEX4_START, 447 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 448 TA_RAS_BLOCK__GFX_TCC_INDEX4_END = 449 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER, 450 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END, 451 /* TCI*/ 452 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM, 453 /* TCP*/ 454 TA_RAS_BLOCK__GFX_TCP_INDEX_START, 455 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START, 456 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM, 457 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO, 458 TA_RAS_BLOCK__GFX_TCP_VM_FIFO, 459 TA_RAS_BLOCK__GFX_TCP_DB_RAM, 460 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0, 461 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 462 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1, 463 /* TD*/ 464 TA_RAS_BLOCK__GFX_TD_INDEX_START, 465 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START, 466 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI, 467 TA_RAS_BLOCK__GFX_TD_CS_FIFO, 468 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO, 469 /* EA (3 sub-ranges)*/ 470 TA_RAS_BLOCK__GFX_EA_INDEX_START, 471 /* EA range 0*/ 472 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START, 473 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START, 474 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM, 475 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM, 476 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM, 477 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM, 478 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM, 479 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM, 480 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 481 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM, 482 /* EA range 1*/ 483 TA_RAS_BLOCK__GFX_EA_INDEX1_START, 484 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START, 485 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM, 486 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM, 487 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM, 488 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM, 489 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM, 490 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 491 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM, 492 /* EA range 2*/ 493 TA_RAS_BLOCK__GFX_EA_INDEX2_START, 494 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START, 495 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM, 496 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM, 497 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 498 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM, 499 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END, 500 /* UTC VM L2 bank*/ 501 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE, 502 /* UTC VM walker*/ 503 TA_RAS_BLOCK__UTC_VML2_WALKER, 504 /* UTC ATC L2 2MB cache*/ 505 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK, 506 /* UTC ATC L2 4KB cache*/ 507 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK, 508 TA_RAS_BLOCK__GFX_MAX 509 }; 510 511 struct ras_gfx_subblock { 512 unsigned char *name; 513 int ta_subblock; 514 int hw_supported_error_type; 515 int sw_supported_error_type; 516 }; 517 518 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \ 519 [AMDGPU_RAS_BLOCK__##subblock] = { \ 520 #subblock, \ 521 TA_RAS_BLOCK__##subblock, \ 522 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \ 523 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \ 524 } 525 526 static const struct ras_gfx_subblock ras_gfx_subblocks[] = { 527 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1), 528 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1), 529 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 530 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 531 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0), 532 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 533 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 534 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 535 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0), 536 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0), 537 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1), 538 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0), 539 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1), 540 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1), 541 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 542 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0), 543 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0, 544 0), 545 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0, 546 0), 547 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0), 548 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0), 549 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0), 550 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1), 551 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0), 552 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0), 553 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1), 554 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 555 0, 0), 556 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 557 0), 558 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 559 0, 0), 560 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0, 561 0), 562 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0, 563 0, 0), 564 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 565 0), 566 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 567 1), 568 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 569 0, 0, 0), 570 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 571 0), 572 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 573 0), 574 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 575 0), 576 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 577 0), 578 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 579 0), 580 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 581 0, 0), 582 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 583 0), 584 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0, 585 0), 586 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0, 587 0, 0, 0), 588 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 589 0), 590 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 591 0), 592 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0, 593 0), 594 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0, 595 0), 596 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0, 597 0), 598 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0, 599 0, 0), 600 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0, 601 0), 602 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1), 603 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 604 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 605 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 606 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0), 607 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0), 608 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 609 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1), 610 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0, 611 1), 612 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0, 613 1), 614 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0, 615 1), 616 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0, 617 0), 618 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0, 619 0), 620 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 621 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0), 622 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0), 623 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0), 624 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0), 625 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0), 626 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 627 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0), 628 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0), 629 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 630 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0), 631 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0, 632 0), 633 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 634 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0, 635 0), 636 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0, 637 0, 0), 638 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0, 639 0), 640 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 641 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1), 642 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0), 643 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 644 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0), 645 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0), 646 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0), 647 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0), 648 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1), 649 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0), 650 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0), 651 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1), 652 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 653 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 654 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 655 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0), 656 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 657 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0), 658 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0), 659 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 660 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 661 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 662 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0), 663 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0), 664 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 665 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0), 666 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0), 667 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0), 668 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0), 669 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0), 670 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0), 671 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0), 672 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0), 673 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0), 674 }; 675 676 static const struct soc15_reg_golden golden_settings_gc_9_0[] = 677 { 678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000), 680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 684 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 685 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 686 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 687 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x00ffff87), 688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x00ffff8f), 689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff), 695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 698 }; 699 700 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] = 701 { 702 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107), 703 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 704 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 705 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 706 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 707 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 708 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042), 709 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 710 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000), 711 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 712 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 713 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 714 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 715 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 716 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 717 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107), 718 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800), 719 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080) 720 }; 721 722 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] = 723 { 724 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080), 725 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 726 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 727 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042), 728 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042), 729 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400), 730 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000), 731 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000), 732 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107), 733 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000), 734 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000) 735 }; 736 737 static const struct soc15_reg_golden golden_settings_gc_9_1[] = 738 { 739 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 740 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080), 741 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080), 742 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080), 743 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 744 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 745 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080), 746 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 747 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 748 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 749 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080), 750 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080), 751 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080), 752 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080), 753 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080), 754 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 755 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 756 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 757 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 758 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff), 759 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080), 760 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 761 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 762 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 763 }; 764 765 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] = 766 { 767 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000), 768 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042), 769 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042), 770 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000), 771 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000), 772 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000), 773 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800) 774 }; 775 776 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] = 777 { 778 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000), 779 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 780 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 781 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080), 782 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080), 783 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080), 784 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041), 785 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041), 786 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 787 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 788 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080), 789 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080), 790 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080), 791 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080), 792 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080), 793 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 794 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010), 795 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 796 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080), 797 }; 798 799 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] = 800 { 801 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 802 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000), 803 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400), 804 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042), 805 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042), 806 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 807 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 808 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 809 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 810 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 811 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120), 812 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc), 813 }; 814 815 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] = 816 { 817 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff), 818 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000), 819 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382) 820 }; 821 822 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] = 823 { 824 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420), 825 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000), 826 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024), 827 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001), 828 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000), 829 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000), 830 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800), 831 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800), 832 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87), 833 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f), 834 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000), 835 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000), 836 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68), 837 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197), 838 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000), 839 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff) 840 }; 841 842 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] = 843 { 844 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080), 845 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104), 846 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000), 847 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041), 848 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041), 849 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000), 850 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107), 851 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000), 852 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410), 853 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000), 854 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800), 855 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800), 856 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000) 857 }; 858 859 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] = 860 { 861 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042), 862 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000), 863 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e), 864 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca), 865 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098), 866 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3), 867 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1), 868 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135), 869 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000), 870 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00), 871 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_UTCL1_CNTL1, 0x30000000, 0x30000000) 872 }; 873 874 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = { 875 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)}, 876 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)}, 877 }; 878 879 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] = 880 { 881 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 882 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 883 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 884 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 885 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 886 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 887 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 888 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0, 889 }; 890 891 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] = 892 { 893 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0, 894 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0, 895 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0, 896 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0, 897 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0, 898 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0, 899 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0, 900 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0, 901 }; 902 903 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042 904 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041 905 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042 906 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041 907 908 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev); 909 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev); 910 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev); 911 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev); 912 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 913 struct amdgpu_cu_info *cu_info); 914 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev); 915 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds); 916 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring); 917 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 918 void *ras_error_status); 919 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 920 void *inject_if, uint32_t instance_mask); 921 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev); 922 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev, 923 unsigned int vmid); 924 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 925 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 926 927 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring, 928 uint64_t queue_mask) 929 { 930 struct amdgpu_device *adev = kiq_ring->adev; 931 u64 shader_mc_addr; 932 933 /* Cleaner shader MC address */ 934 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; 935 936 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 937 amdgpu_ring_write(kiq_ring, 938 PACKET3_SET_RESOURCES_VMID_MASK(0) | 939 /* vmid_mask:0* queue_type:0 (KIQ) */ 940 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); 941 amdgpu_ring_write(kiq_ring, 942 lower_32_bits(queue_mask)); /* queue mask lo */ 943 amdgpu_ring_write(kiq_ring, 944 upper_32_bits(queue_mask)); /* queue mask hi */ 945 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */ 946 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */ 947 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 948 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 949 } 950 951 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring, 952 struct amdgpu_ring *ring) 953 { 954 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 955 uint64_t wptr_addr = ring->wptr_gpu_addr; 956 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 957 958 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 959 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 960 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 961 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 962 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 963 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 964 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 965 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) | 966 /*queue_type: normal compute queue */ 967 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | 968 /* alloc format: all_on_one_pipe */ 969 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | 970 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 971 /* num_queues: must be 1 */ 972 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); 973 amdgpu_ring_write(kiq_ring, 974 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 975 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 976 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 977 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 978 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 979 } 980 981 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 982 struct amdgpu_ring *ring, 983 enum amdgpu_unmap_queues_action action, 984 u64 gpu_addr, u64 seq) 985 { 986 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 987 988 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 989 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 990 PACKET3_UNMAP_QUEUES_ACTION(action) | 991 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 992 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 993 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 994 amdgpu_ring_write(kiq_ring, 995 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 996 997 if (action == PREEMPT_QUEUES_NO_UNMAP) { 998 amdgpu_ring_write(kiq_ring, lower_32_bits(ring->wptr & ring->buf_mask)); 999 amdgpu_ring_write(kiq_ring, 0); 1000 amdgpu_ring_write(kiq_ring, 0); 1001 1002 } else { 1003 amdgpu_ring_write(kiq_ring, 0); 1004 amdgpu_ring_write(kiq_ring, 0); 1005 amdgpu_ring_write(kiq_ring, 0); 1006 } 1007 } 1008 1009 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring, 1010 struct amdgpu_ring *ring, 1011 u64 addr, 1012 u64 seq) 1013 { 1014 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 1015 1016 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 1017 amdgpu_ring_write(kiq_ring, 1018 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 1019 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 1020 PACKET3_QUERY_STATUS_COMMAND(2)); 1021 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 1022 amdgpu_ring_write(kiq_ring, 1023 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 1024 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 1025 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 1026 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 1027 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 1028 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 1029 } 1030 1031 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 1032 uint16_t pasid, uint32_t flush_type, 1033 bool all_hub) 1034 { 1035 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 1036 amdgpu_ring_write(kiq_ring, 1037 PACKET3_INVALIDATE_TLBS_DST_SEL(1) | 1038 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 1039 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 1040 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 1041 } 1042 1043 1044 static void gfx_v9_0_kiq_reset_hw_queue(struct amdgpu_ring *kiq_ring, uint32_t queue_type, 1045 uint32_t me_id, uint32_t pipe_id, uint32_t queue_id, 1046 uint32_t xcc_id, uint32_t vmid) 1047 { 1048 struct amdgpu_device *adev = kiq_ring->adev; 1049 unsigned i; 1050 1051 /* enter save mode */ 1052 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id); 1053 mutex_lock(&adev->srbm_mutex); 1054 soc15_grbm_select(adev, me_id, pipe_id, queue_id, 0, 0); 1055 1056 if (queue_type == AMDGPU_RING_TYPE_COMPUTE) { 1057 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0x2); 1058 WREG32_SOC15(GC, 0, mmSPI_COMPUTE_QUEUE_RESET, 0x1); 1059 /* wait till dequeue take effects */ 1060 for (i = 0; i < adev->usec_timeout; i++) { 1061 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 1062 break; 1063 udelay(1); 1064 } 1065 if (i >= adev->usec_timeout) 1066 dev_err(adev->dev, "fail to wait on hqd deactive\n"); 1067 } else { 1068 dev_err(adev->dev, "reset queue_type(%d) not supported\n", queue_type); 1069 } 1070 1071 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 1072 mutex_unlock(&adev->srbm_mutex); 1073 /* exit safe mode */ 1074 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id); 1075 } 1076 1077 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = { 1078 .kiq_set_resources = gfx_v9_0_kiq_set_resources, 1079 .kiq_map_queues = gfx_v9_0_kiq_map_queues, 1080 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues, 1081 .kiq_query_status = gfx_v9_0_kiq_query_status, 1082 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs, 1083 .kiq_reset_hw_queue = gfx_v9_0_kiq_reset_hw_queue, 1084 .set_resources_size = 8, 1085 .map_queues_size = 7, 1086 .unmap_queues_size = 6, 1087 .query_status_size = 7, 1088 .invalidate_tlbs_size = 2, 1089 }; 1090 1091 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 1092 { 1093 adev->gfx.kiq[0].pmf = &gfx_v9_0_kiq_pm4_funcs; 1094 } 1095 1096 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev) 1097 { 1098 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1099 case IP_VERSION(9, 0, 1): 1100 soc15_program_register_sequence(adev, 1101 golden_settings_gc_9_0, 1102 ARRAY_SIZE(golden_settings_gc_9_0)); 1103 soc15_program_register_sequence(adev, 1104 golden_settings_gc_9_0_vg10, 1105 ARRAY_SIZE(golden_settings_gc_9_0_vg10)); 1106 break; 1107 case IP_VERSION(9, 2, 1): 1108 soc15_program_register_sequence(adev, 1109 golden_settings_gc_9_2_1, 1110 ARRAY_SIZE(golden_settings_gc_9_2_1)); 1111 soc15_program_register_sequence(adev, 1112 golden_settings_gc_9_2_1_vg12, 1113 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12)); 1114 break; 1115 case IP_VERSION(9, 4, 0): 1116 soc15_program_register_sequence(adev, 1117 golden_settings_gc_9_0, 1118 ARRAY_SIZE(golden_settings_gc_9_0)); 1119 soc15_program_register_sequence(adev, 1120 golden_settings_gc_9_0_vg20, 1121 ARRAY_SIZE(golden_settings_gc_9_0_vg20)); 1122 break; 1123 case IP_VERSION(9, 4, 1): 1124 soc15_program_register_sequence(adev, 1125 golden_settings_gc_9_4_1_arct, 1126 ARRAY_SIZE(golden_settings_gc_9_4_1_arct)); 1127 break; 1128 case IP_VERSION(9, 2, 2): 1129 case IP_VERSION(9, 1, 0): 1130 soc15_program_register_sequence(adev, golden_settings_gc_9_1, 1131 ARRAY_SIZE(golden_settings_gc_9_1)); 1132 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1133 soc15_program_register_sequence(adev, 1134 golden_settings_gc_9_1_rv2, 1135 ARRAY_SIZE(golden_settings_gc_9_1_rv2)); 1136 else 1137 soc15_program_register_sequence(adev, 1138 golden_settings_gc_9_1_rv1, 1139 ARRAY_SIZE(golden_settings_gc_9_1_rv1)); 1140 break; 1141 case IP_VERSION(9, 3, 0): 1142 soc15_program_register_sequence(adev, 1143 golden_settings_gc_9_1_rn, 1144 ARRAY_SIZE(golden_settings_gc_9_1_rn)); 1145 return; /* for renoir, don't need common goldensetting */ 1146 case IP_VERSION(9, 4, 2): 1147 gfx_v9_4_2_init_golden_registers(adev, 1148 adev->smuio.funcs->get_die_id(adev)); 1149 break; 1150 default: 1151 break; 1152 } 1153 1154 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && 1155 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2))) 1156 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common, 1157 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common)); 1158 } 1159 1160 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 1161 bool wc, uint32_t reg, uint32_t val) 1162 { 1163 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 1164 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 1165 WRITE_DATA_DST_SEL(0) | 1166 (wc ? WR_CONFIRM : 0)); 1167 amdgpu_ring_write(ring, reg); 1168 amdgpu_ring_write(ring, 0); 1169 amdgpu_ring_write(ring, val); 1170 } 1171 1172 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 1173 int mem_space, int opt, uint32_t addr0, 1174 uint32_t addr1, uint32_t ref, uint32_t mask, 1175 uint32_t inv) 1176 { 1177 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 1178 amdgpu_ring_write(ring, 1179 /* memory (1) or register (0) */ 1180 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 1181 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 1182 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 1183 WAIT_REG_MEM_ENGINE(eng_sel))); 1184 1185 if (mem_space) 1186 WARN_ON(addr0 & 0x3); /* Dword align */ 1187 amdgpu_ring_write(ring, addr0); 1188 amdgpu_ring_write(ring, addr1); 1189 amdgpu_ring_write(ring, ref); 1190 amdgpu_ring_write(ring, mask); 1191 amdgpu_ring_write(ring, inv); /* poll interval */ 1192 } 1193 1194 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring) 1195 { 1196 struct amdgpu_device *adev = ring->adev; 1197 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1198 uint32_t tmp = 0; 1199 unsigned i; 1200 int r; 1201 1202 WREG32(scratch, 0xCAFEDEAD); 1203 r = amdgpu_ring_alloc(ring, 3); 1204 if (r) 1205 return r; 1206 1207 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 1208 amdgpu_ring_write(ring, scratch - PACKET3_SET_UCONFIG_REG_START); 1209 amdgpu_ring_write(ring, 0xDEADBEEF); 1210 amdgpu_ring_commit(ring); 1211 1212 for (i = 0; i < adev->usec_timeout; i++) { 1213 tmp = RREG32(scratch); 1214 if (tmp == 0xDEADBEEF) 1215 break; 1216 udelay(1); 1217 } 1218 1219 if (i >= adev->usec_timeout) 1220 r = -ETIMEDOUT; 1221 return r; 1222 } 1223 1224 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 1225 { 1226 struct amdgpu_device *adev = ring->adev; 1227 struct amdgpu_ib ib; 1228 struct dma_fence *f = NULL; 1229 1230 unsigned index; 1231 uint64_t gpu_addr; 1232 uint32_t tmp; 1233 long r; 1234 1235 r = amdgpu_wb_get(adev, &index); 1236 if (r) 1237 return r; 1238 1239 gpu_addr = adev->wb.gpu_addr + (index * 4); 1240 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 1241 memset(&ib, 0, sizeof(ib)); 1242 1243 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 1244 if (r) 1245 goto err1; 1246 1247 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 1248 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 1249 ib.ptr[2] = lower_32_bits(gpu_addr); 1250 ib.ptr[3] = upper_32_bits(gpu_addr); 1251 ib.ptr[4] = 0xDEADBEEF; 1252 ib.length_dw = 5; 1253 1254 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 1255 if (r) 1256 goto err2; 1257 1258 r = dma_fence_wait_timeout(f, false, timeout); 1259 if (r == 0) { 1260 r = -ETIMEDOUT; 1261 goto err2; 1262 } else if (r < 0) { 1263 goto err2; 1264 } 1265 1266 tmp = adev->wb.wb[index]; 1267 if (tmp == 0xDEADBEEF) 1268 r = 0; 1269 else 1270 r = -EINVAL; 1271 1272 err2: 1273 amdgpu_ib_free(&ib, NULL); 1274 dma_fence_put(f); 1275 err1: 1276 amdgpu_wb_free(adev, index); 1277 return r; 1278 } 1279 1280 1281 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev) 1282 { 1283 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1284 amdgpu_ucode_release(&adev->gfx.me_fw); 1285 amdgpu_ucode_release(&adev->gfx.ce_fw); 1286 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1287 amdgpu_ucode_release(&adev->gfx.mec_fw); 1288 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1289 1290 kfree(adev->gfx.rlc.register_list_format); 1291 } 1292 1293 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev) 1294 { 1295 adev->gfx.me_fw_write_wait = false; 1296 adev->gfx.mec_fw_write_wait = false; 1297 1298 if ((amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) && 1299 (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) && 1300 ((adev->gfx.mec_fw_version < 0x000001a5) || 1301 (adev->gfx.mec_feature_version < 46) || 1302 (adev->gfx.pfp_fw_version < 0x000000b7) || 1303 (adev->gfx.pfp_feature_version < 46))) 1304 drm_warn_once(adev_to_drm(adev), 1305 "CP firmware version too old, please update!"); 1306 1307 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1308 case IP_VERSION(9, 0, 1): 1309 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1310 (adev->gfx.me_feature_version >= 42) && 1311 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1312 (adev->gfx.pfp_feature_version >= 42)) 1313 adev->gfx.me_fw_write_wait = true; 1314 1315 if ((adev->gfx.mec_fw_version >= 0x00000193) && 1316 (adev->gfx.mec_feature_version >= 42)) 1317 adev->gfx.mec_fw_write_wait = true; 1318 break; 1319 case IP_VERSION(9, 2, 1): 1320 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1321 (adev->gfx.me_feature_version >= 44) && 1322 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1323 (adev->gfx.pfp_feature_version >= 44)) 1324 adev->gfx.me_fw_write_wait = true; 1325 1326 if ((adev->gfx.mec_fw_version >= 0x00000196) && 1327 (adev->gfx.mec_feature_version >= 44)) 1328 adev->gfx.mec_fw_write_wait = true; 1329 break; 1330 case IP_VERSION(9, 4, 0): 1331 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1332 (adev->gfx.me_feature_version >= 44) && 1333 (adev->gfx.pfp_fw_version >= 0x000000b2) && 1334 (adev->gfx.pfp_feature_version >= 44)) 1335 adev->gfx.me_fw_write_wait = true; 1336 1337 if ((adev->gfx.mec_fw_version >= 0x00000197) && 1338 (adev->gfx.mec_feature_version >= 44)) 1339 adev->gfx.mec_fw_write_wait = true; 1340 break; 1341 case IP_VERSION(9, 1, 0): 1342 case IP_VERSION(9, 2, 2): 1343 if ((adev->gfx.me_fw_version >= 0x0000009c) && 1344 (adev->gfx.me_feature_version >= 42) && 1345 (adev->gfx.pfp_fw_version >= 0x000000b1) && 1346 (adev->gfx.pfp_feature_version >= 42)) 1347 adev->gfx.me_fw_write_wait = true; 1348 1349 if ((adev->gfx.mec_fw_version >= 0x00000192) && 1350 (adev->gfx.mec_feature_version >= 42)) 1351 adev->gfx.mec_fw_write_wait = true; 1352 break; 1353 default: 1354 adev->gfx.me_fw_write_wait = true; 1355 adev->gfx.mec_fw_write_wait = true; 1356 break; 1357 } 1358 } 1359 1360 struct amdgpu_gfxoff_quirk { 1361 u16 chip_vendor; 1362 u16 chip_device; 1363 u16 subsys_vendor; 1364 u16 subsys_device; 1365 u8 revision; 1366 }; 1367 1368 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = { 1369 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */ 1370 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, 1371 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */ 1372 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 }, 1373 /* GFXOFF is unstable on C6 parts with a VBIOS 113-RAVEN-114 */ 1374 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc6 }, 1375 /* Apple MacBook Pro (15-inch, 2019) Radeon Pro Vega 20 4 GB */ 1376 { 0x1002, 0x69af, 0x106b, 0x019a, 0xc0 }, 1377 /* https://bbs.openkylin.top/t/topic/171497 */ 1378 { 0x1002, 0x15d8, 0x19e5, 0x3e14, 0xc2 }, 1379 /* HP 705G4 DM with R5 2400G */ 1380 { 0x1002, 0x15dd, 0x103c, 0x8464, 0xd6 }, 1381 { 0, 0, 0, 0, 0 }, 1382 }; 1383 1384 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev) 1385 { 1386 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list; 1387 1388 while (p && p->chip_device != 0) { 1389 if (pdev->vendor == p->chip_vendor && 1390 pdev->device == p->chip_device && 1391 pdev->subsystem_vendor == p->subsys_vendor && 1392 pdev->subsystem_device == p->subsys_device && 1393 pdev->revision == p->revision) { 1394 return true; 1395 } 1396 ++p; 1397 } 1398 return false; 1399 } 1400 1401 static bool is_raven_kicker(struct amdgpu_device *adev) 1402 { 1403 if (adev->pm.fw_version >= 0x41e2b) 1404 return true; 1405 else 1406 return false; 1407 } 1408 1409 static bool check_if_enlarge_doorbell_range(struct amdgpu_device *adev) 1410 { 1411 if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) && 1412 (adev->gfx.me_fw_version >= 0x000000a5) && 1413 (adev->gfx.me_feature_version >= 52)) 1414 return true; 1415 else 1416 return false; 1417 } 1418 1419 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev) 1420 { 1421 if (gfx_v9_0_should_disable_gfxoff(adev->pdev)) 1422 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1423 1424 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1425 case IP_VERSION(9, 0, 1): 1426 case IP_VERSION(9, 2, 1): 1427 case IP_VERSION(9, 4, 0): 1428 break; 1429 case IP_VERSION(9, 2, 2): 1430 case IP_VERSION(9, 1, 0): 1431 if (!((adev->apu_flags & AMD_APU_IS_RAVEN2) || 1432 (adev->apu_flags & AMD_APU_IS_PICASSO)) && 1433 ((!is_raven_kicker(adev) && 1434 adev->gfx.rlc_fw_version < 531) || 1435 (adev->gfx.rlc_feature_version < 1) || 1436 !adev->gfx.rlc.is_rlc_v2_1)) 1437 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 1438 1439 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1440 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1441 AMD_PG_SUPPORT_CP | 1442 AMD_PG_SUPPORT_RLC_SMU_HS; 1443 break; 1444 case IP_VERSION(9, 3, 0): 1445 if (adev->pm.pp_feature & PP_GFXOFF_MASK) 1446 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG | 1447 AMD_PG_SUPPORT_CP | 1448 AMD_PG_SUPPORT_RLC_SMU_HS; 1449 break; 1450 default: 1451 break; 1452 } 1453 } 1454 1455 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev, 1456 char *chip_name) 1457 { 1458 int err; 1459 1460 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 1461 AMDGPU_UCODE_REQUIRED, 1462 "amdgpu/%s_pfp.bin", chip_name); 1463 if (err) 1464 goto out; 1465 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 1466 1467 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 1468 AMDGPU_UCODE_REQUIRED, 1469 "amdgpu/%s_me.bin", chip_name); 1470 if (err) 1471 goto out; 1472 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 1473 1474 err = amdgpu_ucode_request(adev, &adev->gfx.ce_fw, 1475 AMDGPU_UCODE_REQUIRED, 1476 "amdgpu/%s_ce.bin", chip_name); 1477 if (err) 1478 goto out; 1479 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE); 1480 1481 out: 1482 if (err) { 1483 amdgpu_ucode_release(&adev->gfx.pfp_fw); 1484 amdgpu_ucode_release(&adev->gfx.me_fw); 1485 amdgpu_ucode_release(&adev->gfx.ce_fw); 1486 } 1487 return err; 1488 } 1489 1490 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev, 1491 char *chip_name) 1492 { 1493 int err; 1494 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1495 uint16_t version_major; 1496 uint16_t version_minor; 1497 uint32_t smu_version; 1498 1499 /* 1500 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin 1501 * instead of picasso_rlc.bin. 1502 * Judgment method: 1503 * PCO AM4: revision >= 0xC8 && revision <= 0xCF 1504 * or revision >= 0xD8 && revision <= 0xDF 1505 * otherwise is PCO FP5 1506 */ 1507 if (!strcmp(chip_name, "picasso") && 1508 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) || 1509 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF)))) 1510 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1511 AMDGPU_UCODE_REQUIRED, 1512 "amdgpu/%s_rlc_am4.bin", chip_name); 1513 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) && 1514 (smu_version >= 0x41e2b)) 1515 /** 1516 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly. 1517 */ 1518 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1519 AMDGPU_UCODE_REQUIRED, 1520 "amdgpu/%s_kicker_rlc.bin", chip_name); 1521 else 1522 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 1523 AMDGPU_UCODE_REQUIRED, 1524 "amdgpu/%s_rlc.bin", chip_name); 1525 if (err) 1526 goto out; 1527 1528 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1529 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1530 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1531 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 1532 out: 1533 if (err) 1534 amdgpu_ucode_release(&adev->gfx.rlc_fw); 1535 1536 return err; 1537 } 1538 1539 static bool gfx_v9_0_load_mec2_fw_bin_support(struct amdgpu_device *adev) 1540 { 1541 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || 1542 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || 1543 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 3, 0)) 1544 return false; 1545 1546 return true; 1547 } 1548 1549 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev, 1550 char *chip_name) 1551 { 1552 int err; 1553 1554 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) 1555 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1556 AMDGPU_UCODE_REQUIRED, 1557 "amdgpu/%s_sjt_mec.bin", chip_name); 1558 else 1559 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 1560 AMDGPU_UCODE_REQUIRED, 1561 "amdgpu/%s_mec.bin", chip_name); 1562 if (err) 1563 goto out; 1564 1565 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 1566 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 1567 1568 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) { 1569 if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN)) 1570 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1571 AMDGPU_UCODE_REQUIRED, 1572 "amdgpu/%s_sjt_mec2.bin", chip_name); 1573 else 1574 err = amdgpu_ucode_request(adev, &adev->gfx.mec2_fw, 1575 AMDGPU_UCODE_REQUIRED, 1576 "amdgpu/%s_mec2.bin", chip_name); 1577 if (!err) { 1578 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2); 1579 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT); 1580 } else { 1581 err = 0; 1582 amdgpu_ucode_release(&adev->gfx.mec2_fw); 1583 } 1584 } else { 1585 adev->gfx.mec2_fw_version = adev->gfx.mec_fw_version; 1586 adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version; 1587 } 1588 1589 gfx_v9_0_check_if_need_gfxoff(adev); 1590 gfx_v9_0_check_fw_write_wait(adev); 1591 1592 out: 1593 if (err) 1594 amdgpu_ucode_release(&adev->gfx.mec_fw); 1595 return err; 1596 } 1597 1598 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev) 1599 { 1600 char ucode_prefix[30]; 1601 int r; 1602 1603 DRM_DEBUG("\n"); 1604 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 1605 1606 /* No CPG in Arcturus */ 1607 if (adev->gfx.num_gfx_rings) { 1608 r = gfx_v9_0_init_cp_gfx_microcode(adev, ucode_prefix); 1609 if (r) 1610 return r; 1611 } 1612 1613 r = gfx_v9_0_init_rlc_microcode(adev, ucode_prefix); 1614 if (r) 1615 return r; 1616 1617 r = gfx_v9_0_init_cp_compute_microcode(adev, ucode_prefix); 1618 if (r) 1619 return r; 1620 1621 return r; 1622 } 1623 1624 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev) 1625 { 1626 u32 count = 0; 1627 const struct cs_section_def *sect = NULL; 1628 const struct cs_extent_def *ext = NULL; 1629 1630 /* begin clear state */ 1631 count += 2; 1632 /* context control state */ 1633 count += 3; 1634 1635 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 1636 for (ext = sect->section; ext->extent != NULL; ++ext) { 1637 if (sect->id == SECT_CONTEXT) 1638 count += 2 + ext->reg_count; 1639 else 1640 return 0; 1641 } 1642 } 1643 1644 /* end clear state */ 1645 count += 2; 1646 /* clear state */ 1647 count += 2; 1648 1649 return count; 1650 } 1651 1652 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 1653 { 1654 u32 count = 0; 1655 1656 if (adev->gfx.rlc.cs_data == NULL) 1657 return; 1658 if (buffer == NULL) 1659 return; 1660 1661 count = amdgpu_gfx_csb_preamble_start(buffer); 1662 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 1663 amdgpu_gfx_csb_preamble_end(buffer, count); 1664 } 1665 1666 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev) 1667 { 1668 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info; 1669 uint32_t pg_always_on_cu_num = 2; 1670 uint32_t always_on_cu_num; 1671 uint32_t i, j, k; 1672 uint32_t mask, cu_bitmap, counter; 1673 1674 if (adev->flags & AMD_IS_APU) 1675 always_on_cu_num = 4; 1676 else if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 2, 1)) 1677 always_on_cu_num = 8; 1678 else 1679 always_on_cu_num = 12; 1680 1681 mutex_lock(&adev->grbm_idx_mutex); 1682 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 1683 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 1684 mask = 1; 1685 cu_bitmap = 0; 1686 counter = 0; 1687 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 1688 1689 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 1690 if (cu_info->bitmap[0][i][j] & mask) { 1691 if (counter == pg_always_on_cu_num) 1692 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap); 1693 if (counter < always_on_cu_num) 1694 cu_bitmap |= mask; 1695 else 1696 break; 1697 counter++; 1698 } 1699 mask <<= 1; 1700 } 1701 1702 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap); 1703 cu_info->ao_cu_bitmap[i][j] = cu_bitmap; 1704 } 1705 } 1706 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1707 mutex_unlock(&adev->grbm_idx_mutex); 1708 } 1709 1710 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev) 1711 { 1712 uint32_t data; 1713 1714 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1715 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1716 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7); 1717 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1718 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16)); 1719 1720 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1721 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1722 1723 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1724 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500); 1725 1726 mutex_lock(&adev->grbm_idx_mutex); 1727 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1728 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1729 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1730 1731 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1732 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1733 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1734 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1735 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1736 1737 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1738 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1739 data &= 0x0000FFFF; 1740 data |= 0x00C00000; 1741 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1742 1743 /* 1744 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven), 1745 * programmed in gfx_v9_0_init_always_on_cu_mask() 1746 */ 1747 1748 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1749 * but used for RLC_LB_CNTL configuration */ 1750 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1751 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1752 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1753 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1754 mutex_unlock(&adev->grbm_idx_mutex); 1755 1756 gfx_v9_0_init_always_on_cu_mask(adev); 1757 } 1758 1759 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev) 1760 { 1761 uint32_t data; 1762 1763 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */ 1764 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F); 1765 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8); 1766 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077); 1767 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16)); 1768 1769 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */ 1770 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000); 1771 1772 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */ 1773 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800); 1774 1775 mutex_lock(&adev->grbm_idx_mutex); 1776 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/ 1777 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 1778 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff); 1779 1780 /* set mmRLC_LB_PARAMS = 0x003F_1006 */ 1781 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003); 1782 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010); 1783 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F); 1784 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data); 1785 1786 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */ 1787 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7); 1788 data &= 0x0000FFFF; 1789 data |= 0x00C00000; 1790 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data); 1791 1792 /* 1793 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON), 1794 * programmed in gfx_v9_0_init_always_on_cu_mask() 1795 */ 1796 1797 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved, 1798 * but used for RLC_LB_CNTL configuration */ 1799 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK; 1800 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09); 1801 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000); 1802 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data); 1803 mutex_unlock(&adev->grbm_idx_mutex); 1804 1805 gfx_v9_0_init_always_on_cu_mask(adev); 1806 } 1807 1808 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable) 1809 { 1810 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0); 1811 } 1812 1813 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev) 1814 { 1815 if (gfx_v9_0_load_mec2_fw_bin_support(adev)) 1816 return 5; 1817 else 1818 return 4; 1819 } 1820 1821 static void gfx_v9_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 1822 { 1823 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 1824 1825 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 1826 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0); 1827 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG1); 1828 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG2); 1829 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG3); 1830 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_CNTL); 1831 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, mmGRBM_GFX_INDEX); 1832 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, mmRLC_SPARE_INT); 1833 adev->gfx.rlc.rlcg_reg_access_supported = true; 1834 } 1835 1836 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev) 1837 { 1838 const struct cs_section_def *cs_data; 1839 int r; 1840 1841 adev->gfx.rlc.cs_data = gfx9_cs_data; 1842 1843 cs_data = adev->gfx.rlc.cs_data; 1844 1845 if (cs_data) { 1846 /* init clear state block */ 1847 r = amdgpu_gfx_rlc_init_csb(adev); 1848 if (r) 1849 return r; 1850 } 1851 1852 if (adev->flags & AMD_IS_APU) { 1853 /* TODO: double check the cp_table_size for RV */ 1854 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */ 1855 r = amdgpu_gfx_rlc_init_cpt(adev); 1856 if (r) 1857 return r; 1858 } 1859 1860 return 0; 1861 } 1862 1863 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev) 1864 { 1865 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 1866 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 1867 } 1868 1869 static int gfx_v9_0_mec_init(struct amdgpu_device *adev) 1870 { 1871 int r; 1872 u32 *hpd; 1873 const __le32 *fw_data; 1874 unsigned fw_size; 1875 u32 *fw; 1876 size_t mec_hpd_size; 1877 1878 const struct gfx_firmware_header_v1_0 *mec_hdr; 1879 1880 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 1881 1882 /* take ownership of the relevant compute queues */ 1883 amdgpu_gfx_compute_queue_acquire(adev); 1884 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE; 1885 if (mec_hpd_size) { 1886 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 1887 AMDGPU_GEM_DOMAIN_VRAM | 1888 AMDGPU_GEM_DOMAIN_GTT, 1889 &adev->gfx.mec.hpd_eop_obj, 1890 &adev->gfx.mec.hpd_eop_gpu_addr, 1891 (void **)&hpd); 1892 if (r) { 1893 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 1894 gfx_v9_0_mec_fini(adev); 1895 return r; 1896 } 1897 1898 memset(hpd, 0, mec_hpd_size); 1899 1900 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 1901 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 1902 } 1903 1904 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 1905 1906 fw_data = (const __le32 *) 1907 (adev->gfx.mec_fw->data + 1908 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 1909 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 1910 1911 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 1912 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1913 &adev->gfx.mec.mec_fw_obj, 1914 &adev->gfx.mec.mec_fw_gpu_addr, 1915 (void **)&fw); 1916 if (r) { 1917 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r); 1918 gfx_v9_0_mec_fini(adev); 1919 return r; 1920 } 1921 1922 memcpy(fw, fw_data, fw_size); 1923 1924 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 1925 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 1926 1927 return 0; 1928 } 1929 1930 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address) 1931 { 1932 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1933 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1934 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1935 (address << SQ_IND_INDEX__INDEX__SHIFT) | 1936 (SQ_IND_INDEX__FORCE_READ_MASK)); 1937 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1938 } 1939 1940 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd, 1941 uint32_t wave, uint32_t thread, 1942 uint32_t regno, uint32_t num, uint32_t *out) 1943 { 1944 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX, 1945 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1946 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) | 1947 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1948 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) | 1949 (SQ_IND_INDEX__FORCE_READ_MASK) | 1950 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1951 while (num--) 1952 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA); 1953 } 1954 1955 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1956 { 1957 /* type 1 wave data */ 1958 dst[(*no_fields)++] = 1; 1959 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS); 1960 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO); 1961 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI); 1962 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO); 1963 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI); 1964 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID); 1965 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0); 1966 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1); 1967 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC); 1968 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC); 1969 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS); 1970 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS); 1971 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0); 1972 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0); 1973 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_MODE); 1974 } 1975 1976 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1977 uint32_t wave, uint32_t start, 1978 uint32_t size, uint32_t *dst) 1979 { 1980 wave_read_regs( 1981 adev, simd, wave, 0, 1982 start + SQIND_WAVE_SGPRS_OFFSET, size, dst); 1983 } 1984 1985 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1986 uint32_t wave, uint32_t thread, 1987 uint32_t start, uint32_t size, 1988 uint32_t *dst) 1989 { 1990 wave_read_regs( 1991 adev, simd, wave, thread, 1992 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1993 } 1994 1995 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev, 1996 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1997 { 1998 soc15_grbm_select(adev, me, pipe, q, vm, 0); 1999 } 2000 2001 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = { 2002 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter, 2003 .select_se_sh = &gfx_v9_0_select_se_sh, 2004 .read_wave_data = &gfx_v9_0_read_wave_data, 2005 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs, 2006 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs, 2007 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q, 2008 .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask, 2009 }; 2010 2011 const struct amdgpu_ras_block_hw_ops gfx_v9_0_ras_ops = { 2012 .ras_error_inject = &gfx_v9_0_ras_error_inject, 2013 .query_ras_error_count = &gfx_v9_0_query_ras_error_count, 2014 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count, 2015 }; 2016 2017 static struct amdgpu_gfx_ras gfx_v9_0_ras = { 2018 .ras_block = { 2019 .hw_ops = &gfx_v9_0_ras_ops, 2020 }, 2021 }; 2022 2023 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev) 2024 { 2025 u32 gb_addr_config; 2026 int err; 2027 2028 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2029 case IP_VERSION(9, 0, 1): 2030 adev->gfx.config.max_hw_contexts = 8; 2031 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2032 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2033 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2034 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2035 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN; 2036 break; 2037 case IP_VERSION(9, 2, 1): 2038 adev->gfx.config.max_hw_contexts = 8; 2039 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2040 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2041 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2042 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2043 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN; 2044 drm_info(adev_to_drm(adev), "fix gfx.config for vega12\n"); 2045 break; 2046 case IP_VERSION(9, 4, 0): 2047 adev->gfx.ras = &gfx_v9_0_ras; 2048 adev->gfx.config.max_hw_contexts = 8; 2049 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2050 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2051 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2052 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2053 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2054 gb_addr_config &= ~0xf3e777ff; 2055 gb_addr_config |= 0x22014042; 2056 /* check vbios table if gpu info is not available */ 2057 err = amdgpu_atomfirmware_get_gfx_info(adev); 2058 if (err) 2059 return err; 2060 break; 2061 case IP_VERSION(9, 2, 2): 2062 case IP_VERSION(9, 1, 0): 2063 adev->gfx.config.max_hw_contexts = 8; 2064 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2065 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2066 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2067 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2068 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 2069 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN; 2070 else 2071 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN; 2072 break; 2073 case IP_VERSION(9, 4, 1): 2074 adev->gfx.ras = &gfx_v9_4_ras; 2075 adev->gfx.config.max_hw_contexts = 8; 2076 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2077 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2078 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2079 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2080 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2081 gb_addr_config &= ~0xf3e777ff; 2082 gb_addr_config |= 0x22014042; 2083 break; 2084 case IP_VERSION(9, 3, 0): 2085 adev->gfx.config.max_hw_contexts = 8; 2086 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2087 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2088 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 2089 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2090 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2091 gb_addr_config &= ~0xf3e777ff; 2092 gb_addr_config |= 0x22010042; 2093 break; 2094 case IP_VERSION(9, 4, 2): 2095 adev->gfx.ras = &gfx_v9_4_2_ras; 2096 adev->gfx.config.max_hw_contexts = 8; 2097 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 2098 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 2099 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30; 2100 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 2101 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG); 2102 gb_addr_config &= ~0xf3e777ff; 2103 gb_addr_config |= 0x22014042; 2104 /* check vbios table if gpu info is not available */ 2105 err = amdgpu_atomfirmware_get_gfx_info(adev); 2106 if (err) 2107 return err; 2108 break; 2109 default: 2110 dev_warn(adev->dev, "Unsupported GC version 0x%08x\n", 2111 amdgpu_ip_version(adev, GC_HWIP, 0)); 2112 return -EINVAL; 2113 } 2114 2115 adev->gfx.config.gb_addr_config = gb_addr_config; 2116 2117 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2118 REG_GET_FIELD( 2119 adev->gfx.config.gb_addr_config, 2120 GB_ADDR_CONFIG, 2121 NUM_PIPES); 2122 2123 adev->gfx.config.max_tile_pipes = 2124 adev->gfx.config.gb_addr_config_fields.num_pipes; 2125 2126 adev->gfx.config.gb_addr_config_fields.num_banks = 1 << 2127 REG_GET_FIELD( 2128 adev->gfx.config.gb_addr_config, 2129 GB_ADDR_CONFIG, 2130 NUM_BANKS); 2131 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2132 REG_GET_FIELD( 2133 adev->gfx.config.gb_addr_config, 2134 GB_ADDR_CONFIG, 2135 MAX_COMPRESSED_FRAGS); 2136 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2137 REG_GET_FIELD( 2138 adev->gfx.config.gb_addr_config, 2139 GB_ADDR_CONFIG, 2140 NUM_RB_PER_SE); 2141 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2142 REG_GET_FIELD( 2143 adev->gfx.config.gb_addr_config, 2144 GB_ADDR_CONFIG, 2145 NUM_SHADER_ENGINES); 2146 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2147 REG_GET_FIELD( 2148 adev->gfx.config.gb_addr_config, 2149 GB_ADDR_CONFIG, 2150 PIPE_INTERLEAVE_SIZE)); 2151 2152 return 0; 2153 } 2154 2155 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 2156 int mec, int pipe, int queue) 2157 { 2158 unsigned irq_type; 2159 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id]; 2160 unsigned int hw_prio; 2161 2162 ring = &adev->gfx.compute_ring[ring_id]; 2163 2164 /* mec0 is me1 */ 2165 ring->me = mec + 1; 2166 ring->pipe = pipe; 2167 ring->queue = queue; 2168 2169 ring->ring_obj = NULL; 2170 ring->use_doorbell = true; 2171 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 2172 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 2173 + (ring_id * GFX9_MEC_HPD_SIZE); 2174 ring->vm_hub = AMDGPU_GFXHUB(0); 2175 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 2176 2177 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 2178 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 2179 + ring->pipe; 2180 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 2181 AMDGPU_RING_PRIO_2 : AMDGPU_RING_PRIO_DEFAULT; 2182 /* type-2 packets are deprecated on MEC, use type-3 instead */ 2183 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 2184 hw_prio, NULL); 2185 } 2186 2187 static void gfx_v9_0_alloc_ip_dump(struct amdgpu_device *adev) 2188 { 2189 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9); 2190 uint32_t *ptr; 2191 uint32_t inst; 2192 2193 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL); 2194 if (!ptr) { 2195 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 2196 adev->gfx.ip_dump_core = NULL; 2197 } else { 2198 adev->gfx.ip_dump_core = ptr; 2199 } 2200 2201 /* Allocate memory for compute queue registers for all the instances */ 2202 reg_count = ARRAY_SIZE(gc_cp_reg_list_9); 2203 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 2204 adev->gfx.mec.num_queue_per_pipe; 2205 2206 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 2207 if (!ptr) { 2208 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 2209 adev->gfx.ip_dump_compute_queues = NULL; 2210 } else { 2211 adev->gfx.ip_dump_compute_queues = ptr; 2212 } 2213 } 2214 2215 static int gfx_v9_0_sw_init(struct amdgpu_ip_block *ip_block) 2216 { 2217 int i, j, k, r, ring_id; 2218 int xcc_id = 0; 2219 struct amdgpu_ring *ring; 2220 struct amdgpu_device *adev = ip_block->adev; 2221 unsigned int hw_prio; 2222 2223 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2224 case IP_VERSION(9, 0, 1): 2225 case IP_VERSION(9, 2, 1): 2226 case IP_VERSION(9, 4, 0): 2227 case IP_VERSION(9, 2, 2): 2228 case IP_VERSION(9, 1, 0): 2229 case IP_VERSION(9, 4, 1): 2230 case IP_VERSION(9, 3, 0): 2231 case IP_VERSION(9, 4, 2): 2232 adev->gfx.mec.num_mec = 2; 2233 break; 2234 default: 2235 adev->gfx.mec.num_mec = 1; 2236 break; 2237 } 2238 2239 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2240 case IP_VERSION(9, 0, 1): 2241 case IP_VERSION(9, 2, 1): 2242 case IP_VERSION(9, 4, 0): 2243 case IP_VERSION(9, 2, 2): 2244 case IP_VERSION(9, 1, 0): 2245 case IP_VERSION(9, 3, 0): 2246 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex; 2247 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex); 2248 if (adev->gfx.me_fw_version >= 167 && 2249 adev->gfx.pfp_fw_version >= 196 && 2250 adev->gfx.mec_fw_version >= 474) { 2251 adev->gfx.enable_cleaner_shader = true; 2252 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 2253 if (r) { 2254 adev->gfx.enable_cleaner_shader = false; 2255 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 2256 } 2257 } 2258 break; 2259 case IP_VERSION(9, 4, 2): 2260 adev->gfx.cleaner_shader_ptr = gfx_9_4_2_cleaner_shader_hex; 2261 adev->gfx.cleaner_shader_size = sizeof(gfx_9_4_2_cleaner_shader_hex); 2262 if (adev->gfx.mec_fw_version >= 88) { 2263 adev->gfx.enable_cleaner_shader = true; 2264 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 2265 if (r) { 2266 adev->gfx.enable_cleaner_shader = false; 2267 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 2268 } 2269 } 2270 break; 2271 default: 2272 adev->gfx.enable_cleaner_shader = false; 2273 break; 2274 } 2275 2276 adev->gfx.mec.num_pipe_per_mec = 4; 2277 adev->gfx.mec.num_queue_per_pipe = 8; 2278 2279 /* EOP Event */ 2280 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq); 2281 if (r) 2282 return r; 2283 2284 /* Bad opcode Event */ 2285 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, 2286 GFX_9_0__SRCID__CP_BAD_OPCODE_ERROR, 2287 &adev->gfx.bad_op_irq); 2288 if (r) 2289 return r; 2290 2291 /* Privileged reg */ 2292 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT, 2293 &adev->gfx.priv_reg_irq); 2294 if (r) 2295 return r; 2296 2297 /* Privileged inst */ 2298 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT, 2299 &adev->gfx.priv_inst_irq); 2300 if (r) 2301 return r; 2302 2303 /* ECC error */ 2304 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR, 2305 &adev->gfx.cp_ecc_error_irq); 2306 if (r) 2307 return r; 2308 2309 /* FUE error */ 2310 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR, 2311 &adev->gfx.cp_ecc_error_irq); 2312 if (r) 2313 return r; 2314 2315 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 2316 2317 if (adev->gfx.rlc.funcs) { 2318 if (adev->gfx.rlc.funcs->init) { 2319 r = adev->gfx.rlc.funcs->init(adev); 2320 if (r) { 2321 dev_err(adev->dev, "Failed to init rlc BOs!\n"); 2322 return r; 2323 } 2324 } 2325 } 2326 2327 r = gfx_v9_0_mec_init(adev); 2328 if (r) { 2329 DRM_ERROR("Failed to init MEC BOs!\n"); 2330 return r; 2331 } 2332 2333 /* set up the gfx ring */ 2334 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 2335 ring = &adev->gfx.gfx_ring[i]; 2336 ring->ring_obj = NULL; 2337 if (!i) 2338 sprintf(ring->name, "gfx"); 2339 else 2340 sprintf(ring->name, "gfx_%d", i); 2341 ring->use_doorbell = true; 2342 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2343 2344 /* disable scheduler on the real ring */ 2345 ring->no_scheduler = adev->gfx.mcbp; 2346 ring->vm_hub = AMDGPU_GFXHUB(0); 2347 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2348 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, 2349 AMDGPU_RING_PRIO_DEFAULT, NULL); 2350 if (r) 2351 return r; 2352 } 2353 2354 /* set up the software rings */ 2355 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { 2356 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) { 2357 ring = &adev->gfx.sw_gfx_ring[i]; 2358 ring->ring_obj = NULL; 2359 strscpy(ring->name, amdgpu_sw_ring_name(i), sizeof(ring->name)); 2360 ring->use_doorbell = true; 2361 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 2362 ring->is_sw_ring = true; 2363 hw_prio = amdgpu_sw_ring_priority(i); 2364 ring->vm_hub = AMDGPU_GFXHUB(0); 2365 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, 2366 AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP, hw_prio, 2367 NULL); 2368 if (r) 2369 return r; 2370 ring->wptr = 0; 2371 } 2372 2373 /* init the muxer and add software rings */ 2374 r = amdgpu_ring_mux_init(&adev->gfx.muxer, &adev->gfx.gfx_ring[0], 2375 GFX9_NUM_SW_GFX_RINGS); 2376 if (r) { 2377 DRM_ERROR("amdgpu_ring_mux_init failed(%d)\n", r); 2378 return r; 2379 } 2380 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) { 2381 r = amdgpu_ring_mux_add_sw_ring(&adev->gfx.muxer, 2382 &adev->gfx.sw_gfx_ring[i]); 2383 if (r) { 2384 DRM_ERROR("amdgpu_ring_mux_add_sw_ring failed(%d)\n", r); 2385 return r; 2386 } 2387 } 2388 } 2389 2390 /* set up the compute queues - allocate horizontally across pipes */ 2391 ring_id = 0; 2392 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 2393 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 2394 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 2395 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 2396 k, j)) 2397 continue; 2398 2399 r = gfx_v9_0_compute_ring_init(adev, 2400 ring_id, 2401 i, k, j); 2402 if (r) 2403 return r; 2404 2405 ring_id++; 2406 } 2407 } 2408 } 2409 2410 /* TODO: Add queue reset mask when FW fully supports it */ 2411 adev->gfx.gfx_supported_reset = 2412 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 2413 adev->gfx.compute_supported_reset = 2414 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 2415 if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_gpu_ring_reset) { 2416 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 2417 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 2418 } 2419 2420 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE, 0); 2421 if (r) { 2422 DRM_ERROR("Failed to init KIQ BOs!\n"); 2423 return r; 2424 } 2425 2426 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 2427 if (r) 2428 return r; 2429 2430 /* create MQD for all compute queues as wel as KIQ for SRIOV case */ 2431 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation), 0); 2432 if (r) 2433 return r; 2434 2435 adev->gfx.ce_ram_size = 0x8000; 2436 2437 r = gfx_v9_0_gpu_early_init(adev); 2438 if (r) 2439 return r; 2440 2441 if (amdgpu_gfx_ras_sw_init(adev)) { 2442 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 2443 return -EINVAL; 2444 } 2445 2446 r = amdgpu_gfx_sysfs_init(adev); 2447 if (r) 2448 return r; 2449 2450 gfx_v9_0_alloc_ip_dump(adev); 2451 2452 return 0; 2453 } 2454 2455 2456 static int gfx_v9_0_sw_fini(struct amdgpu_ip_block *ip_block) 2457 { 2458 int i; 2459 struct amdgpu_device *adev = ip_block->adev; 2460 2461 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { 2462 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 2463 amdgpu_ring_fini(&adev->gfx.sw_gfx_ring[i]); 2464 amdgpu_ring_mux_fini(&adev->gfx.muxer); 2465 } 2466 2467 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 2468 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 2469 for (i = 0; i < adev->gfx.num_compute_rings; i++) 2470 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 2471 2472 amdgpu_gfx_mqd_sw_fini(adev, 0); 2473 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 2474 amdgpu_gfx_kiq_fini(adev, 0); 2475 2476 amdgpu_gfx_cleaner_shader_sw_fini(adev); 2477 2478 gfx_v9_0_mec_fini(adev); 2479 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 2480 &adev->gfx.rlc.clear_state_gpu_addr, 2481 (void **)&adev->gfx.rlc.cs_ptr); 2482 if (adev->flags & AMD_IS_APU) { 2483 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 2484 &adev->gfx.rlc.cp_table_gpu_addr, 2485 (void **)&adev->gfx.rlc.cp_table_ptr); 2486 } 2487 gfx_v9_0_free_microcode(adev); 2488 2489 amdgpu_gfx_sysfs_fini(adev); 2490 2491 kfree(adev->gfx.ip_dump_core); 2492 kfree(adev->gfx.ip_dump_compute_queues); 2493 2494 return 0; 2495 } 2496 2497 2498 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev) 2499 { 2500 /* TODO */ 2501 } 2502 2503 void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, 2504 u32 instance, int xcc_id) 2505 { 2506 u32 data; 2507 2508 if (instance == 0xffffffff) 2509 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1); 2510 else 2511 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance); 2512 2513 if (se_num == 0xffffffff) 2514 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1); 2515 else 2516 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2517 2518 if (sh_num == 0xffffffff) 2519 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1); 2520 else 2521 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num); 2522 2523 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data); 2524 } 2525 2526 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2527 { 2528 u32 data, mask; 2529 2530 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE); 2531 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE); 2532 2533 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK; 2534 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT; 2535 2536 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se / 2537 adev->gfx.config.max_sh_per_se); 2538 2539 return (~data) & mask; 2540 } 2541 2542 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev) 2543 { 2544 int i, j; 2545 u32 data; 2546 u32 active_rbs = 0; 2547 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se / 2548 adev->gfx.config.max_sh_per_se; 2549 2550 mutex_lock(&adev->grbm_idx_mutex); 2551 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2552 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2553 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 2554 data = gfx_v9_0_get_rb_active_bitmap(adev); 2555 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) * 2556 rb_bitmap_width_per_sh); 2557 } 2558 } 2559 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 2560 mutex_unlock(&adev->grbm_idx_mutex); 2561 2562 adev->gfx.config.backend_enable_mask = active_rbs; 2563 adev->gfx.config.num_rbs = hweight32(active_rbs); 2564 } 2565 2566 static void gfx_v9_0_debug_trap_config_init(struct amdgpu_device *adev, 2567 uint32_t first_vmid, 2568 uint32_t last_vmid) 2569 { 2570 uint32_t data; 2571 uint32_t trap_config_vmid_mask = 0; 2572 int i; 2573 2574 /* Calculate trap config vmid mask */ 2575 for (i = first_vmid; i < last_vmid; i++) 2576 trap_config_vmid_mask |= (1 << i); 2577 2578 data = REG_SET_FIELD(0, SPI_GDBG_TRAP_CONFIG, 2579 VMID_SEL, trap_config_vmid_mask); 2580 data = REG_SET_FIELD(data, SPI_GDBG_TRAP_CONFIG, 2581 TRAP_EN, 1); 2582 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_CONFIG), data); 2583 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_MASK), 0); 2584 2585 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA0), 0); 2586 WREG32(SOC15_REG_OFFSET(GC, 0, mmSPI_GDBG_TRAP_DATA1), 0); 2587 } 2588 2589 #define DEFAULT_SH_MEM_BASES (0x6000) 2590 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev) 2591 { 2592 int i; 2593 uint32_t sh_mem_config; 2594 uint32_t sh_mem_bases; 2595 2596 /* 2597 * Configure apertures: 2598 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2599 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2600 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2601 */ 2602 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16); 2603 2604 sh_mem_config = SH_MEM_ADDRESS_MODE_64 | 2605 SH_MEM_ALIGNMENT_MODE_UNALIGNED << 2606 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT; 2607 2608 mutex_lock(&adev->srbm_mutex); 2609 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2610 soc15_grbm_select(adev, 0, 0, 0, i, 0); 2611 /* CP and shaders */ 2612 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config); 2613 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases); 2614 } 2615 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 2616 mutex_unlock(&adev->srbm_mutex); 2617 2618 /* Initialize all compute VMIDs to have no GDS, GWS, or OA 2619 access. These should be enabled by FW for target VMIDs. */ 2620 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2621 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0); 2622 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0); 2623 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0); 2624 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0); 2625 } 2626 } 2627 2628 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev) 2629 { 2630 int vmid; 2631 2632 /* 2633 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2634 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2635 * the driver can enable them for graphics. VMID0 should maintain 2636 * access so that HWS firmware can save/restore entries. 2637 */ 2638 for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) { 2639 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0); 2640 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0); 2641 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0); 2642 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0); 2643 } 2644 } 2645 2646 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev) 2647 { 2648 uint32_t tmp; 2649 2650 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 2651 case IP_VERSION(9, 4, 1): 2652 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG); 2653 tmp = REG_SET_FIELD(tmp, SQ_CONFIG, DISABLE_BARRIER_WAITCNT, 2654 !READ_ONCE(adev->barrier_has_auto_waitcnt)); 2655 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp); 2656 break; 2657 case IP_VERSION(9, 4, 2): 2658 gfx_v9_4_2_init_sq(adev); 2659 break; 2660 default: 2661 break; 2662 } 2663 } 2664 2665 static void gfx_v9_0_constants_init(struct amdgpu_device *adev) 2666 { 2667 u32 tmp; 2668 int i; 2669 2670 if (!amdgpu_sriov_vf(adev) || 2671 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) { 2672 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2673 } 2674 2675 gfx_v9_0_tiling_mode_table_init(adev); 2676 2677 if (adev->gfx.num_gfx_rings) 2678 gfx_v9_0_setup_rb(adev); 2679 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info); 2680 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2); 2681 2682 /* XXX SH_MEM regs */ 2683 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2684 mutex_lock(&adev->srbm_mutex); 2685 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2686 soc15_grbm_select(adev, 0, 0, 0, i, 0); 2687 /* CP and shaders */ 2688 if (i == 0) { 2689 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2690 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2691 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2692 !!adev->gmc.noretry); 2693 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2694 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0); 2695 } else { 2696 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE, 2697 SH_MEM_ALIGNMENT_MODE_UNALIGNED); 2698 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE, 2699 !!adev->gmc.noretry); 2700 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp); 2701 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2702 (adev->gmc.private_aperture_start >> 48)); 2703 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2704 (adev->gmc.shared_aperture_start >> 48)); 2705 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp); 2706 } 2707 } 2708 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 2709 2710 mutex_unlock(&adev->srbm_mutex); 2711 2712 gfx_v9_0_init_compute_vmid(adev); 2713 gfx_v9_0_init_gds_vmid(adev); 2714 gfx_v9_0_init_sq_config(adev); 2715 } 2716 2717 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev) 2718 { 2719 u32 i, j, k; 2720 u32 mask; 2721 2722 mutex_lock(&adev->grbm_idx_mutex); 2723 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 2724 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 2725 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 2726 for (k = 0; k < adev->usec_timeout; k++) { 2727 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0) 2728 break; 2729 udelay(1); 2730 } 2731 if (k == adev->usec_timeout) { 2732 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 2733 0xffffffff, 0xffffffff, 0); 2734 mutex_unlock(&adev->grbm_idx_mutex); 2735 drm_info(adev_to_drm(adev), "Timeout wait for RLC serdes %u,%u\n", 2736 i, j); 2737 return; 2738 } 2739 } 2740 } 2741 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 2742 mutex_unlock(&adev->grbm_idx_mutex); 2743 2744 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK | 2745 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK | 2746 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK | 2747 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK; 2748 for (k = 0; k < adev->usec_timeout; k++) { 2749 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0) 2750 break; 2751 udelay(1); 2752 } 2753 } 2754 2755 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2756 bool enable) 2757 { 2758 u32 tmp; 2759 2760 /* These interrupts should be enabled to drive DS clock */ 2761 2762 tmp= RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0); 2763 2764 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0); 2765 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0); 2766 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0); 2767 if (adev->gfx.num_gfx_rings) 2768 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0); 2769 2770 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp); 2771 } 2772 2773 static void gfx_v9_0_init_csb(struct amdgpu_device *adev) 2774 { 2775 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2776 /* csib */ 2777 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI), 2778 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2779 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO), 2780 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2781 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH), 2782 adev->gfx.rlc.clear_state_size); 2783 } 2784 2785 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format, 2786 int indirect_offset, 2787 int list_size, 2788 int *unique_indirect_regs, 2789 int unique_indirect_reg_count, 2790 int *indirect_start_offsets, 2791 int *indirect_start_offsets_count, 2792 int max_start_offsets_count) 2793 { 2794 int idx; 2795 2796 for (; indirect_offset < list_size; indirect_offset++) { 2797 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count); 2798 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset; 2799 *indirect_start_offsets_count = *indirect_start_offsets_count + 1; 2800 2801 while (register_list_format[indirect_offset] != 0xFFFFFFFF) { 2802 indirect_offset += 2; 2803 2804 /* look for the matching indice */ 2805 for (idx = 0; idx < unique_indirect_reg_count; idx++) { 2806 if (unique_indirect_regs[idx] == 2807 register_list_format[indirect_offset] || 2808 !unique_indirect_regs[idx]) 2809 break; 2810 } 2811 2812 BUG_ON(idx >= unique_indirect_reg_count); 2813 2814 if (!unique_indirect_regs[idx]) 2815 unique_indirect_regs[idx] = register_list_format[indirect_offset]; 2816 2817 indirect_offset++; 2818 } 2819 } 2820 } 2821 2822 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev) 2823 { 2824 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2825 int unique_indirect_reg_count = 0; 2826 2827 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}; 2828 int indirect_start_offsets_count = 0; 2829 2830 int list_size = 0; 2831 int i = 0, j = 0; 2832 u32 tmp = 0; 2833 2834 u32 *register_list_format = 2835 kmemdup(adev->gfx.rlc.register_list_format, 2836 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL); 2837 if (!register_list_format) 2838 return -ENOMEM; 2839 2840 /* setup unique_indirect_regs array and indirect_start_offsets array */ 2841 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs); 2842 gfx_v9_1_parse_ind_reg_list(register_list_format, 2843 adev->gfx.rlc.reg_list_format_direct_reg_list_length, 2844 adev->gfx.rlc.reg_list_format_size_bytes >> 2, 2845 unique_indirect_regs, 2846 unique_indirect_reg_count, 2847 indirect_start_offsets, 2848 &indirect_start_offsets_count, 2849 ARRAY_SIZE(indirect_start_offsets)); 2850 2851 /* enable auto inc in case it is disabled */ 2852 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL)); 2853 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2854 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp); 2855 2856 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */ 2857 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR), 2858 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET); 2859 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++) 2860 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA), 2861 adev->gfx.rlc.register_restore[i]); 2862 2863 /* load indirect register */ 2864 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2865 adev->gfx.rlc.reg_list_format_start); 2866 2867 /* direct register portion */ 2868 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++) 2869 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2870 register_list_format[i]); 2871 2872 /* indirect register portion */ 2873 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) { 2874 if (register_list_format[i] == 0xFFFFFFFF) { 2875 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2876 continue; 2877 } 2878 2879 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2880 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]); 2881 2882 for (j = 0; j < unique_indirect_reg_count; j++) { 2883 if (register_list_format[i] == unique_indirect_regs[j]) { 2884 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j); 2885 break; 2886 } 2887 } 2888 2889 BUG_ON(j >= unique_indirect_reg_count); 2890 2891 i++; 2892 } 2893 2894 /* set save/restore list size */ 2895 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2; 2896 list_size = list_size >> 1; 2897 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2898 adev->gfx.rlc.reg_restore_list_size); 2899 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size); 2900 2901 /* write the starting offsets to RLC scratch ram */ 2902 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR), 2903 adev->gfx.rlc.starting_offsets_start); 2904 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++) 2905 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), 2906 indirect_start_offsets[i]); 2907 2908 /* load unique indirect regs*/ 2909 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) { 2910 if (unique_indirect_regs[i] != 0) { 2911 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0) 2912 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i], 2913 unique_indirect_regs[i] & 0x3FFFF); 2914 2915 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0) 2916 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i], 2917 unique_indirect_regs[i] >> 20); 2918 } 2919 } 2920 2921 kfree(register_list_format); 2922 return 0; 2923 } 2924 2925 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev) 2926 { 2927 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1); 2928 } 2929 2930 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev, 2931 bool enable) 2932 { 2933 uint32_t data = 0; 2934 uint32_t default_data = 0; 2935 2936 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS)); 2937 if (enable) { 2938 /* enable GFXIP control over CGPG */ 2939 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2940 if(default_data != data) 2941 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2942 2943 /* update status */ 2944 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK; 2945 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT); 2946 if(default_data != data) 2947 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2948 } else { 2949 /* restore GFXIP control over GCPG */ 2950 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK; 2951 if(default_data != data) 2952 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data); 2953 } 2954 } 2955 2956 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev) 2957 { 2958 uint32_t data = 0; 2959 2960 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 2961 AMD_PG_SUPPORT_GFX_SMG | 2962 AMD_PG_SUPPORT_GFX_DMG)) { 2963 /* init IDLE_POLL_COUNT = 60 */ 2964 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL)); 2965 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 2966 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 2967 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data); 2968 2969 /* init RLC PG Delay */ 2970 data = 0; 2971 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT); 2972 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT); 2973 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT); 2974 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT); 2975 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data); 2976 2977 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2)); 2978 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK; 2979 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT); 2980 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data); 2981 2982 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3)); 2983 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK; 2984 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT); 2985 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data); 2986 2987 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL)); 2988 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK; 2989 2990 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */ 2991 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT); 2992 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data); 2993 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 3, 0)) 2994 pwr_10_0_gfxip_control_over_cgpg(adev, true); 2995 } 2996 } 2997 2998 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev, 2999 bool enable) 3000 { 3001 uint32_t data = 0; 3002 uint32_t default_data = 0; 3003 3004 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3005 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3006 SMU_CLK_SLOWDOWN_ON_PU_ENABLE, 3007 enable ? 1 : 0); 3008 if (default_data != data) 3009 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3010 } 3011 3012 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev, 3013 bool enable) 3014 { 3015 uint32_t data = 0; 3016 uint32_t default_data = 0; 3017 3018 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3019 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3020 SMU_CLK_SLOWDOWN_ON_PD_ENABLE, 3021 enable ? 1 : 0); 3022 if(default_data != data) 3023 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3024 } 3025 3026 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev, 3027 bool enable) 3028 { 3029 uint32_t data = 0; 3030 uint32_t default_data = 0; 3031 3032 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3033 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3034 CP_PG_DISABLE, 3035 enable ? 0 : 1); 3036 if(default_data != data) 3037 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3038 } 3039 3040 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev, 3041 bool enable) 3042 { 3043 uint32_t data, default_data; 3044 3045 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3046 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3047 GFX_POWER_GATING_ENABLE, 3048 enable ? 1 : 0); 3049 if(default_data != data) 3050 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3051 } 3052 3053 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev, 3054 bool enable) 3055 { 3056 uint32_t data, default_data; 3057 3058 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3059 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3060 GFX_PIPELINE_PG_ENABLE, 3061 enable ? 1 : 0); 3062 if(default_data != data) 3063 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3064 3065 if (!enable) 3066 /* read any GFX register to wake up GFX */ 3067 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL)); 3068 } 3069 3070 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev, 3071 bool enable) 3072 { 3073 uint32_t data, default_data; 3074 3075 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3076 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3077 STATIC_PER_CU_PG_ENABLE, 3078 enable ? 1 : 0); 3079 if(default_data != data) 3080 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3081 } 3082 3083 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev, 3084 bool enable) 3085 { 3086 uint32_t data, default_data; 3087 3088 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL)); 3089 data = REG_SET_FIELD(data, RLC_PG_CNTL, 3090 DYN_PER_CU_PG_ENABLE, 3091 enable ? 1 : 0); 3092 if(default_data != data) 3093 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data); 3094 } 3095 3096 static void gfx_v9_0_init_pg(struct amdgpu_device *adev) 3097 { 3098 gfx_v9_0_init_csb(adev); 3099 3100 /* 3101 * Rlc save restore list is workable since v2_1. 3102 * And it's needed by gfxoff feature. 3103 */ 3104 if (adev->gfx.rlc.is_rlc_v2_1) { 3105 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 3106 IP_VERSION(9, 2, 1) || 3107 (adev->apu_flags & AMD_APU_IS_RAVEN2)) 3108 gfx_v9_1_init_rlc_save_restore_list(adev); 3109 gfx_v9_0_enable_save_restore_machine(adev); 3110 } 3111 3112 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG | 3113 AMD_PG_SUPPORT_GFX_SMG | 3114 AMD_PG_SUPPORT_GFX_DMG | 3115 AMD_PG_SUPPORT_CP | 3116 AMD_PG_SUPPORT_GDS | 3117 AMD_PG_SUPPORT_RLC_SMU_HS)) { 3118 WREG32_SOC15(GC, 0, mmRLC_JUMP_TABLE_RESTORE, 3119 adev->gfx.rlc.cp_table_gpu_addr >> 8); 3120 gfx_v9_0_init_gfx_power_gating(adev); 3121 } 3122 } 3123 3124 static void gfx_v9_0_rlc_stop(struct amdgpu_device *adev) 3125 { 3126 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0); 3127 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3128 gfx_v9_0_wait_for_rlc_serdes(adev); 3129 } 3130 3131 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev) 3132 { 3133 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 3134 udelay(50); 3135 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 3136 udelay(50); 3137 } 3138 3139 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev) 3140 { 3141 #ifdef AMDGPU_RLC_DEBUG_RETRY 3142 u32 rlc_ucode_ver; 3143 #endif 3144 3145 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 3146 udelay(50); 3147 3148 /* carrizo do enable cp interrupt after cp inited */ 3149 if (!(adev->flags & AMD_IS_APU)) { 3150 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3151 udelay(50); 3152 } 3153 3154 #ifdef AMDGPU_RLC_DEBUG_RETRY 3155 /* RLC_GPM_GENERAL_6 : RLC Ucode version */ 3156 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6); 3157 if(rlc_ucode_ver == 0x108) { 3158 drm_info(adev_to_drm(adev), "Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i\n", 3159 rlc_ucode_ver, adev->gfx.rlc_fw_version); 3160 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles, 3161 * default is 0x9C4 to create a 100us interval */ 3162 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4); 3163 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr 3164 * to disable the page fault retry interrupts, default is 3165 * 0x100 (256) */ 3166 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100); 3167 } 3168 #endif 3169 } 3170 3171 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev) 3172 { 3173 const struct rlc_firmware_header_v2_0 *hdr; 3174 const __le32 *fw_data; 3175 unsigned i, fw_size; 3176 3177 if (!adev->gfx.rlc_fw) 3178 return -EINVAL; 3179 3180 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 3181 amdgpu_ucode_print_rlc_hdr(&hdr->header); 3182 3183 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 3184 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 3185 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 3186 3187 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, 3188 RLCG_UCODE_LOADING_START_ADDRESS); 3189 for (i = 0; i < fw_size; i++) 3190 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++)); 3191 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 3192 3193 return 0; 3194 } 3195 3196 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev) 3197 { 3198 int r; 3199 3200 if (amdgpu_sriov_vf(adev)) { 3201 gfx_v9_0_init_csb(adev); 3202 return 0; 3203 } 3204 3205 adev->gfx.rlc.funcs->stop(adev); 3206 3207 /* disable CG */ 3208 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0); 3209 3210 gfx_v9_0_init_pg(adev); 3211 3212 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3213 /* legacy rlc firmware loading */ 3214 r = gfx_v9_0_rlc_load_microcode(adev); 3215 if (r) 3216 return r; 3217 } 3218 3219 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 3220 case IP_VERSION(9, 2, 2): 3221 case IP_VERSION(9, 1, 0): 3222 gfx_v9_0_init_lbpw(adev); 3223 if (amdgpu_lbpw == 0) 3224 gfx_v9_0_enable_lbpw(adev, false); 3225 else 3226 gfx_v9_0_enable_lbpw(adev, true); 3227 break; 3228 case IP_VERSION(9, 4, 0): 3229 gfx_v9_4_init_lbpw(adev); 3230 if (amdgpu_lbpw > 0) 3231 gfx_v9_0_enable_lbpw(adev, true); 3232 else 3233 gfx_v9_0_enable_lbpw(adev, false); 3234 break; 3235 default: 3236 break; 3237 } 3238 3239 gfx_v9_0_update_spm_vmid_internal(adev, 0xf); 3240 3241 adev->gfx.rlc.funcs->start(adev); 3242 3243 return 0; 3244 } 3245 3246 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3247 { 3248 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL); 3249 3250 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_INVALIDATE_ICACHE, enable ? 0 : 1); 3251 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_INVALIDATE_ICACHE, enable ? 0 : 1); 3252 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_INVALIDATE_ICACHE, enable ? 0 : 1); 3253 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE0_RESET, enable ? 0 : 1); 3254 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_PIPE1_RESET, enable ? 0 : 1); 3255 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, enable ? 0 : 1); 3256 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, enable ? 0 : 1); 3257 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, enable ? 0 : 1); 3258 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, enable ? 0 : 1); 3259 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3260 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3261 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1); 3262 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp); 3263 udelay(50); 3264 } 3265 3266 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3267 { 3268 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3269 const struct gfx_firmware_header_v1_0 *ce_hdr; 3270 const struct gfx_firmware_header_v1_0 *me_hdr; 3271 const __le32 *fw_data; 3272 unsigned i, fw_size; 3273 3274 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw) 3275 return -EINVAL; 3276 3277 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3278 adev->gfx.pfp_fw->data; 3279 ce_hdr = (const struct gfx_firmware_header_v1_0 *) 3280 adev->gfx.ce_fw->data; 3281 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3282 adev->gfx.me_fw->data; 3283 3284 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3285 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header); 3286 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3287 3288 gfx_v9_0_cp_gfx_enable(adev, false); 3289 3290 /* PFP */ 3291 fw_data = (const __le32 *) 3292 (adev->gfx.pfp_fw->data + 3293 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3294 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4; 3295 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0); 3296 for (i = 0; i < fw_size; i++) 3297 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++)); 3298 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3299 3300 /* CE */ 3301 fw_data = (const __le32 *) 3302 (adev->gfx.ce_fw->data + 3303 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes)); 3304 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4; 3305 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0); 3306 for (i = 0; i < fw_size; i++) 3307 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++)); 3308 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version); 3309 3310 /* ME */ 3311 fw_data = (const __le32 *) 3312 (adev->gfx.me_fw->data + 3313 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3314 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4; 3315 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0); 3316 for (i = 0; i < fw_size; i++) 3317 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++)); 3318 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version); 3319 3320 return 0; 3321 } 3322 3323 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev) 3324 { 3325 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0]; 3326 const struct cs_section_def *sect = NULL; 3327 const struct cs_extent_def *ext = NULL; 3328 int r, i, tmp; 3329 3330 /* init the CP */ 3331 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1); 3332 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1); 3333 3334 gfx_v9_0_cp_gfx_enable(adev, true); 3335 3336 /* Now only limit the quirk on the APU gfx9 series and already 3337 * confirmed that the APU gfx10/gfx11 needn't such update. 3338 */ 3339 if (adev->flags & AMD_IS_APU && 3340 adev->in_s3 && !pm_resume_via_firmware()) { 3341 drm_info(adev_to_drm(adev), "Will skip the CSB packet resubmit\n"); 3342 return 0; 3343 } 3344 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3); 3345 if (r) { 3346 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 3347 return r; 3348 } 3349 3350 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3351 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3352 3353 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3354 amdgpu_ring_write(ring, 0x80000000); 3355 amdgpu_ring_write(ring, 0x80000000); 3356 3357 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) { 3358 for (ext = sect->section; ext->extent != NULL; ++ext) { 3359 if (sect->id == SECT_CONTEXT) { 3360 amdgpu_ring_write(ring, 3361 PACKET3(PACKET3_SET_CONTEXT_REG, 3362 ext->reg_count)); 3363 amdgpu_ring_write(ring, 3364 ext->reg_index - PACKET3_SET_CONTEXT_REG_START); 3365 for (i = 0; i < ext->reg_count; i++) 3366 amdgpu_ring_write(ring, ext->extent[i]); 3367 } 3368 } 3369 } 3370 3371 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3372 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3373 3374 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3375 amdgpu_ring_write(ring, 0); 3376 3377 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2)); 3378 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE)); 3379 amdgpu_ring_write(ring, 0x8000); 3380 amdgpu_ring_write(ring, 0x8000); 3381 3382 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1)); 3383 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE | 3384 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START)); 3385 amdgpu_ring_write(ring, tmp); 3386 amdgpu_ring_write(ring, 0); 3387 3388 amdgpu_ring_commit(ring); 3389 3390 return 0; 3391 } 3392 3393 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev) 3394 { 3395 struct amdgpu_ring *ring; 3396 u32 tmp; 3397 u32 rb_bufsz; 3398 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3399 3400 /* Set the write pointer delay */ 3401 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0); 3402 3403 /* set the RB to use vmid 0 */ 3404 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0); 3405 3406 /* Set ring buffer size */ 3407 ring = &adev->gfx.gfx_ring[0]; 3408 rb_bufsz = order_base_2(ring->ring_size / 8); 3409 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3410 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3411 #ifdef __BIG_ENDIAN 3412 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1); 3413 #endif 3414 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3415 3416 /* Initialize the ring buffer's write pointers */ 3417 ring->wptr = 0; 3418 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3419 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3420 3421 /* set the wb address whether it's enabled or not */ 3422 rptr_addr = ring->rptr_gpu_addr; 3423 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3424 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3425 3426 wptr_gpu_addr = ring->wptr_gpu_addr; 3427 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr)); 3428 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr)); 3429 3430 mdelay(1); 3431 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp); 3432 3433 rb_addr = ring->gpu_addr >> 8; 3434 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr); 3435 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3436 3437 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL); 3438 if (ring->use_doorbell) { 3439 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3440 DOORBELL_OFFSET, ring->doorbell_index); 3441 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3442 DOORBELL_EN, 1); 3443 } else { 3444 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0); 3445 } 3446 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp); 3447 3448 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3449 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3450 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp); 3451 3452 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER, 3453 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3454 3455 3456 /* start the ring */ 3457 gfx_v9_0_cp_gfx_start(adev); 3458 3459 return 0; 3460 } 3461 3462 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3463 { 3464 if (enable) { 3465 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0); 3466 } else { 3467 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 3468 (CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK | 3469 CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK | 3470 CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK | 3471 CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK | 3472 CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK | 3473 CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK | 3474 CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK | 3475 CP_MEC_CNTL__MEC_ME1_HALT_MASK | 3476 CP_MEC_CNTL__MEC_ME2_HALT_MASK)); 3477 adev->gfx.kiq[0].ring.sched.ready = false; 3478 } 3479 udelay(50); 3480 } 3481 3482 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3483 { 3484 const struct gfx_firmware_header_v1_0 *mec_hdr; 3485 const __le32 *fw_data; 3486 unsigned i; 3487 u32 tmp; 3488 3489 if (!adev->gfx.mec_fw) 3490 return -EINVAL; 3491 3492 gfx_v9_0_cp_compute_enable(adev, false); 3493 3494 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3495 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3496 3497 fw_data = (const __le32 *) 3498 (adev->gfx.mec_fw->data + 3499 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3500 tmp = 0; 3501 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 3502 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 3503 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp); 3504 3505 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO, 3506 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000); 3507 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI, 3508 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 3509 3510 /* MEC1 */ 3511 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3512 mec_hdr->jt_offset); 3513 for (i = 0; i < mec_hdr->jt_size; i++) 3514 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA, 3515 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3516 3517 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR, 3518 adev->gfx.mec_fw_version); 3519 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */ 3520 3521 return 0; 3522 } 3523 3524 /* KIQ functions */ 3525 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring) 3526 { 3527 uint32_t tmp; 3528 struct amdgpu_device *adev = ring->adev; 3529 3530 /* tell RLC which is KIQ queue */ 3531 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS); 3532 tmp &= 0xffffff00; 3533 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 3534 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp | 0x80); 3535 } 3536 3537 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd) 3538 { 3539 struct amdgpu_device *adev = ring->adev; 3540 3541 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { 3542 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring)) { 3543 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH; 3544 mqd->cp_hqd_queue_priority = 3545 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM; 3546 } 3547 } 3548 } 3549 3550 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring) 3551 { 3552 struct amdgpu_device *adev = ring->adev; 3553 struct v9_mqd *mqd = ring->mqd_ptr; 3554 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 3555 uint32_t tmp; 3556 3557 mqd->header = 0xC0310800; 3558 mqd->compute_pipelinestat_enable = 0x00000001; 3559 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 3560 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 3561 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 3562 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 3563 mqd->compute_static_thread_mgmt_se4 = 0xffffffff; 3564 mqd->compute_static_thread_mgmt_se5 = 0xffffffff; 3565 mqd->compute_static_thread_mgmt_se6 = 0xffffffff; 3566 mqd->compute_static_thread_mgmt_se7 = 0xffffffff; 3567 mqd->compute_misc_reserved = 0x00000003; 3568 3569 mqd->dynamic_cu_mask_addr_lo = 3570 lower_32_bits(ring->mqd_gpu_addr 3571 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3572 mqd->dynamic_cu_mask_addr_hi = 3573 upper_32_bits(ring->mqd_gpu_addr 3574 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask)); 3575 3576 eop_base_addr = ring->eop_gpu_addr >> 8; 3577 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 3578 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 3579 3580 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3581 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL); 3582 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 3583 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1)); 3584 3585 mqd->cp_hqd_eop_control = tmp; 3586 3587 /* enable doorbell? */ 3588 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL); 3589 3590 if (ring->use_doorbell) { 3591 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3592 DOORBELL_OFFSET, ring->doorbell_index); 3593 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3594 DOORBELL_EN, 1); 3595 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3596 DOORBELL_SOURCE, 0); 3597 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3598 DOORBELL_HIT, 0); 3599 } else { 3600 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 3601 DOORBELL_EN, 0); 3602 } 3603 3604 mqd->cp_hqd_pq_doorbell_control = tmp; 3605 3606 /* disable the queue if it's active */ 3607 ring->wptr = 0; 3608 mqd->cp_hqd_dequeue_request = 0; 3609 mqd->cp_hqd_pq_rptr = 0; 3610 mqd->cp_hqd_pq_wptr_lo = 0; 3611 mqd->cp_hqd_pq_wptr_hi = 0; 3612 3613 /* set the pointer to the MQD */ 3614 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc; 3615 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr); 3616 3617 /* set MQD vmid to 0 */ 3618 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL); 3619 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 3620 mqd->cp_mqd_control = tmp; 3621 3622 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3623 hqd_gpu_addr = ring->gpu_addr >> 8; 3624 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 3625 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 3626 3627 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3628 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL); 3629 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 3630 (order_base_2(ring->ring_size / 4) - 1)); 3631 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 3632 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 3633 #ifdef __BIG_ENDIAN 3634 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1); 3635 #endif 3636 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 3637 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0); 3638 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 3639 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 3640 mqd->cp_hqd_pq_control = tmp; 3641 3642 /* set the wb address whether it's enabled or not */ 3643 wb_gpu_addr = ring->rptr_gpu_addr; 3644 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 3645 mqd->cp_hqd_pq_rptr_report_addr_hi = 3646 upper_32_bits(wb_gpu_addr) & 0xffff; 3647 3648 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3649 wb_gpu_addr = ring->wptr_gpu_addr; 3650 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 3651 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 3652 3653 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3654 ring->wptr = 0; 3655 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR); 3656 3657 /* set the vmid for the queue */ 3658 mqd->cp_hqd_vmid = 0; 3659 3660 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE); 3661 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53); 3662 mqd->cp_hqd_persistent_state = tmp; 3663 3664 /* set MIN_IB_AVAIL_SIZE */ 3665 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL); 3666 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 3667 mqd->cp_hqd_ib_control = tmp; 3668 3669 /* set static priority for a queue/ring */ 3670 gfx_v9_0_mqd_set_priority(ring, mqd); 3671 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM); 3672 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1); 3673 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 3674 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); 3675 mqd->cp_hqd_quantum = tmp; 3676 3677 /* map_queues packet doesn't need activate the queue, 3678 * so only kiq need set this field. 3679 */ 3680 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 3681 mqd->cp_hqd_active = 1; 3682 3683 return 0; 3684 } 3685 3686 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring) 3687 { 3688 struct amdgpu_device *adev = ring->adev; 3689 struct v9_mqd *mqd = ring->mqd_ptr; 3690 int j; 3691 3692 /* disable wptr polling */ 3693 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 3694 3695 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR, 3696 mqd->cp_hqd_eop_base_addr_lo); 3697 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI, 3698 mqd->cp_hqd_eop_base_addr_hi); 3699 3700 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 3701 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL, 3702 mqd->cp_hqd_eop_control); 3703 3704 /* enable doorbell? */ 3705 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3706 mqd->cp_hqd_pq_doorbell_control); 3707 3708 /* disable the queue if it's active */ 3709 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3710 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3711 for (j = 0; j < adev->usec_timeout; j++) { 3712 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3713 break; 3714 udelay(1); 3715 } 3716 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3717 mqd->cp_hqd_dequeue_request); 3718 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 3719 mqd->cp_hqd_pq_rptr); 3720 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3721 mqd->cp_hqd_pq_wptr_lo); 3722 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3723 mqd->cp_hqd_pq_wptr_hi); 3724 } 3725 3726 /* set the pointer to the MQD */ 3727 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR, 3728 mqd->cp_mqd_base_addr_lo); 3729 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI, 3730 mqd->cp_mqd_base_addr_hi); 3731 3732 /* set MQD vmid to 0 */ 3733 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL, 3734 mqd->cp_mqd_control); 3735 3736 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 3737 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE, 3738 mqd->cp_hqd_pq_base_lo); 3739 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI, 3740 mqd->cp_hqd_pq_base_hi); 3741 3742 /* set up the HQD, this is similar to CP_RB0_CNTL */ 3743 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL, 3744 mqd->cp_hqd_pq_control); 3745 3746 /* set the wb address whether it's enabled or not */ 3747 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR, 3748 mqd->cp_hqd_pq_rptr_report_addr_lo); 3749 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 3750 mqd->cp_hqd_pq_rptr_report_addr_hi); 3751 3752 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 3753 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR, 3754 mqd->cp_hqd_pq_wptr_poll_addr_lo); 3755 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI, 3756 mqd->cp_hqd_pq_wptr_poll_addr_hi); 3757 3758 /* enable the doorbell if requested */ 3759 if (ring->use_doorbell) { 3760 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER, 3761 (adev->doorbell_index.kiq * 2) << 2); 3762 /* If GC has entered CGPG, ringing doorbell > first page 3763 * doesn't wakeup GC. Enlarge CP_MEC_DOORBELL_RANGE_UPPER to 3764 * workaround this issue. And this change has to align with firmware 3765 * update. 3766 */ 3767 if (check_if_enlarge_doorbell_range(adev)) 3768 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3769 (adev->doorbell.size - 4)); 3770 else 3771 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER, 3772 (adev->doorbell_index.userqueue_end * 2) << 2); 3773 } 3774 3775 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 3776 mqd->cp_hqd_pq_doorbell_control); 3777 3778 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 3779 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 3780 mqd->cp_hqd_pq_wptr_lo); 3781 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 3782 mqd->cp_hqd_pq_wptr_hi); 3783 3784 /* set the vmid for the queue */ 3785 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid); 3786 3787 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 3788 mqd->cp_hqd_persistent_state); 3789 3790 /* activate the queue */ 3791 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 3792 mqd->cp_hqd_active); 3793 3794 if (ring->use_doorbell) 3795 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 3796 3797 return 0; 3798 } 3799 3800 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring) 3801 { 3802 struct amdgpu_device *adev = ring->adev; 3803 int j; 3804 3805 /* disable the queue if it's active */ 3806 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) { 3807 3808 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 3809 3810 for (j = 0; j < adev->usec_timeout; j++) { 3811 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 3812 break; 3813 udelay(1); 3814 } 3815 3816 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 3817 DRM_DEBUG("KIQ dequeue request failed.\n"); 3818 3819 /* Manual disable if dequeue request times out */ 3820 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0); 3821 } 3822 3823 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 3824 0); 3825 } 3826 3827 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0); 3828 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0); 3829 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0); 3830 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000); 3831 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0); 3832 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0); 3833 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0); 3834 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0); 3835 3836 return 0; 3837 } 3838 3839 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring) 3840 { 3841 struct amdgpu_device *adev = ring->adev; 3842 struct v9_mqd *mqd = ring->mqd_ptr; 3843 struct v9_mqd *tmp_mqd; 3844 3845 gfx_v9_0_kiq_setting(ring); 3846 3847 /* GPU could be in bad state during probe, driver trigger the reset 3848 * after load the SMU, in this case , the mqd is not be initialized. 3849 * driver need to re-init the mqd. 3850 * check mqd->cp_hqd_pq_control since this value should not be 0 3851 */ 3852 tmp_mqd = (struct v9_mqd *)adev->gfx.kiq[0].mqd_backup; 3853 if (amdgpu_in_reset(adev) && tmp_mqd->cp_hqd_pq_control){ 3854 /* for GPU_RESET case , reset MQD to a clean status */ 3855 if (adev->gfx.kiq[0].mqd_backup) 3856 memcpy(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(struct v9_mqd_allocation)); 3857 3858 /* reset ring buffer */ 3859 ring->wptr = 0; 3860 amdgpu_ring_clear_ring(ring); 3861 3862 mutex_lock(&adev->srbm_mutex); 3863 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3864 gfx_v9_0_kiq_init_register(ring); 3865 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3866 mutex_unlock(&adev->srbm_mutex); 3867 } else { 3868 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3869 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3870 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3871 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 3872 amdgpu_ring_clear_ring(ring); 3873 mutex_lock(&adev->srbm_mutex); 3874 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3875 gfx_v9_0_mqd_init(ring); 3876 gfx_v9_0_kiq_init_register(ring); 3877 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3878 mutex_unlock(&adev->srbm_mutex); 3879 3880 if (adev->gfx.kiq[0].mqd_backup) 3881 memcpy(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(struct v9_mqd_allocation)); 3882 } 3883 3884 return 0; 3885 } 3886 3887 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring, bool restore) 3888 { 3889 struct amdgpu_device *adev = ring->adev; 3890 struct v9_mqd *mqd = ring->mqd_ptr; 3891 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 3892 struct v9_mqd *tmp_mqd; 3893 3894 /* Same as above kiq init, driver need to re-init the mqd if mqd->cp_hqd_pq_control 3895 * is not be initialized before 3896 */ 3897 tmp_mqd = (struct v9_mqd *)adev->gfx.mec.mqd_backup[mqd_idx]; 3898 3899 if (!restore && (!tmp_mqd->cp_hqd_pq_control || 3900 (!amdgpu_in_reset(adev) && !adev->in_suspend))) { 3901 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation)); 3902 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF; 3903 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF; 3904 mutex_lock(&adev->srbm_mutex); 3905 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 3906 gfx_v9_0_mqd_init(ring); 3907 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 3908 mutex_unlock(&adev->srbm_mutex); 3909 3910 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3911 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation)); 3912 } else { 3913 /* restore MQD to a clean status */ 3914 if (adev->gfx.mec.mqd_backup[mqd_idx]) 3915 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation)); 3916 /* reset ring buffer */ 3917 ring->wptr = 0; 3918 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 3919 amdgpu_ring_clear_ring(ring); 3920 } 3921 3922 return 0; 3923 } 3924 3925 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev) 3926 { 3927 gfx_v9_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 3928 return 0; 3929 } 3930 3931 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev) 3932 { 3933 int i, r; 3934 3935 gfx_v9_0_cp_compute_enable(adev, true); 3936 3937 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3938 r = gfx_v9_0_kcq_init_queue(&adev->gfx.compute_ring[i], false); 3939 if (r) 3940 return r; 3941 } 3942 3943 return amdgpu_gfx_enable_kcq(adev, 0); 3944 } 3945 3946 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev) 3947 { 3948 int r, i; 3949 struct amdgpu_ring *ring; 3950 3951 if (!(adev->flags & AMD_IS_APU)) 3952 gfx_v9_0_enable_gui_idle_interrupt(adev, false); 3953 3954 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { 3955 if (adev->gfx.num_gfx_rings) { 3956 /* legacy firmware loading */ 3957 r = gfx_v9_0_cp_gfx_load_microcode(adev); 3958 if (r) 3959 return r; 3960 } 3961 3962 r = gfx_v9_0_cp_compute_load_microcode(adev); 3963 if (r) 3964 return r; 3965 } 3966 3967 if (adev->gfx.num_gfx_rings) 3968 gfx_v9_0_cp_gfx_enable(adev, false); 3969 gfx_v9_0_cp_compute_enable(adev, false); 3970 3971 r = gfx_v9_0_kiq_resume(adev); 3972 if (r) 3973 return r; 3974 3975 if (adev->gfx.num_gfx_rings) { 3976 r = gfx_v9_0_cp_gfx_resume(adev); 3977 if (r) 3978 return r; 3979 } 3980 3981 r = gfx_v9_0_kcq_resume(adev); 3982 if (r) 3983 return r; 3984 3985 if (adev->gfx.num_gfx_rings) { 3986 ring = &adev->gfx.gfx_ring[0]; 3987 r = amdgpu_ring_test_helper(ring); 3988 if (r) 3989 return r; 3990 } 3991 3992 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 3993 ring = &adev->gfx.compute_ring[i]; 3994 amdgpu_ring_test_helper(ring); 3995 } 3996 3997 gfx_v9_0_enable_gui_idle_interrupt(adev, true); 3998 3999 return 0; 4000 } 4001 4002 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) 4003 { 4004 u32 tmp; 4005 4006 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1) && 4007 amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 2)) 4008 return; 4009 4010 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG); 4011 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH, 4012 adev->df.hash_status.hash_64k); 4013 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH, 4014 adev->df.hash_status.hash_2m); 4015 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH, 4016 adev->df.hash_status.hash_1g); 4017 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp); 4018 } 4019 4020 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable) 4021 { 4022 if (adev->gfx.num_gfx_rings) 4023 gfx_v9_0_cp_gfx_enable(adev, enable); 4024 gfx_v9_0_cp_compute_enable(adev, enable); 4025 } 4026 4027 static int gfx_v9_0_hw_init(struct amdgpu_ip_block *ip_block) 4028 { 4029 int r; 4030 struct amdgpu_device *adev = ip_block->adev; 4031 4032 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, 4033 adev->gfx.cleaner_shader_ptr); 4034 4035 if (!amdgpu_sriov_vf(adev)) 4036 gfx_v9_0_init_golden_registers(adev); 4037 4038 gfx_v9_0_constants_init(adev); 4039 4040 gfx_v9_0_init_tcp_config(adev); 4041 4042 r = adev->gfx.rlc.funcs->resume(adev); 4043 if (r) 4044 return r; 4045 4046 r = gfx_v9_0_cp_resume(adev); 4047 if (r) 4048 return r; 4049 4050 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) && 4051 !amdgpu_sriov_vf(adev)) 4052 gfx_v9_4_2_set_power_brake_sequence(adev); 4053 4054 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4055 if (r) 4056 return r; 4057 4058 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4059 if (r) 4060 goto err_priv_inst; 4061 4062 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); 4063 if (r) 4064 goto err_bad_op; 4065 4066 return 0; 4067 4068 err_bad_op: 4069 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4070 err_priv_inst: 4071 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4072 return r; 4073 } 4074 4075 static void gfx_v9_0_deactivate_kcq_hqd(struct amdgpu_device *adev) 4076 { 4077 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4078 for (int i = 0; i < adev->gfx.num_compute_rings; i++) { 4079 u32 tmp; 4080 struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; 4081 4082 mutex_lock(&adev->srbm_mutex); 4083 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 4084 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); 4085 /* disable the queue if it's active */ 4086 if (tmp & CP_HQD_ACTIVE__ACTIVE_MASK) { 4087 int j; 4088 4089 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1); 4090 for (j = 0; j < adev->usec_timeout; j++) { 4091 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE); 4092 if (!(tmp & CP_HQD_ACTIVE__ACTIVE_MASK)) 4093 break; 4094 udelay(1); 4095 } 4096 if (j == AMDGPU_MAX_USEC_TIMEOUT) { 4097 DRM_DEBUG("comp_%u_%u_%u dequeue request failed.\n", 4098 ring->me, ring->pipe, ring->queue); 4099 /* Manual disable if dequeue request times out */ 4100 WREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE, 0); 4101 } 4102 WREG32_SOC15(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 0); 4103 } 4104 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 4105 mutex_unlock(&adev->srbm_mutex); 4106 } 4107 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4108 } 4109 4110 static int gfx_v9_0_hw_fini(struct amdgpu_ip_block *ip_block) 4111 { 4112 struct amdgpu_device *adev = ip_block->adev; 4113 4114 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 4115 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 4116 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 4117 4118 /* DF freeze and kcq disable will fail */ 4119 if (!amdgpu_ras_intr_triggered()) 4120 /* disable KCQ to avoid CPC touch memory not valid anymore */ 4121 amdgpu_gfx_disable_kcq(adev, 0); 4122 4123 if (amdgpu_sriov_vf(adev)) { 4124 gfx_v9_0_cp_gfx_enable(adev, false); 4125 /* must disable polling for SRIOV when hw finished, otherwise 4126 * CPC engine may still keep fetching WB address which is already 4127 * invalid after sw finished and trigger DMAR reading error in 4128 * hypervisor side. 4129 */ 4130 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4131 return 0; 4132 } 4133 4134 if ((adev->flags & AMD_IS_APU) && amdgpu_in_reset(adev) && 4135 amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_MODE2) 4136 gfx_v9_0_deactivate_kcq_hqd(adev); 4137 4138 /* Use deinitialize sequence from CAIL when unbinding device from driver, 4139 * otherwise KIQ is hanging when binding back 4140 */ 4141 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4142 mutex_lock(&adev->srbm_mutex); 4143 soc15_grbm_select(adev, adev->gfx.kiq[0].ring.me, 4144 adev->gfx.kiq[0].ring.pipe, 4145 adev->gfx.kiq[0].ring.queue, 0, 0); 4146 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq[0].ring); 4147 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 4148 mutex_unlock(&adev->srbm_mutex); 4149 } 4150 4151 gfx_v9_0_cp_enable(adev, false); 4152 4153 /* Skip stopping RLC with A+A reset or when RLC controls GFX clock */ 4154 if ((adev->gmc.xgmi.connected_to_cpu && amdgpu_in_reset(adev)) || 4155 (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 4, 2))) { 4156 dev_dbg(adev->dev, "Skipping RLC halt\n"); 4157 return 0; 4158 } 4159 4160 adev->gfx.rlc.funcs->stop(adev); 4161 return 0; 4162 } 4163 4164 static int gfx_v9_0_suspend(struct amdgpu_ip_block *ip_block) 4165 { 4166 return gfx_v9_0_hw_fini(ip_block); 4167 } 4168 4169 static int gfx_v9_0_resume(struct amdgpu_ip_block *ip_block) 4170 { 4171 return gfx_v9_0_hw_init(ip_block); 4172 } 4173 4174 static bool gfx_v9_0_is_idle(struct amdgpu_ip_block *ip_block) 4175 { 4176 struct amdgpu_device *adev = ip_block->adev; 4177 4178 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS), 4179 GRBM_STATUS, GUI_ACTIVE)) 4180 return false; 4181 else 4182 return true; 4183 } 4184 4185 static int gfx_v9_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 4186 { 4187 unsigned i; 4188 struct amdgpu_device *adev = ip_block->adev; 4189 4190 for (i = 0; i < adev->usec_timeout; i++) { 4191 if (gfx_v9_0_is_idle(ip_block)) 4192 return 0; 4193 udelay(1); 4194 } 4195 return -ETIMEDOUT; 4196 } 4197 4198 static int gfx_v9_0_soft_reset(struct amdgpu_ip_block *ip_block) 4199 { 4200 u32 grbm_soft_reset = 0; 4201 u32 tmp; 4202 struct amdgpu_device *adev = ip_block->adev; 4203 4204 /* GRBM_STATUS */ 4205 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS); 4206 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | 4207 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | 4208 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | 4209 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | 4210 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | 4211 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) { 4212 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4213 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4214 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4215 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); 4216 } 4217 4218 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { 4219 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4220 GRBM_SOFT_RESET, SOFT_RESET_CP, 1); 4221 } 4222 4223 /* GRBM_STATUS2 */ 4224 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2); 4225 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) 4226 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, 4227 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 4228 4229 4230 if (grbm_soft_reset) { 4231 /* stop the rlc */ 4232 adev->gfx.rlc.funcs->stop(adev); 4233 4234 if (adev->gfx.num_gfx_rings) 4235 /* Disable GFX parsing/prefetching */ 4236 gfx_v9_0_cp_gfx_enable(adev, false); 4237 4238 /* Disable MEC parsing/prefetching */ 4239 gfx_v9_0_cp_compute_enable(adev, false); 4240 4241 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4242 tmp |= grbm_soft_reset; 4243 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp); 4244 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4245 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4246 4247 udelay(50); 4248 4249 tmp &= ~grbm_soft_reset; 4250 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp); 4251 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET); 4252 4253 /* Wait a little for things to settle down */ 4254 udelay(50); 4255 } 4256 return 0; 4257 } 4258 4259 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev) 4260 { 4261 signed long r, cnt = 0; 4262 unsigned long flags; 4263 uint32_t seq, reg_val_offs = 0; 4264 uint64_t value = 0; 4265 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 4266 struct amdgpu_ring *ring = &kiq->ring; 4267 4268 BUG_ON(!ring->funcs->emit_rreg); 4269 4270 spin_lock_irqsave(&kiq->ring_lock, flags); 4271 if (amdgpu_wb_get(adev, ®_val_offs)) { 4272 pr_err("critical bug! too many kiq readers\n"); 4273 goto failed_unlock; 4274 } 4275 amdgpu_ring_alloc(ring, 32); 4276 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 4277 amdgpu_ring_write(ring, 9 | /* src: register*/ 4278 (5 << 8) | /* dst: memory */ 4279 (1 << 16) | /* count sel */ 4280 (1 << 20)); /* write confirm */ 4281 amdgpu_ring_write(ring, 0); 4282 amdgpu_ring_write(ring, 0); 4283 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 4284 reg_val_offs * 4)); 4285 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 4286 reg_val_offs * 4)); 4287 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 4288 if (r) 4289 goto failed_undo; 4290 4291 amdgpu_ring_commit(ring); 4292 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4293 4294 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4295 4296 /* don't wait anymore for gpu reset case because this way may 4297 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg 4298 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will 4299 * never return if we keep waiting in virt_kiq_rreg, which cause 4300 * gpu_recover() hang there. 4301 * 4302 * also don't wait anymore for IRQ context 4303 * */ 4304 if (r < 1 && (amdgpu_in_reset(adev))) 4305 goto failed_kiq_read; 4306 4307 might_sleep(); 4308 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 4309 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 4310 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 4311 } 4312 4313 if (cnt > MAX_KIQ_REG_TRY) 4314 goto failed_kiq_read; 4315 4316 mb(); 4317 value = (uint64_t)adev->wb.wb[reg_val_offs] | 4318 (uint64_t)adev->wb.wb[reg_val_offs + 1 ] << 32ULL; 4319 amdgpu_wb_free(adev, reg_val_offs); 4320 return value; 4321 4322 failed_undo: 4323 amdgpu_ring_undo(ring); 4324 failed_unlock: 4325 spin_unlock_irqrestore(&kiq->ring_lock, flags); 4326 failed_kiq_read: 4327 if (reg_val_offs) 4328 amdgpu_wb_free(adev, reg_val_offs); 4329 pr_err("failed to read gpu clock\n"); 4330 return ~0; 4331 } 4332 4333 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev) 4334 { 4335 uint64_t clock, clock_lo, clock_hi, hi_check; 4336 4337 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 4338 case IP_VERSION(9, 3, 0): 4339 preempt_disable(); 4340 clock_hi = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 4341 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 4342 hi_check = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_UPPER_Renoir); 4343 /* The SMUIO TSC clock frequency is 100MHz, which sets 32-bit carry over 4344 * roughly every 42 seconds. 4345 */ 4346 if (hi_check != clock_hi) { 4347 clock_lo = RREG32_SOC15_NO_KIQ(SMUIO, 0, mmGOLDEN_TSC_COUNT_LOWER_Renoir); 4348 clock_hi = hi_check; 4349 } 4350 preempt_enable(); 4351 clock = clock_lo | (clock_hi << 32ULL); 4352 break; 4353 default: 4354 amdgpu_gfx_off_ctrl(adev, false); 4355 mutex_lock(&adev->gfx.gpu_clock_mutex); 4356 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 4357 IP_VERSION(9, 0, 1) && 4358 amdgpu_sriov_runtime(adev)) { 4359 clock = gfx_v9_0_kiq_read_clock(adev); 4360 } else { 4361 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1); 4362 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) | 4363 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL); 4364 } 4365 mutex_unlock(&adev->gfx.gpu_clock_mutex); 4366 amdgpu_gfx_off_ctrl(adev, true); 4367 break; 4368 } 4369 return clock; 4370 } 4371 4372 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 4373 uint32_t vmid, 4374 uint32_t gds_base, uint32_t gds_size, 4375 uint32_t gws_base, uint32_t gws_size, 4376 uint32_t oa_base, uint32_t oa_size) 4377 { 4378 struct amdgpu_device *adev = ring->adev; 4379 4380 /* GDS Base */ 4381 gfx_v9_0_write_data_to_reg(ring, 0, false, 4382 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid, 4383 gds_base); 4384 4385 /* GDS Size */ 4386 gfx_v9_0_write_data_to_reg(ring, 0, false, 4387 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid, 4388 gds_size); 4389 4390 /* GWS */ 4391 gfx_v9_0_write_data_to_reg(ring, 0, false, 4392 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid, 4393 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 4394 4395 /* OA */ 4396 gfx_v9_0_write_data_to_reg(ring, 0, false, 4397 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid, 4398 (1 << (oa_size + oa_base)) - (1 << oa_base)); 4399 } 4400 4401 static const u32 vgpr_init_compute_shader[] = 4402 { 4403 0xb07c0000, 0xbe8000ff, 4404 0x000000f8, 0xbf110800, 4405 0x7e000280, 0x7e020280, 4406 0x7e040280, 0x7e060280, 4407 0x7e080280, 0x7e0a0280, 4408 0x7e0c0280, 0x7e0e0280, 4409 0x80808800, 0xbe803200, 4410 0xbf84fff5, 0xbf9c0000, 4411 0xd28c0001, 0x0001007f, 4412 0xd28d0001, 0x0002027e, 4413 0x10020288, 0xb8810904, 4414 0xb7814000, 0xd1196a01, 4415 0x00000301, 0xbe800087, 4416 0xbefc00c1, 0xd89c4000, 4417 0x00020201, 0xd89cc080, 4418 0x00040401, 0x320202ff, 4419 0x00000800, 0x80808100, 4420 0xbf84fff8, 0x7e020280, 4421 0xbf810000, 0x00000000, 4422 }; 4423 4424 static const u32 sgpr_init_compute_shader[] = 4425 { 4426 0xb07c0000, 0xbe8000ff, 4427 0x0000005f, 0xbee50080, 4428 0xbe812c65, 0xbe822c65, 4429 0xbe832c65, 0xbe842c65, 4430 0xbe852c65, 0xb77c0005, 4431 0x80808500, 0xbf84fff8, 4432 0xbe800080, 0xbf810000, 4433 }; 4434 4435 static const u32 vgpr_init_compute_shader_arcturus[] = { 4436 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 4437 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 4438 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 4439 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 4440 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 4441 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 4442 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 4443 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 4444 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 4445 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 4446 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 4447 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 4448 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 4449 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 4450 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 4451 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 4452 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 4453 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 4454 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 4455 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 4456 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 4457 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 4458 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 4459 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 4460 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 4461 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 4462 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 4463 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 4464 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 4465 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 4466 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 4467 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 4468 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 4469 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 4470 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 4471 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 4472 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 4473 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 4474 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 4475 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 4476 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 4477 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 4478 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 4479 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 4480 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 4481 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 4482 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 4483 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 4484 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 4485 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 4486 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 4487 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 4488 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 4489 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 4490 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 4491 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 4492 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 4493 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 4494 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 4495 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 4496 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 4497 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 4498 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 4499 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 4500 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 4501 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 4502 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 4503 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 4504 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 4505 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 4506 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 4507 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 4508 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 4509 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 4510 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 4511 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 4512 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 4513 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 4514 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 4515 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 4516 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 4517 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 4518 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 4519 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 4520 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 4521 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 4522 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 4523 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 4524 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904, 4525 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 4526 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 4527 0xbf84fff8, 0xbf810000, 4528 }; 4529 4530 /* When below register arrays changed, please update gpr_reg_size, 4531 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds, 4532 to cover all gfx9 ASICs */ 4533 static const struct soc15_reg_entry vgpr_init_regs[] = { 4534 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4535 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4536 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4537 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4538 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f }, 4539 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4540 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4541 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4542 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4543 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4544 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4545 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4546 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4547 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4548 }; 4549 4550 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = { 4551 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4552 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4553 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 }, 4554 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4555 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf }, 4556 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */ 4557 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff }, 4558 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff }, 4559 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff }, 4560 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff }, 4561 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff }, 4562 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff }, 4563 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff }, 4564 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff }, 4565 }; 4566 4567 static const struct soc15_reg_entry sgpr1_init_regs[] = { 4568 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4569 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4570 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4571 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4572 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4573 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4574 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff }, 4575 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff }, 4576 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff }, 4577 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff }, 4578 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff }, 4579 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff }, 4580 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff }, 4581 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff }, 4582 }; 4583 4584 static const struct soc15_reg_entry sgpr2_init_regs[] = { 4585 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 }, 4586 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 }, 4587 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 }, 4588 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 }, 4589 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */ 4590 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 }, 4591 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 }, 4592 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 }, 4593 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 }, 4594 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 }, 4595 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 }, 4596 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 }, 4597 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 }, 4598 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 }, 4599 }; 4600 4601 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = { 4602 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1}, 4603 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1}, 4604 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1}, 4605 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1}, 4606 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1}, 4607 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1}, 4608 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1}, 4609 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1}, 4610 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1}, 4611 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1}, 4612 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1}, 4613 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1}, 4614 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1}, 4615 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6}, 4616 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16}, 4617 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16}, 4618 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16}, 4619 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16}, 4620 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16}, 4621 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16}, 4622 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16}, 4623 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16}, 4624 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6}, 4625 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16}, 4626 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16}, 4627 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1}, 4628 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1}, 4629 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32}, 4630 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32}, 4631 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72}, 4632 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16}, 4633 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2}, 4634 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6}, 4635 }; 4636 4637 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev) 4638 { 4639 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4640 int i, r; 4641 4642 /* only support when RAS is enabled */ 4643 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4644 return 0; 4645 4646 r = amdgpu_ring_alloc(ring, 7); 4647 if (r) { 4648 drm_err(adev_to_drm(adev), "GDS workarounds failed to lock ring %s (%d).\n", 4649 ring->name, r); 4650 return r; 4651 } 4652 4653 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000); 4654 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size); 4655 4656 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5)); 4657 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC | 4658 PACKET3_DMA_DATA_DST_SEL(1) | 4659 PACKET3_DMA_DATA_SRC_SEL(2) | 4660 PACKET3_DMA_DATA_ENGINE(0))); 4661 amdgpu_ring_write(ring, 0); 4662 amdgpu_ring_write(ring, 0); 4663 amdgpu_ring_write(ring, 0); 4664 amdgpu_ring_write(ring, 0); 4665 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT | 4666 adev->gds.gds_size); 4667 4668 amdgpu_ring_commit(ring); 4669 4670 for (i = 0; i < adev->usec_timeout; i++) { 4671 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring)) 4672 break; 4673 udelay(1); 4674 } 4675 4676 if (i >= adev->usec_timeout) 4677 r = -ETIMEDOUT; 4678 4679 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000); 4680 4681 return r; 4682 } 4683 4684 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) 4685 { 4686 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0]; 4687 struct amdgpu_ib ib; 4688 struct dma_fence *f = NULL; 4689 int r, i; 4690 unsigned total_size, vgpr_offset, sgpr_offset; 4691 u64 gpu_addr; 4692 4693 int compute_dim_x = adev->gfx.config.max_shader_engines * 4694 adev->gfx.config.max_cu_per_sh * 4695 adev->gfx.config.max_sh_per_se; 4696 int sgpr_work_group_size = 5; 4697 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6; 4698 int vgpr_init_shader_size; 4699 const u32 *vgpr_init_shader_ptr; 4700 const struct soc15_reg_entry *vgpr_init_regs_ptr; 4701 4702 /* only support when RAS is enabled */ 4703 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 4704 return 0; 4705 4706 /* bail if the compute ring is not ready */ 4707 if (!ring->sched.ready) 4708 return 0; 4709 4710 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) { 4711 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus; 4712 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus); 4713 vgpr_init_regs_ptr = vgpr_init_regs_arcturus; 4714 } else { 4715 vgpr_init_shader_ptr = vgpr_init_compute_shader; 4716 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader); 4717 vgpr_init_regs_ptr = vgpr_init_regs; 4718 } 4719 4720 total_size = 4721 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */ 4722 total_size += 4723 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */ 4724 total_size += 4725 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */ 4726 total_size = ALIGN(total_size, 256); 4727 vgpr_offset = total_size; 4728 total_size += ALIGN(vgpr_init_shader_size, 256); 4729 sgpr_offset = total_size; 4730 total_size += sizeof(sgpr_init_compute_shader); 4731 4732 /* allocate an indirect buffer to put the commands in */ 4733 memset(&ib, 0, sizeof(ib)); 4734 r = amdgpu_ib_get(adev, NULL, total_size, 4735 AMDGPU_IB_POOL_DIRECT, &ib); 4736 if (r) { 4737 drm_err(adev_to_drm(adev), "failed to get ib (%d).\n", r); 4738 return r; 4739 } 4740 4741 /* load the compute shaders */ 4742 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++) 4743 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i]; 4744 4745 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++) 4746 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i]; 4747 4748 /* init the ib length to 0 */ 4749 ib.length_dw = 0; 4750 4751 /* VGPR */ 4752 /* write the register state for the compute dispatch */ 4753 for (i = 0; i < gpr_reg_size; i++) { 4754 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4755 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i]) 4756 - PACKET3_SET_SH_REG_START; 4757 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value; 4758 } 4759 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4760 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8; 4761 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4762 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4763 - PACKET3_SET_SH_REG_START; 4764 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4765 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4766 4767 /* write dispatch packet */ 4768 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4769 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */ 4770 ib.ptr[ib.length_dw++] = 1; /* y */ 4771 ib.ptr[ib.length_dw++] = 1; /* z */ 4772 ib.ptr[ib.length_dw++] = 4773 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4774 4775 /* write CS partial flush packet */ 4776 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4777 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4778 4779 /* SGPR1 */ 4780 /* write the register state for the compute dispatch */ 4781 for (i = 0; i < gpr_reg_size; i++) { 4782 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4783 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i]) 4784 - PACKET3_SET_SH_REG_START; 4785 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value; 4786 } 4787 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4788 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4789 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4790 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4791 - PACKET3_SET_SH_REG_START; 4792 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4793 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4794 4795 /* write dispatch packet */ 4796 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4797 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4798 ib.ptr[ib.length_dw++] = 1; /* y */ 4799 ib.ptr[ib.length_dw++] = 1; /* z */ 4800 ib.ptr[ib.length_dw++] = 4801 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4802 4803 /* write CS partial flush packet */ 4804 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4805 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4806 4807 /* SGPR2 */ 4808 /* write the register state for the compute dispatch */ 4809 for (i = 0; i < gpr_reg_size; i++) { 4810 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1); 4811 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i]) 4812 - PACKET3_SET_SH_REG_START; 4813 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value; 4814 } 4815 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */ 4816 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8; 4817 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2); 4818 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO) 4819 - PACKET3_SET_SH_REG_START; 4820 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr); 4821 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr); 4822 4823 /* write dispatch packet */ 4824 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3); 4825 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */ 4826 ib.ptr[ib.length_dw++] = 1; /* y */ 4827 ib.ptr[ib.length_dw++] = 1; /* z */ 4828 ib.ptr[ib.length_dw++] = 4829 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1); 4830 4831 /* write CS partial flush packet */ 4832 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0); 4833 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4); 4834 4835 /* shedule the ib on the ring */ 4836 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 4837 if (r) { 4838 drm_err(adev_to_drm(adev), "ib schedule failed (%d).\n", r); 4839 goto fail; 4840 } 4841 4842 /* wait for the GPU to finish processing the IB */ 4843 r = dma_fence_wait(f, false); 4844 if (r) { 4845 drm_err(adev_to_drm(adev), "fence wait failed (%d).\n", r); 4846 goto fail; 4847 } 4848 4849 fail: 4850 amdgpu_ib_free(&ib, NULL); 4851 dma_fence_put(f); 4852 4853 return r; 4854 } 4855 4856 static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block) 4857 { 4858 struct amdgpu_device *adev = ip_block->adev; 4859 4860 adev->gfx.funcs = &gfx_v9_0_gfx_funcs; 4861 4862 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || 4863 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4864 adev->gfx.num_gfx_rings = 0; 4865 else 4866 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS; 4867 adev->gfx.xcc_mask = 1; 4868 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 4869 AMDGPU_MAX_COMPUTE_RINGS); 4870 gfx_v9_0_set_kiq_pm4_funcs(adev); 4871 gfx_v9_0_set_ring_funcs(adev); 4872 gfx_v9_0_set_irq_funcs(adev); 4873 gfx_v9_0_set_gds_init(adev); 4874 gfx_v9_0_set_rlc_funcs(adev); 4875 4876 /* init rlcg reg access ctrl */ 4877 gfx_v9_0_init_rlcg_reg_access_ctrl(adev); 4878 4879 amdgpu_init_rlc_reg_funcs(adev); 4880 4881 return gfx_v9_0_init_microcode(adev); 4882 } 4883 4884 static int gfx_v9_0_ecc_late_init(struct amdgpu_ip_block *ip_block) 4885 { 4886 struct amdgpu_device *adev = ip_block->adev; 4887 int r; 4888 4889 /* 4890 * Temp workaround to fix the issue that CP firmware fails to 4891 * update read pointer when CPDMA is writing clearing operation 4892 * to GDS in suspend/resume sequence on several cards. So just 4893 * limit this operation in cold boot sequence. 4894 */ 4895 if ((!adev->in_suspend) && 4896 (adev->gds.gds_size)) { 4897 r = gfx_v9_0_do_edc_gds_workarounds(adev); 4898 if (r) 4899 return r; 4900 } 4901 4902 /* requires IBs so do in late init after IB pool is initialized */ 4903 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4904 r = gfx_v9_4_2_do_edc_gpr_workarounds(adev); 4905 else 4906 r = gfx_v9_0_do_edc_gpr_workarounds(adev); 4907 4908 if (r) 4909 return r; 4910 4911 if (adev->gfx.ras && 4912 adev->gfx.ras->enable_watchdog_timer) 4913 adev->gfx.ras->enable_watchdog_timer(adev); 4914 4915 return 0; 4916 } 4917 4918 static int gfx_v9_0_late_init(struct amdgpu_ip_block *ip_block) 4919 { 4920 struct amdgpu_device *adev = ip_block->adev; 4921 int r; 4922 4923 r = gfx_v9_0_ecc_late_init(ip_block); 4924 if (r) 4925 return r; 4926 4927 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 4928 gfx_v9_4_2_debug_trap_config_init(adev, 4929 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); 4930 else 4931 gfx_v9_0_debug_trap_config_init(adev, 4932 adev->vm_manager.first_kfd_vmid, AMDGPU_NUM_VMID); 4933 4934 return 0; 4935 } 4936 4937 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev) 4938 { 4939 uint32_t rlc_setting; 4940 4941 /* if RLC is not enabled, do nothing */ 4942 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL); 4943 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK)) 4944 return false; 4945 4946 return true; 4947 } 4948 4949 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 4950 { 4951 uint32_t data; 4952 unsigned i; 4953 4954 data = RLC_SAFE_MODE__CMD_MASK; 4955 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 4956 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4957 4958 /* wait for RLC_SAFE_MODE */ 4959 for (i = 0; i < adev->usec_timeout; i++) { 4960 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 4961 break; 4962 udelay(1); 4963 } 4964 } 4965 4966 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 4967 { 4968 uint32_t data; 4969 4970 data = RLC_SAFE_MODE__CMD_MASK; 4971 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data); 4972 } 4973 4974 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev, 4975 bool enable) 4976 { 4977 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 4978 4979 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) { 4980 gfx_v9_0_enable_gfx_cg_power_gating(adev, true); 4981 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4982 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true); 4983 } else { 4984 gfx_v9_0_enable_gfx_cg_power_gating(adev, false); 4985 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE) 4986 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false); 4987 } 4988 4989 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 4990 } 4991 4992 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev, 4993 bool enable) 4994 { 4995 /* TODO: double check if we need to perform under safe mode */ 4996 /* gfx_v9_0_enter_rlc_safe_mode(adev); */ 4997 4998 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable) 4999 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true); 5000 else 5001 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false); 5002 5003 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable) 5004 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true); 5005 else 5006 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false); 5007 5008 /* gfx_v9_0_exit_rlc_safe_mode(adev); */ 5009 } 5010 5011 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5012 bool enable) 5013 { 5014 uint32_t data, def; 5015 5016 /* It is disabled by HW by default */ 5017 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) { 5018 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 5019 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5020 5021 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) 5022 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 5023 5024 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5025 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 5026 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 5027 5028 /* only for Vega10 & Raven1 */ 5029 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK; 5030 5031 if (def != data) 5032 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5033 5034 /* MGLS is a global flag to control all MGLS in GFX */ 5035 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) { 5036 /* 2 - RLC memory Light sleep */ 5037 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) { 5038 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 5039 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 5040 if (def != data) 5041 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 5042 } 5043 /* 3 - CP memory Light sleep */ 5044 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) { 5045 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 5046 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 5047 if (def != data) 5048 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 5049 } 5050 } 5051 } else { 5052 /* 1 - MGCG_OVERRIDE */ 5053 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5054 5055 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 2, 1)) 5056 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK; 5057 5058 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5059 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5060 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK | 5061 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK); 5062 5063 if (def != data) 5064 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5065 5066 /* 2 - disable MGLS in RLC */ 5067 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL); 5068 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) { 5069 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK; 5070 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data); 5071 } 5072 5073 /* 3 - disable MGLS in CP */ 5074 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL); 5075 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) { 5076 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK; 5077 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data); 5078 } 5079 } 5080 } 5081 5082 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev, 5083 bool enable) 5084 { 5085 uint32_t data, def; 5086 5087 if (!adev->gfx.num_gfx_rings) 5088 return; 5089 5090 /* Enable 3D CGCG/CGLS */ 5091 if (enable) { 5092 /* write cmd to clear cgcg/cgls ov */ 5093 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5094 /* unset CGCG override */ 5095 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 5096 /* update CGCG and CGLS override bits */ 5097 if (def != data) 5098 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5099 5100 /* enable 3Dcgcg FSM(0x0000363f) */ 5101 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 5102 5103 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 5104 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5105 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5106 else 5107 data = 0x0 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT; 5108 5109 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5110 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5111 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5112 if (def != data) 5113 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 5114 5115 /* set IDLE_POLL_COUNT(0x00900100) */ 5116 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5117 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5118 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5119 if (def != data) 5120 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5121 } else { 5122 /* Disable CGCG/CGLS */ 5123 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D); 5124 /* disable cgcg, cgls should be disabled */ 5125 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK | 5126 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK); 5127 /* disable cgcg and cgls in FSM */ 5128 if (def != data) 5129 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data); 5130 } 5131 } 5132 5133 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5134 bool enable) 5135 { 5136 uint32_t def, data; 5137 5138 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) { 5139 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE); 5140 /* unset CGCG override */ 5141 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5142 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5143 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5144 else 5145 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5146 /* update CGCG and CGLS override bits */ 5147 if (def != data) 5148 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data); 5149 5150 /* enable cgcg FSM(0x0000363F) */ 5151 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5152 5153 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1)) 5154 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5155 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5156 else 5157 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5158 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5159 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5160 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5161 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5162 if (def != data) 5163 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5164 5165 /* set IDLE_POLL_COUNT(0x00900100) */ 5166 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL); 5167 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5168 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5169 if (def != data) 5170 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data); 5171 } else { 5172 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL); 5173 /* reset CGCG/CGLS bits */ 5174 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK); 5175 /* disable cgcg and cgls in FSM */ 5176 if (def != data) 5177 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data); 5178 } 5179 } 5180 5181 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5182 bool enable) 5183 { 5184 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5185 if (enable) { 5186 /* CGCG/CGLS should be enabled after MGCG/MGLS 5187 * === MGCG + MGLS === 5188 */ 5189 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5190 /* === CGCG /CGLS for GFX 3D Only === */ 5191 gfx_v9_0_update_3d_clock_gating(adev, enable); 5192 /* === CGCG + CGLS === */ 5193 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5194 } else { 5195 /* CGCG/CGLS should be disabled before MGCG/MGLS 5196 * === CGCG + CGLS === 5197 */ 5198 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable); 5199 /* === CGCG /CGLS for GFX 3D Only === */ 5200 gfx_v9_0_update_3d_clock_gating(adev, enable); 5201 /* === MGCG + MGLS === */ 5202 gfx_v9_0_update_medium_grain_clock_gating(adev, enable); 5203 } 5204 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5205 return 0; 5206 } 5207 5208 static void gfx_v9_0_update_spm_vmid_internal(struct amdgpu_device *adev, 5209 unsigned int vmid) 5210 { 5211 u32 reg, data; 5212 5213 reg = SOC15_REG_OFFSET(GC, 0, mmRLC_SPM_MC_CNTL); 5214 if (amdgpu_sriov_is_pp_one_vf(adev)) 5215 data = RREG32_NO_KIQ(reg); 5216 else 5217 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL); 5218 5219 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 5220 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5221 5222 if (amdgpu_sriov_is_pp_one_vf(adev)) 5223 WREG32_SOC15_NO_KIQ(GC, 0, mmRLC_SPM_MC_CNTL, data); 5224 else 5225 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data); 5226 } 5227 5228 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5229 struct amdgpu_ring *ring, unsigned int vmid) 5230 { 5231 amdgpu_gfx_off_ctrl(adev, false); 5232 5233 gfx_v9_0_update_spm_vmid_internal(adev, vmid); 5234 5235 amdgpu_gfx_off_ctrl(adev, true); 5236 } 5237 5238 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev, 5239 uint32_t offset, 5240 struct soc15_reg_rlcg *entries, int arr_size) 5241 { 5242 int i; 5243 uint32_t reg; 5244 5245 if (!entries) 5246 return false; 5247 5248 for (i = 0; i < arr_size; i++) { 5249 const struct soc15_reg_rlcg *entry; 5250 5251 entry = &entries[i]; 5252 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg; 5253 if (offset == reg) 5254 return true; 5255 } 5256 5257 return false; 5258 } 5259 5260 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset) 5261 { 5262 return gfx_v9_0_check_rlcg_range(adev, offset, 5263 (void *)rlcg_access_gc_9_0, 5264 ARRAY_SIZE(rlcg_access_gc_9_0)); 5265 } 5266 5267 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = { 5268 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled, 5269 .set_safe_mode = gfx_v9_0_set_safe_mode, 5270 .unset_safe_mode = gfx_v9_0_unset_safe_mode, 5271 .init = gfx_v9_0_rlc_init, 5272 .get_csb_size = gfx_v9_0_get_csb_size, 5273 .get_csb_buffer = gfx_v9_0_get_csb_buffer, 5274 .get_cp_table_num = gfx_v9_0_cp_jump_table_num, 5275 .resume = gfx_v9_0_rlc_resume, 5276 .stop = gfx_v9_0_rlc_stop, 5277 .reset = gfx_v9_0_rlc_reset, 5278 .start = gfx_v9_0_rlc_start, 5279 .update_spm_vmid = gfx_v9_0_update_spm_vmid, 5280 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range, 5281 }; 5282 5283 static int gfx_v9_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5284 enum amd_powergating_state state) 5285 { 5286 struct amdgpu_device *adev = ip_block->adev; 5287 bool enable = (state == AMD_PG_STATE_GATE); 5288 5289 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5290 case IP_VERSION(9, 2, 2): 5291 case IP_VERSION(9, 1, 0): 5292 case IP_VERSION(9, 3, 0): 5293 if (!enable) 5294 amdgpu_gfx_off_ctrl_immediate(adev, false); 5295 5296 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) { 5297 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true); 5298 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true); 5299 } else { 5300 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false); 5301 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false); 5302 } 5303 5304 if (adev->pg_flags & AMD_PG_SUPPORT_CP) 5305 gfx_v9_0_enable_cp_power_gating(adev, true); 5306 else 5307 gfx_v9_0_enable_cp_power_gating(adev, false); 5308 5309 /* update gfx cgpg state */ 5310 gfx_v9_0_update_gfx_cg_power_gating(adev, enable); 5311 5312 /* update mgcg state */ 5313 gfx_v9_0_update_gfx_mg_power_gating(adev, enable); 5314 5315 if (enable) 5316 amdgpu_gfx_off_ctrl_immediate(adev, true); 5317 break; 5318 case IP_VERSION(9, 2, 1): 5319 amdgpu_gfx_off_ctrl_immediate(adev, enable); 5320 break; 5321 default: 5322 break; 5323 } 5324 5325 return 0; 5326 } 5327 5328 static int gfx_v9_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5329 enum amd_clockgating_state state) 5330 { 5331 struct amdgpu_device *adev = ip_block->adev; 5332 5333 if (amdgpu_sriov_vf(adev)) 5334 return 0; 5335 5336 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5337 case IP_VERSION(9, 0, 1): 5338 case IP_VERSION(9, 2, 1): 5339 case IP_VERSION(9, 4, 0): 5340 case IP_VERSION(9, 2, 2): 5341 case IP_VERSION(9, 1, 0): 5342 case IP_VERSION(9, 4, 1): 5343 case IP_VERSION(9, 3, 0): 5344 case IP_VERSION(9, 4, 2): 5345 gfx_v9_0_update_gfx_clock_gating(adev, 5346 state == AMD_CG_STATE_GATE); 5347 break; 5348 default: 5349 break; 5350 } 5351 return 0; 5352 } 5353 5354 static void gfx_v9_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5355 { 5356 struct amdgpu_device *adev = ip_block->adev; 5357 int data; 5358 5359 if (amdgpu_sriov_vf(adev)) 5360 *flags = 0; 5361 5362 /* AMD_CG_SUPPORT_GFX_MGCG */ 5363 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE)); 5364 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5365 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5366 5367 /* AMD_CG_SUPPORT_GFX_CGCG */ 5368 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL)); 5369 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5370 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5371 5372 /* AMD_CG_SUPPORT_GFX_CGLS */ 5373 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5374 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5375 5376 /* AMD_CG_SUPPORT_GFX_RLC_LS */ 5377 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL)); 5378 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) 5379 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS; 5380 5381 /* AMD_CG_SUPPORT_GFX_CP_LS */ 5382 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL)); 5383 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) 5384 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS; 5385 5386 if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 1)) { 5387 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5388 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D)); 5389 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5390 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5391 5392 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5393 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5394 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5395 } 5396 } 5397 5398 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5399 { 5400 return *ring->rptr_cpu_addr; /* gfx9 is 32bit rptr*/ 5401 } 5402 5403 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5404 { 5405 struct amdgpu_device *adev = ring->adev; 5406 u64 wptr; 5407 5408 /* XXX check if swapping is necessary on BE */ 5409 if (ring->use_doorbell) { 5410 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5411 } else { 5412 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR); 5413 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32; 5414 } 5415 5416 return wptr; 5417 } 5418 5419 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5420 { 5421 struct amdgpu_device *adev = ring->adev; 5422 5423 if (ring->use_doorbell) { 5424 /* XXX check if swapping is necessary on BE */ 5425 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); 5426 WDOORBELL64(ring->doorbell_index, ring->wptr); 5427 } else { 5428 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr)); 5429 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 5430 } 5431 } 5432 5433 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5434 { 5435 struct amdgpu_device *adev = ring->adev; 5436 u32 ref_and_mask, reg_mem_engine; 5437 5438 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5439 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5440 return; 5441 } 5442 5443 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 5444 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5445 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5446 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5447 ref_and_mask, ref_and_mask, 0x20); 5448 } 5449 5450 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5451 struct amdgpu_job *job, 5452 struct amdgpu_ib *ib, 5453 uint32_t flags) 5454 { 5455 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5456 u32 header, control = 0; 5457 5458 if (ib->flags & AMDGPU_IB_FLAG_CE) 5459 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2); 5460 else 5461 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5462 5463 control |= ib->length_dw | (vmid << 24); 5464 5465 if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 5466 control |= INDIRECT_BUFFER_PRE_ENB(1); 5467 5468 if (flags & AMDGPU_IB_PREEMPTED) 5469 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5470 5471 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid) 5472 gfx_v9_0_ring_emit_de_meta(ring, 5473 (!amdgpu_sriov_vf(ring->adev) && 5474 flags & AMDGPU_IB_PREEMPTED) ? 5475 true : false, 5476 job->gds_size > 0 && job->gds_base != 0); 5477 } 5478 5479 amdgpu_ring_write(ring, header); 5480 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 5481 amdgpu_ring_write(ring, 5482 #ifdef __BIG_ENDIAN 5483 (2 << 0) | 5484 #endif 5485 lower_32_bits(ib->gpu_addr)); 5486 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5487 amdgpu_ring_ib_on_emit_cntl(ring); 5488 amdgpu_ring_write(ring, control); 5489 } 5490 5491 static void gfx_v9_0_ring_patch_cntl(struct amdgpu_ring *ring, 5492 unsigned offset) 5493 { 5494 u32 control = ring->ring[offset]; 5495 5496 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5497 ring->ring[offset] = control; 5498 } 5499 5500 static void gfx_v9_0_ring_patch_ce_meta(struct amdgpu_ring *ring, 5501 unsigned offset) 5502 { 5503 struct amdgpu_device *adev = ring->adev; 5504 void *ce_payload_cpu_addr; 5505 uint64_t payload_offset, payload_size; 5506 5507 payload_size = sizeof(struct v9_ce_ib_state); 5508 5509 payload_offset = offsetof(struct v9_gfx_meta_data, ce_payload); 5510 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; 5511 5512 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { 5513 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, payload_size); 5514 } else { 5515 memcpy((void *)&ring->ring[offset], ce_payload_cpu_addr, 5516 (ring->buf_mask + 1 - offset) << 2); 5517 payload_size -= (ring->buf_mask + 1 - offset) << 2; 5518 memcpy((void *)&ring->ring[0], 5519 ce_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), 5520 payload_size); 5521 } 5522 } 5523 5524 static void gfx_v9_0_ring_patch_de_meta(struct amdgpu_ring *ring, 5525 unsigned offset) 5526 { 5527 struct amdgpu_device *adev = ring->adev; 5528 void *de_payload_cpu_addr; 5529 uint64_t payload_offset, payload_size; 5530 5531 payload_size = sizeof(struct v9_de_ib_state); 5532 5533 payload_offset = offsetof(struct v9_gfx_meta_data, de_payload); 5534 de_payload_cpu_addr = adev->virt.csa_cpu_addr + payload_offset; 5535 5536 ((struct v9_de_ib_state *)de_payload_cpu_addr)->ib_completion_status = 5537 IB_COMPLETION_STATUS_PREEMPTED; 5538 5539 if (offset + (payload_size >> 2) <= ring->buf_mask + 1) { 5540 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, payload_size); 5541 } else { 5542 memcpy((void *)&ring->ring[offset], de_payload_cpu_addr, 5543 (ring->buf_mask + 1 - offset) << 2); 5544 payload_size -= (ring->buf_mask + 1 - offset) << 2; 5545 memcpy((void *)&ring->ring[0], 5546 de_payload_cpu_addr + ((ring->buf_mask + 1 - offset) << 2), 5547 payload_size); 5548 } 5549 } 5550 5551 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5552 struct amdgpu_job *job, 5553 struct amdgpu_ib *ib, 5554 uint32_t flags) 5555 { 5556 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5557 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5558 5559 /* Currently, there is a high possibility to get wave ID mismatch 5560 * between ME and GDS, leading to a hw deadlock, because ME generates 5561 * different wave IDs than the GDS expects. This situation happens 5562 * randomly when at least 5 compute pipes use GDS ordered append. 5563 * The wave IDs generated by ME are also wrong after suspend/resume. 5564 * Those are probably bugs somewhere else in the kernel driver. 5565 * 5566 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 5567 * GDS to 0 for this ring (me/pipe). 5568 */ 5569 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 5570 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 5571 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID); 5572 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 5573 } 5574 5575 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 5576 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 5577 amdgpu_ring_write(ring, 5578 #ifdef __BIG_ENDIAN 5579 (2 << 0) | 5580 #endif 5581 lower_32_bits(ib->gpu_addr)); 5582 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5583 amdgpu_ring_write(ring, control); 5584 } 5585 5586 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 5587 u64 seq, unsigned flags) 5588 { 5589 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 5590 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 5591 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY; 5592 bool exec = flags & AMDGPU_FENCE_FLAG_EXEC; 5593 uint32_t dw2 = 0; 5594 5595 /* RELEASE_MEM - flush caches, send int */ 5596 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 5597 5598 if (writeback) { 5599 dw2 = EOP_TC_NC_ACTION_EN; 5600 } else { 5601 dw2 = EOP_TCL1_ACTION_EN | EOP_TC_ACTION_EN | 5602 EOP_TC_MD_ACTION_EN; 5603 } 5604 dw2 |= EOP_TC_WB_ACTION_EN | EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 5605 EVENT_INDEX(5); 5606 if (exec) 5607 dw2 |= EOP_EXEC; 5608 5609 amdgpu_ring_write(ring, dw2); 5610 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0)); 5611 5612 /* 5613 * the address should be Qword aligned if 64bit write, Dword 5614 * aligned if only send 32bit data low (discard data high) 5615 */ 5616 if (write64bit) 5617 WARN_ON(addr & 0x7); 5618 else 5619 WARN_ON(addr & 0x3); 5620 amdgpu_ring_write(ring, lower_32_bits(addr)); 5621 amdgpu_ring_write(ring, upper_32_bits(addr)); 5622 amdgpu_ring_write(ring, lower_32_bits(seq)); 5623 amdgpu_ring_write(ring, upper_32_bits(seq)); 5624 amdgpu_ring_write(ring, 0); 5625 } 5626 5627 static void gfx_v9_0_ring_emit_event_write(struct amdgpu_ring *ring, 5628 uint32_t event_type, 5629 uint32_t event_index) 5630 { 5631 amdgpu_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE, 0)); 5632 amdgpu_ring_write(ring, EVENT_TYPE(event_type) | 5633 EVENT_INDEX(event_index)); 5634 } 5635 5636 static void gfx_v9_0_emit_mem_sync(struct amdgpu_ring *ring) 5637 { 5638 const unsigned int cp_coher_cntl = 5639 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_ICACHE_ACTION_ENA(1) | 5640 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_SH_KCACHE_ACTION_ENA(1) | 5641 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_ACTION_ENA(1) | 5642 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TCL1_ACTION_ENA(1) | 5643 PACKET3_ACQUIRE_MEM_CP_COHER_CNTL_TC_WB_ACTION_ENA(1); 5644 5645 /* ACQUIRE_MEM -make one or more surfaces valid for use by the subsequent operations */ 5646 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 5)); 5647 amdgpu_ring_write(ring, cp_coher_cntl); /* CP_COHER_CNTL */ 5648 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 5649 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 5650 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 5651 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 5652 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 5653 } 5654 5655 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 5656 { 5657 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5658 gfx_v9_0_ring_emit_event_write(ring, VS_PARTIAL_FLUSH, 4); 5659 gfx_v9_0_ring_emit_event_write(ring, PS_PARTIAL_FLUSH, 4); 5660 } 5661 gfx_v9_0_ring_emit_event_write(ring, CS_PARTIAL_FLUSH, 4); 5662 gfx_v9_0_emit_mem_sync(ring); 5663 } 5664 5665 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 5666 unsigned vmid, uint64_t pd_addr) 5667 { 5668 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 5669 5670 /* compute doesn't have PFP */ 5671 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 5672 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 5673 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 5674 amdgpu_ring_write(ring, 0x0); 5675 } 5676 } 5677 5678 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5679 { 5680 return *ring->rptr_cpu_addr; /* gfx9 hardware is 32bit rptr */ 5681 } 5682 5683 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5684 { 5685 struct amdgpu_device *adev = ring->adev; 5686 u64 wptr; 5687 5688 /* XXX check if swapping is necessary on BE */ 5689 if (ring->use_doorbell) { 5690 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5691 } else { 5692 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__); 5693 wptr = 0; 5694 } 5695 return wptr; 5696 } 5697 5698 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5699 { 5700 struct amdgpu_device *adev = ring->adev; 5701 5702 /* XXX check if swapping is necessary on BE */ 5703 if (ring->use_doorbell) { 5704 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, ring->wptr); 5705 WDOORBELL64(ring->doorbell_index, ring->wptr); 5706 } else { 5707 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__); 5708 } 5709 } 5710 5711 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 5712 u64 seq, unsigned int flags) 5713 { 5714 struct amdgpu_device *adev = ring->adev; 5715 5716 /* write fence seq to the "addr" */ 5717 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5718 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5719 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 5720 amdgpu_ring_write(ring, lower_32_bits(addr)); 5721 amdgpu_ring_write(ring, upper_32_bits(addr)); 5722 amdgpu_ring_write(ring, lower_32_bits(seq)); 5723 5724 if (flags & AMDGPU_FENCE_FLAG_INT) { 5725 /* set register to trigger INT */ 5726 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5727 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 5728 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 5729 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS)); 5730 amdgpu_ring_write(ring, 0); 5731 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 5732 } 5733 } 5734 5735 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring) 5736 { 5737 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0)); 5738 amdgpu_ring_write(ring, 0); 5739 } 5740 5741 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring, bool resume) 5742 { 5743 struct amdgpu_device *adev = ring->adev; 5744 struct v9_ce_ib_state ce_payload = {0}; 5745 uint64_t offset, ce_payload_gpu_addr; 5746 void *ce_payload_cpu_addr; 5747 int cnt; 5748 5749 cnt = (sizeof(ce_payload) >> 2) + 4 - 2; 5750 5751 offset = offsetof(struct v9_gfx_meta_data, ce_payload); 5752 ce_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 5753 ce_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 5754 5755 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5756 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) | 5757 WRITE_DATA_DST_SEL(8) | 5758 WR_CONFIRM) | 5759 WRITE_DATA_CACHE_POLICY(0)); 5760 amdgpu_ring_write(ring, lower_32_bits(ce_payload_gpu_addr)); 5761 amdgpu_ring_write(ring, upper_32_bits(ce_payload_gpu_addr)); 5762 5763 amdgpu_ring_ib_on_emit_ce(ring); 5764 5765 if (resume) 5766 amdgpu_ring_write_multiple(ring, ce_payload_cpu_addr, 5767 sizeof(ce_payload) >> 2); 5768 else 5769 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, 5770 sizeof(ce_payload) >> 2); 5771 } 5772 5773 static int gfx_v9_0_ring_preempt_ib(struct amdgpu_ring *ring) 5774 { 5775 int i, r = 0; 5776 struct amdgpu_device *adev = ring->adev; 5777 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 5778 struct amdgpu_ring *kiq_ring = &kiq->ring; 5779 unsigned long flags; 5780 5781 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 5782 return -EINVAL; 5783 5784 spin_lock_irqsave(&kiq->ring_lock, flags); 5785 5786 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { 5787 spin_unlock_irqrestore(&kiq->ring_lock, flags); 5788 return -ENOMEM; 5789 } 5790 5791 /* assert preemption condition */ 5792 amdgpu_ring_set_preempt_cond_exec(ring, false); 5793 5794 ring->trail_seq += 1; 5795 amdgpu_ring_alloc(ring, 13); 5796 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr, 5797 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC | AMDGPU_FENCE_FLAG_INT); 5798 5799 /* assert IB preemption, emit the trailing fence */ 5800 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP, 5801 ring->trail_fence_gpu_addr, 5802 ring->trail_seq); 5803 5804 amdgpu_ring_commit(kiq_ring); 5805 spin_unlock_irqrestore(&kiq->ring_lock, flags); 5806 5807 /* poll the trailing fence */ 5808 for (i = 0; i < adev->usec_timeout; i++) { 5809 if (ring->trail_seq == 5810 le32_to_cpu(*ring->trail_fence_cpu_addr)) 5811 break; 5812 udelay(1); 5813 } 5814 5815 if (i >= adev->usec_timeout) { 5816 r = -EINVAL; 5817 drm_warn(adev_to_drm(adev), "ring %d timeout to preempt ib\n", ring->idx); 5818 } 5819 5820 /*reset the CP_VMID_PREEMPT after trailing fence*/ 5821 amdgpu_ring_emit_wreg(ring, 5822 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_PREEMPT), 5823 0x0); 5824 amdgpu_ring_commit(ring); 5825 5826 /* deassert preemption condition */ 5827 amdgpu_ring_set_preempt_cond_exec(ring, true); 5828 return r; 5829 } 5830 5831 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume, bool usegds) 5832 { 5833 struct amdgpu_device *adev = ring->adev; 5834 struct v9_de_ib_state de_payload = {0}; 5835 uint64_t offset, gds_addr, de_payload_gpu_addr; 5836 void *de_payload_cpu_addr; 5837 int cnt; 5838 5839 offset = offsetof(struct v9_gfx_meta_data, de_payload); 5840 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 5841 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 5842 5843 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 5844 AMDGPU_CSA_SIZE - adev->gds.gds_size, 5845 PAGE_SIZE); 5846 5847 if (usegds) { 5848 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 5849 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 5850 } 5851 5852 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 5853 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 5854 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 5855 WRITE_DATA_DST_SEL(8) | 5856 WR_CONFIRM) | 5857 WRITE_DATA_CACHE_POLICY(0)); 5858 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 5859 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 5860 5861 amdgpu_ring_ib_on_emit_de(ring); 5862 if (resume) 5863 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 5864 sizeof(de_payload) >> 2); 5865 else 5866 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 5867 sizeof(de_payload) >> 2); 5868 } 5869 5870 static void gfx_v9_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 5871 bool secure) 5872 { 5873 uint32_t v = secure ? FRAME_TMZ : 0; 5874 5875 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 5876 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 5877 } 5878 5879 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags) 5880 { 5881 uint32_t dw2 = 0; 5882 5883 gfx_v9_0_ring_emit_ce_meta(ring, 5884 (!amdgpu_sriov_vf(ring->adev) && 5885 flags & AMDGPU_IB_PREEMPTED) ? true : false); 5886 5887 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 5888 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 5889 /* set load_global_config & load_global_uconfig */ 5890 dw2 |= 0x8001; 5891 /* set load_cs_sh_regs */ 5892 dw2 |= 0x01000000; 5893 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 5894 dw2 |= 0x10002; 5895 5896 /* set load_ce_ram if preamble presented */ 5897 if (AMDGPU_PREAMBLE_IB_PRESENT & flags) 5898 dw2 |= 0x10000000; 5899 } else { 5900 /* still load_ce_ram if this is the first time preamble presented 5901 * although there is no context switch happens. 5902 */ 5903 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags) 5904 dw2 |= 0x10000000; 5905 } 5906 5907 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 5908 amdgpu_ring_write(ring, dw2); 5909 amdgpu_ring_write(ring, 0); 5910 } 5911 5912 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 5913 uint64_t addr) 5914 { 5915 unsigned ret; 5916 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 5917 amdgpu_ring_write(ring, lower_32_bits(addr)); 5918 amdgpu_ring_write(ring, upper_32_bits(addr)); 5919 /* discard following DWs if *cond_exec_gpu_addr==0 */ 5920 amdgpu_ring_write(ring, 0); 5921 ret = ring->wptr & ring->buf_mask; 5922 /* patch dummy value later */ 5923 amdgpu_ring_write(ring, 0); 5924 return ret; 5925 } 5926 5927 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 5928 uint32_t reg_val_offs) 5929 { 5930 struct amdgpu_device *adev = ring->adev; 5931 5932 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 5933 amdgpu_ring_write(ring, 0 | /* src: register*/ 5934 (5 << 8) | /* dst: memory */ 5935 (1 << 20)); /* write confirm */ 5936 amdgpu_ring_write(ring, reg); 5937 amdgpu_ring_write(ring, 0); 5938 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 5939 reg_val_offs * 4)); 5940 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 5941 reg_val_offs * 4)); 5942 } 5943 5944 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 5945 uint32_t val) 5946 { 5947 uint32_t cmd = 0; 5948 5949 switch (ring->funcs->type) { 5950 case AMDGPU_RING_TYPE_GFX: 5951 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 5952 break; 5953 case AMDGPU_RING_TYPE_KIQ: 5954 cmd = (1 << 16); /* no inc addr */ 5955 break; 5956 default: 5957 cmd = WR_CONFIRM; 5958 break; 5959 } 5960 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 5961 amdgpu_ring_write(ring, cmd); 5962 amdgpu_ring_write(ring, reg); 5963 amdgpu_ring_write(ring, 0); 5964 amdgpu_ring_write(ring, val); 5965 } 5966 5967 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 5968 uint32_t val, uint32_t mask) 5969 { 5970 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 5971 } 5972 5973 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 5974 uint32_t reg0, uint32_t reg1, 5975 uint32_t ref, uint32_t mask) 5976 { 5977 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 5978 struct amdgpu_device *adev = ring->adev; 5979 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ? 5980 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait; 5981 5982 if (fw_version_ok) 5983 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 5984 ref, mask, 0x20); 5985 else 5986 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1, 5987 ref, mask); 5988 } 5989 5990 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid) 5991 { 5992 struct amdgpu_device *adev = ring->adev; 5993 uint32_t value = 0; 5994 5995 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03); 5996 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01); 5997 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1); 5998 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid); 5999 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 6000 WREG32_SOC15(GC, 0, mmSQ_CMD, value); 6001 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 6002 } 6003 6004 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6005 enum amdgpu_interrupt_state state) 6006 { 6007 switch (state) { 6008 case AMDGPU_IRQ_STATE_DISABLE: 6009 case AMDGPU_IRQ_STATE_ENABLE: 6010 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6011 TIME_STAMP_INT_ENABLE, 6012 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6013 break; 6014 default: 6015 break; 6016 } 6017 } 6018 6019 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6020 int me, int pipe, 6021 enum amdgpu_interrupt_state state) 6022 { 6023 u32 mec_int_cntl, mec_int_cntl_reg; 6024 6025 /* 6026 * amdgpu controls only the first MEC. That's why this function only 6027 * handles the setting of interrupts for this specific MEC. All other 6028 * pipes' interrupts are set by amdkfd. 6029 */ 6030 6031 if (me == 1) { 6032 switch (pipe) { 6033 case 0: 6034 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 6035 break; 6036 case 1: 6037 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 6038 break; 6039 case 2: 6040 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 6041 break; 6042 case 3: 6043 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 6044 break; 6045 default: 6046 DRM_DEBUG("invalid pipe %d\n", pipe); 6047 return; 6048 } 6049 } else { 6050 DRM_DEBUG("invalid me %d\n", me); 6051 return; 6052 } 6053 6054 switch (state) { 6055 case AMDGPU_IRQ_STATE_DISABLE: 6056 mec_int_cntl = RREG32_SOC15_IP(GC,mec_int_cntl_reg); 6057 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6058 TIME_STAMP_INT_ENABLE, 0); 6059 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6060 break; 6061 case AMDGPU_IRQ_STATE_ENABLE: 6062 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6063 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6064 TIME_STAMP_INT_ENABLE, 1); 6065 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6066 break; 6067 default: 6068 break; 6069 } 6070 } 6071 6072 static u32 gfx_v9_0_get_cpc_int_cntl(struct amdgpu_device *adev, 6073 int me, int pipe) 6074 { 6075 /* 6076 * amdgpu controls only the first MEC. That's why this function only 6077 * handles the setting of interrupts for this specific MEC. All other 6078 * pipes' interrupts are set by amdkfd. 6079 */ 6080 if (me != 1) 6081 return 0; 6082 6083 switch (pipe) { 6084 case 0: 6085 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL); 6086 case 1: 6087 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL); 6088 case 2: 6089 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL); 6090 case 3: 6091 return SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL); 6092 default: 6093 return 0; 6094 } 6095 } 6096 6097 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6098 struct amdgpu_irq_src *source, 6099 unsigned type, 6100 enum amdgpu_interrupt_state state) 6101 { 6102 u32 cp_int_cntl_reg, cp_int_cntl; 6103 int i, j; 6104 6105 switch (state) { 6106 case AMDGPU_IRQ_STATE_DISABLE: 6107 case AMDGPU_IRQ_STATE_ENABLE: 6108 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6109 PRIV_REG_INT_ENABLE, 6110 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6111 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6112 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6113 /* MECs start at 1 */ 6114 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j); 6115 6116 if (cp_int_cntl_reg) { 6117 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6118 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6119 PRIV_REG_INT_ENABLE, 6120 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6121 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6122 } 6123 } 6124 } 6125 break; 6126 default: 6127 break; 6128 } 6129 6130 return 0; 6131 } 6132 6133 static int gfx_v9_0_set_bad_op_fault_state(struct amdgpu_device *adev, 6134 struct amdgpu_irq_src *source, 6135 unsigned type, 6136 enum amdgpu_interrupt_state state) 6137 { 6138 u32 cp_int_cntl_reg, cp_int_cntl; 6139 int i, j; 6140 6141 switch (state) { 6142 case AMDGPU_IRQ_STATE_DISABLE: 6143 case AMDGPU_IRQ_STATE_ENABLE: 6144 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6145 OPCODE_ERROR_INT_ENABLE, 6146 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6147 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6148 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6149 /* MECs start at 1 */ 6150 cp_int_cntl_reg = gfx_v9_0_get_cpc_int_cntl(adev, i + 1, j); 6151 6152 if (cp_int_cntl_reg) { 6153 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6154 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6155 OPCODE_ERROR_INT_ENABLE, 6156 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6157 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6158 } 6159 } 6160 } 6161 break; 6162 default: 6163 break; 6164 } 6165 6166 return 0; 6167 } 6168 6169 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6170 struct amdgpu_irq_src *source, 6171 unsigned type, 6172 enum amdgpu_interrupt_state state) 6173 { 6174 switch (state) { 6175 case AMDGPU_IRQ_STATE_DISABLE: 6176 case AMDGPU_IRQ_STATE_ENABLE: 6177 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6178 PRIV_INSTR_INT_ENABLE, 6179 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6180 break; 6181 default: 6182 break; 6183 } 6184 6185 return 0; 6186 } 6187 6188 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \ 6189 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 6190 CP_ECC_ERROR_INT_ENABLE, 1) 6191 6192 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \ 6193 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\ 6194 CP_ECC_ERROR_INT_ENABLE, 0) 6195 6196 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev, 6197 struct amdgpu_irq_src *source, 6198 unsigned type, 6199 enum amdgpu_interrupt_state state) 6200 { 6201 switch (state) { 6202 case AMDGPU_IRQ_STATE_DISABLE: 6203 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6204 CP_ECC_ERROR_INT_ENABLE, 0); 6205 DISABLE_ECC_ON_ME_PIPE(1, 0); 6206 DISABLE_ECC_ON_ME_PIPE(1, 1); 6207 DISABLE_ECC_ON_ME_PIPE(1, 2); 6208 DISABLE_ECC_ON_ME_PIPE(1, 3); 6209 break; 6210 6211 case AMDGPU_IRQ_STATE_ENABLE: 6212 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0, 6213 CP_ECC_ERROR_INT_ENABLE, 1); 6214 ENABLE_ECC_ON_ME_PIPE(1, 0); 6215 ENABLE_ECC_ON_ME_PIPE(1, 1); 6216 ENABLE_ECC_ON_ME_PIPE(1, 2); 6217 ENABLE_ECC_ON_ME_PIPE(1, 3); 6218 break; 6219 default: 6220 break; 6221 } 6222 6223 return 0; 6224 } 6225 6226 6227 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6228 struct amdgpu_irq_src *src, 6229 unsigned type, 6230 enum amdgpu_interrupt_state state) 6231 { 6232 switch (type) { 6233 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6234 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state); 6235 break; 6236 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6237 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6238 break; 6239 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6240 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6241 break; 6242 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6243 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6244 break; 6245 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6246 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6247 break; 6248 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP: 6249 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state); 6250 break; 6251 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP: 6252 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state); 6253 break; 6254 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP: 6255 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state); 6256 break; 6257 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP: 6258 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state); 6259 break; 6260 default: 6261 break; 6262 } 6263 return 0; 6264 } 6265 6266 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev, 6267 struct amdgpu_irq_src *source, 6268 struct amdgpu_iv_entry *entry) 6269 { 6270 int i; 6271 u8 me_id, pipe_id, queue_id; 6272 struct amdgpu_ring *ring; 6273 6274 DRM_DEBUG("IH: CP EOP\n"); 6275 me_id = (entry->ring_id & 0x0c) >> 2; 6276 pipe_id = (entry->ring_id & 0x03) >> 0; 6277 queue_id = (entry->ring_id & 0x70) >> 4; 6278 6279 switch (me_id) { 6280 case 0: 6281 if (adev->gfx.num_gfx_rings) { 6282 if (!adev->gfx.mcbp) { 6283 amdgpu_fence_process(&adev->gfx.gfx_ring[0]); 6284 } else if (!amdgpu_mcbp_handle_trailing_fence_irq(&adev->gfx.muxer)) { 6285 /* Fence signals are handled on the software rings*/ 6286 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 6287 amdgpu_fence_process(&adev->gfx.sw_gfx_ring[i]); 6288 } 6289 } 6290 break; 6291 case 1: 6292 case 2: 6293 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6294 ring = &adev->gfx.compute_ring[i]; 6295 /* Per-queue interrupt is supported for MEC starting from VI. 6296 * The interrupt can only be enabled/disabled per pipe instead of per queue. 6297 */ 6298 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id)) 6299 amdgpu_fence_process(ring); 6300 } 6301 break; 6302 } 6303 return 0; 6304 } 6305 6306 static void gfx_v9_0_fault(struct amdgpu_device *adev, 6307 struct amdgpu_iv_entry *entry) 6308 { 6309 u8 me_id, pipe_id, queue_id; 6310 struct amdgpu_ring *ring; 6311 int i; 6312 6313 me_id = (entry->ring_id & 0x0c) >> 2; 6314 pipe_id = (entry->ring_id & 0x03) >> 0; 6315 queue_id = (entry->ring_id & 0x70) >> 4; 6316 6317 switch (me_id) { 6318 case 0: 6319 drm_sched_fault(&adev->gfx.gfx_ring[0].sched); 6320 break; 6321 case 1: 6322 case 2: 6323 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6324 ring = &adev->gfx.compute_ring[i]; 6325 if (ring->me == me_id && ring->pipe == pipe_id && 6326 ring->queue == queue_id) 6327 drm_sched_fault(&ring->sched); 6328 } 6329 break; 6330 } 6331 } 6332 6333 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev, 6334 struct amdgpu_irq_src *source, 6335 struct amdgpu_iv_entry *entry) 6336 { 6337 DRM_ERROR("Illegal register access in command stream\n"); 6338 gfx_v9_0_fault(adev, entry); 6339 return 0; 6340 } 6341 6342 static int gfx_v9_0_bad_op_irq(struct amdgpu_device *adev, 6343 struct amdgpu_irq_src *source, 6344 struct amdgpu_iv_entry *entry) 6345 { 6346 DRM_ERROR("Illegal opcode in command stream\n"); 6347 gfx_v9_0_fault(adev, entry); 6348 return 0; 6349 } 6350 6351 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev, 6352 struct amdgpu_irq_src *source, 6353 struct amdgpu_iv_entry *entry) 6354 { 6355 DRM_ERROR("Illegal instruction in command stream\n"); 6356 gfx_v9_0_fault(adev, entry); 6357 return 0; 6358 } 6359 6360 6361 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = { 6362 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 6363 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT), 6364 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT) 6365 }, 6366 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 6367 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT), 6368 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT) 6369 }, 6370 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 6371 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1), 6372 0, 0 6373 }, 6374 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 6375 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2), 6376 0, 0 6377 }, 6378 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 6379 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT), 6380 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT) 6381 }, 6382 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 6383 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT), 6384 0, 0 6385 }, 6386 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 6387 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT), 6388 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT) 6389 }, 6390 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 6391 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT), 6392 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT) 6393 }, 6394 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 6395 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1), 6396 0, 0 6397 }, 6398 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 6399 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1), 6400 0, 0 6401 }, 6402 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 6403 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1), 6404 0, 0 6405 }, 6406 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6407 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC), 6408 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED) 6409 }, 6410 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 6411 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED), 6412 0, 0 6413 }, 6414 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6415 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC), 6416 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED) 6417 }, 6418 { "GDS_OA_PHY_PHY_CMD_RAM_MEM", 6419 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6420 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC), 6421 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED) 6422 }, 6423 { "GDS_OA_PHY_PHY_DATA_RAM_MEM", 6424 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 6425 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED), 6426 0, 0 6427 }, 6428 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM", 6429 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6430 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC), 6431 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED) 6432 }, 6433 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM", 6434 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6435 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC), 6436 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED) 6437 }, 6438 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM", 6439 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6440 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC), 6441 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED) 6442 }, 6443 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM", 6444 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 6445 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC), 6446 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED) 6447 }, 6448 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 6449 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT), 6450 0, 0 6451 }, 6452 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6453 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT), 6454 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT) 6455 }, 6456 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6457 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT), 6458 0, 0 6459 }, 6460 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6461 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT), 6462 0, 0 6463 }, 6464 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6465 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT), 6466 0, 0 6467 }, 6468 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 6469 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT), 6470 0, 0 6471 }, 6472 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6473 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT), 6474 0, 0 6475 }, 6476 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 6477 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT), 6478 0, 0 6479 }, 6480 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6481 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT), 6482 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT) 6483 }, 6484 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6485 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT), 6486 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT) 6487 }, 6488 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6489 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT), 6490 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT) 6491 }, 6492 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6493 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT), 6494 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT) 6495 }, 6496 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6497 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT), 6498 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT) 6499 }, 6500 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6501 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT), 6502 0, 0 6503 }, 6504 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6505 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT), 6506 0, 0 6507 }, 6508 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6509 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT), 6510 0, 0 6511 }, 6512 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6513 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT), 6514 0, 0 6515 }, 6516 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6517 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT), 6518 0, 0 6519 }, 6520 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 6521 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT), 6522 0, 0 6523 }, 6524 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6525 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT), 6526 0, 0 6527 }, 6528 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6529 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT), 6530 0, 0 6531 }, 6532 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6533 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT), 6534 0, 0 6535 }, 6536 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6537 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT), 6538 0, 0 6539 }, 6540 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6541 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT), 6542 0, 0 6543 }, 6544 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6545 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT), 6546 0, 0 6547 }, 6548 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 6549 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT), 6550 0, 0 6551 }, 6552 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 6553 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT), 6554 0, 0 6555 }, 6556 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6557 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT), 6558 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT) 6559 }, 6560 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6561 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT), 6562 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT) 6563 }, 6564 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6565 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT), 6566 0, 0 6567 }, 6568 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6569 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT), 6570 0, 0 6571 }, 6572 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6573 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT), 6574 0, 0 6575 }, 6576 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6577 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT), 6578 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT) 6579 }, 6580 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 6581 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT), 6582 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT) 6583 }, 6584 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6585 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT), 6586 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT) 6587 }, 6588 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6589 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT), 6590 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT) 6591 }, 6592 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 6593 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT), 6594 0, 0 6595 }, 6596 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6597 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT), 6598 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT) 6599 }, 6600 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6601 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT), 6602 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT) 6603 }, 6604 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6605 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT), 6606 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT) 6607 }, 6608 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6609 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT), 6610 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT) 6611 }, 6612 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6613 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT), 6614 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT) 6615 }, 6616 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6617 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT), 6618 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT) 6619 }, 6620 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 6621 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT), 6622 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT) 6623 }, 6624 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6625 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT), 6626 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT) 6627 }, 6628 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6629 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT), 6630 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT) 6631 }, 6632 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6633 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT), 6634 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT) 6635 }, 6636 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6637 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT), 6638 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT) 6639 }, 6640 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6641 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT), 6642 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT) 6643 }, 6644 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 6645 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT), 6646 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT) 6647 }, 6648 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6649 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT), 6650 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT) 6651 }, 6652 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6653 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT), 6654 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT) 6655 }, 6656 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6657 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT), 6658 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT) 6659 }, 6660 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6661 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT), 6662 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT) 6663 }, 6664 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6665 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT), 6666 0, 0 6667 }, 6668 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6669 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT), 6670 0, 0 6671 }, 6672 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6673 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT), 6674 0, 0 6675 }, 6676 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6677 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT), 6678 0, 0 6679 }, 6680 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6681 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT), 6682 0, 0 6683 }, 6684 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 6685 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT), 6686 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT) 6687 }, 6688 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6689 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT), 6690 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT) 6691 }, 6692 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6693 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT), 6694 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT) 6695 }, 6696 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6697 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT), 6698 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT) 6699 }, 6700 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6701 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT), 6702 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT) 6703 }, 6704 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6705 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT), 6706 0, 0 6707 }, 6708 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6709 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT), 6710 0, 0 6711 }, 6712 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6713 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT), 6714 0, 0 6715 }, 6716 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6717 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT), 6718 0, 0 6719 }, 6720 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 6721 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT), 6722 0, 0 6723 }, 6724 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6725 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT), 6726 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT) 6727 }, 6728 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6729 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT), 6730 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT) 6731 }, 6732 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6733 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT), 6734 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT) 6735 }, 6736 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6737 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT), 6738 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT) 6739 }, 6740 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6741 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT), 6742 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT) 6743 }, 6744 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6745 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT), 6746 0, 0 6747 }, 6748 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6749 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT), 6750 0, 0 6751 }, 6752 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6753 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT), 6754 0, 0 6755 }, 6756 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6757 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT), 6758 0, 0 6759 }, 6760 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 6761 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT), 6762 0, 0 6763 }, 6764 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6765 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT), 6766 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT) 6767 }, 6768 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6769 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT), 6770 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT) 6771 }, 6772 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6773 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT), 6774 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT) 6775 }, 6776 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6777 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT), 6778 0, 0 6779 }, 6780 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6781 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT), 6782 0, 0 6783 }, 6784 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6785 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT), 6786 0, 0 6787 }, 6788 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6789 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT), 6790 0, 0 6791 }, 6792 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6793 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT), 6794 0, 0 6795 }, 6796 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 6797 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT), 6798 0, 0 6799 } 6800 }; 6801 6802 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev, 6803 void *inject_if, uint32_t instance_mask) 6804 { 6805 struct ras_inject_if *info = (struct ras_inject_if *)inject_if; 6806 int ret; 6807 struct ta_ras_trigger_error_input block_info = { 0 }; 6808 6809 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 6810 return -EINVAL; 6811 6812 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks)) 6813 return -EINVAL; 6814 6815 if (!ras_gfx_subblocks[info->head.sub_block_index].name) 6816 return -EPERM; 6817 6818 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type & 6819 info->head.type)) { 6820 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n", 6821 ras_gfx_subblocks[info->head.sub_block_index].name, 6822 info->head.type); 6823 return -EPERM; 6824 } 6825 6826 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type & 6827 info->head.type)) { 6828 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n", 6829 ras_gfx_subblocks[info->head.sub_block_index].name, 6830 info->head.type); 6831 return -EPERM; 6832 } 6833 6834 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block); 6835 block_info.sub_block_index = 6836 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock; 6837 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type); 6838 block_info.address = info->address; 6839 block_info.value = info->value; 6840 6841 mutex_lock(&adev->grbm_idx_mutex); 6842 ret = psp_ras_trigger_error(&adev->psp, &block_info, instance_mask); 6843 mutex_unlock(&adev->grbm_idx_mutex); 6844 6845 return ret; 6846 } 6847 6848 static const char * const vml2_mems[] = { 6849 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0", 6850 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1", 6851 "UTC_VML2_BANK_CACHE_0_4K_MEM0", 6852 "UTC_VML2_BANK_CACHE_0_4K_MEM1", 6853 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0", 6854 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1", 6855 "UTC_VML2_BANK_CACHE_1_4K_MEM0", 6856 "UTC_VML2_BANK_CACHE_1_4K_MEM1", 6857 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0", 6858 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1", 6859 "UTC_VML2_BANK_CACHE_2_4K_MEM0", 6860 "UTC_VML2_BANK_CACHE_2_4K_MEM1", 6861 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0", 6862 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1", 6863 "UTC_VML2_BANK_CACHE_3_4K_MEM0", 6864 "UTC_VML2_BANK_CACHE_3_4K_MEM1", 6865 }; 6866 6867 static const char * const vml2_walker_mems[] = { 6868 "UTC_VML2_CACHE_PDE0_MEM0", 6869 "UTC_VML2_CACHE_PDE0_MEM1", 6870 "UTC_VML2_CACHE_PDE1_MEM0", 6871 "UTC_VML2_CACHE_PDE1_MEM1", 6872 "UTC_VML2_CACHE_PDE2_MEM0", 6873 "UTC_VML2_CACHE_PDE2_MEM1", 6874 "UTC_VML2_RDIF_LOG_FIFO", 6875 }; 6876 6877 static const char * const atc_l2_cache_2m_mems[] = { 6878 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM", 6879 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM", 6880 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM", 6881 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM", 6882 }; 6883 6884 static const char *atc_l2_cache_4k_mems[] = { 6885 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0", 6886 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1", 6887 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2", 6888 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3", 6889 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4", 6890 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5", 6891 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6", 6892 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7", 6893 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0", 6894 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1", 6895 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2", 6896 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3", 6897 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4", 6898 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5", 6899 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6", 6900 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7", 6901 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0", 6902 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1", 6903 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2", 6904 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3", 6905 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4", 6906 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5", 6907 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6", 6908 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7", 6909 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0", 6910 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1", 6911 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2", 6912 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3", 6913 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4", 6914 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5", 6915 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6", 6916 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7", 6917 }; 6918 6919 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev, 6920 struct ras_err_data *err_data) 6921 { 6922 uint32_t i, data; 6923 uint32_t sec_count, ded_count; 6924 6925 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 6926 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 6927 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 6928 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 6929 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 6930 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 6931 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 6932 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 6933 6934 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 6935 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 6936 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 6937 6938 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT); 6939 if (sec_count) { 6940 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6941 "SEC %d\n", i, vml2_mems[i], sec_count); 6942 err_data->ce_count += sec_count; 6943 } 6944 6945 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT); 6946 if (ded_count) { 6947 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6948 "DED %d\n", i, vml2_mems[i], ded_count); 6949 err_data->ue_count += ded_count; 6950 } 6951 } 6952 6953 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 6954 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 6955 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 6956 6957 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6958 SEC_COUNT); 6959 if (sec_count) { 6960 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6961 "SEC %d\n", i, vml2_walker_mems[i], sec_count); 6962 err_data->ce_count += sec_count; 6963 } 6964 6965 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT, 6966 DED_COUNT); 6967 if (ded_count) { 6968 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6969 "DED %d\n", i, vml2_walker_mems[i], ded_count); 6970 err_data->ue_count += ded_count; 6971 } 6972 } 6973 6974 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 6975 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 6976 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 6977 6978 sec_count = (data & 0x00006000L) >> 0xd; 6979 if (sec_count) { 6980 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6981 "SEC %d\n", i, atc_l2_cache_2m_mems[i], 6982 sec_count); 6983 err_data->ce_count += sec_count; 6984 } 6985 } 6986 6987 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 6988 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 6989 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 6990 6991 sec_count = (data & 0x00006000L) >> 0xd; 6992 if (sec_count) { 6993 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 6994 "SEC %d\n", i, atc_l2_cache_4k_mems[i], 6995 sec_count); 6996 err_data->ce_count += sec_count; 6997 } 6998 6999 ded_count = (data & 0x00018000L) >> 0xf; 7000 if (ded_count) { 7001 dev_info(adev->dev, "Instance[%d]: SubBlock %s, " 7002 "DED %d\n", i, atc_l2_cache_4k_mems[i], 7003 ded_count); 7004 err_data->ue_count += ded_count; 7005 } 7006 } 7007 7008 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 7009 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 7010 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 7011 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 7012 7013 return 0; 7014 } 7015 7016 static int gfx_v9_0_ras_error_count(struct amdgpu_device *adev, 7017 const struct soc15_reg_entry *reg, 7018 uint32_t se_id, uint32_t inst_id, uint32_t value, 7019 uint32_t *sec_count, uint32_t *ded_count) 7020 { 7021 uint32_t i; 7022 uint32_t sec_cnt, ded_cnt; 7023 7024 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) { 7025 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset || 7026 gfx_v9_0_ras_fields[i].seg != reg->seg || 7027 gfx_v9_0_ras_fields[i].inst != reg->inst) 7028 continue; 7029 7030 sec_cnt = (value & 7031 gfx_v9_0_ras_fields[i].sec_count_mask) >> 7032 gfx_v9_0_ras_fields[i].sec_count_shift; 7033 if (sec_cnt) { 7034 dev_info(adev->dev, "GFX SubBlock %s, " 7035 "Instance[%d][%d], SEC %d\n", 7036 gfx_v9_0_ras_fields[i].name, 7037 se_id, inst_id, 7038 sec_cnt); 7039 *sec_count += sec_cnt; 7040 } 7041 7042 ded_cnt = (value & 7043 gfx_v9_0_ras_fields[i].ded_count_mask) >> 7044 gfx_v9_0_ras_fields[i].ded_count_shift; 7045 if (ded_cnt) { 7046 dev_info(adev->dev, "GFX SubBlock %s, " 7047 "Instance[%d][%d], DED %d\n", 7048 gfx_v9_0_ras_fields[i].name, 7049 se_id, inst_id, 7050 ded_cnt); 7051 *ded_count += ded_cnt; 7052 } 7053 } 7054 7055 return 0; 7056 } 7057 7058 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev) 7059 { 7060 int i, j, k; 7061 7062 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 7063 return; 7064 7065 /* read back registers to clear the counters */ 7066 mutex_lock(&adev->grbm_idx_mutex); 7067 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 7068 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 7069 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 7070 amdgpu_gfx_select_se_sh(adev, j, 0x0, k, 0); 7071 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 7072 } 7073 } 7074 } 7075 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000); 7076 mutex_unlock(&adev->grbm_idx_mutex); 7077 7078 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 7079 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0); 7080 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 7081 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0); 7082 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 7083 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0); 7084 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 7085 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0); 7086 7087 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) { 7088 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i); 7089 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT); 7090 } 7091 7092 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) { 7093 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i); 7094 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT); 7095 } 7096 7097 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) { 7098 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i); 7099 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT); 7100 } 7101 7102 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) { 7103 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i); 7104 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT); 7105 } 7106 7107 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255); 7108 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255); 7109 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255); 7110 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255); 7111 } 7112 7113 static void gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev, 7114 void *ras_error_status) 7115 { 7116 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; 7117 uint32_t sec_count = 0, ded_count = 0; 7118 uint32_t i, j, k; 7119 uint32_t reg_value; 7120 7121 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX)) 7122 return; 7123 7124 err_data->ue_count = 0; 7125 err_data->ce_count = 0; 7126 7127 mutex_lock(&adev->grbm_idx_mutex); 7128 7129 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) { 7130 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) { 7131 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) { 7132 amdgpu_gfx_select_se_sh(adev, j, 0, k, 0); 7133 reg_value = 7134 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i])); 7135 if (reg_value) 7136 gfx_v9_0_ras_error_count(adev, 7137 &gfx_v9_0_edc_counter_regs[i], 7138 j, k, reg_value, 7139 &sec_count, &ded_count); 7140 } 7141 } 7142 } 7143 7144 err_data->ce_count += sec_count; 7145 err_data->ue_count += ded_count; 7146 7147 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7148 mutex_unlock(&adev->grbm_idx_mutex); 7149 7150 gfx_v9_0_query_utc_edc_status(adev, err_data); 7151 } 7152 7153 static void gfx_v9_0_emit_wave_limit_cs(struct amdgpu_ring *ring, 7154 uint32_t pipe, bool enable) 7155 { 7156 struct amdgpu_device *adev = ring->adev; 7157 uint32_t val; 7158 uint32_t wcl_cs_reg; 7159 7160 /* mmSPI_WCL_PIPE_PERCENT_CS[0-7]_DEFAULT values are same */ 7161 val = enable ? 0x1 : mmSPI_WCL_PIPE_PERCENT_CS0_DEFAULT; 7162 7163 switch (pipe) { 7164 case 0: 7165 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS0); 7166 break; 7167 case 1: 7168 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS1); 7169 break; 7170 case 2: 7171 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS2); 7172 break; 7173 case 3: 7174 wcl_cs_reg = SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_CS3); 7175 break; 7176 default: 7177 DRM_DEBUG("invalid pipe %d\n", pipe); 7178 return; 7179 } 7180 7181 amdgpu_ring_emit_wreg(ring, wcl_cs_reg, val); 7182 7183 } 7184 static void gfx_v9_0_emit_wave_limit(struct amdgpu_ring *ring, bool enable) 7185 { 7186 struct amdgpu_device *adev = ring->adev; 7187 uint32_t val; 7188 int i; 7189 7190 7191 /* mmSPI_WCL_PIPE_PERCENT_GFX is 7 bit multiplier register to limit 7192 * number of gfx waves. Setting 5 bit will make sure gfx only gets 7193 * around 25% of gpu resources. 7194 */ 7195 val = enable ? 0x1f : mmSPI_WCL_PIPE_PERCENT_GFX_DEFAULT; 7196 amdgpu_ring_emit_wreg(ring, 7197 SOC15_REG_OFFSET(GC, 0, mmSPI_WCL_PIPE_PERCENT_GFX), 7198 val); 7199 7200 /* Restrict waves for normal/low priority compute queues as well 7201 * to get best QoS for high priority compute jobs. 7202 * 7203 * amdgpu controls only 1st ME(0-3 CS pipes). 7204 */ 7205 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 7206 if (i != ring->pipe) 7207 gfx_v9_0_emit_wave_limit_cs(ring, i, enable); 7208 7209 } 7210 } 7211 7212 static void gfx_v9_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop) 7213 { 7214 /* Header itself is a NOP packet */ 7215 if (num_nop == 1) { 7216 amdgpu_ring_write(ring, ring->funcs->nop); 7217 return; 7218 } 7219 7220 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 7221 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); 7222 7223 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ 7224 amdgpu_ring_insert_nop(ring, num_nop - 1); 7225 } 7226 7227 static void gfx_v9_0_ring_emit_wreg_me(struct amdgpu_ring *ring, 7228 uint32_t reg, 7229 uint32_t val) 7230 { 7231 uint32_t cmd = 0; 7232 7233 switch (ring->funcs->type) { 7234 case AMDGPU_RING_TYPE_KIQ: 7235 cmd = (1 << 16); /* no inc addr */ 7236 break; 7237 default: 7238 cmd = WR_CONFIRM; 7239 break; 7240 } 7241 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 7242 amdgpu_ring_write(ring, cmd); 7243 amdgpu_ring_write(ring, reg); 7244 amdgpu_ring_write(ring, 0); 7245 amdgpu_ring_write(ring, val); 7246 } 7247 7248 static int gfx_v9_0_reset_kgq(struct amdgpu_ring *ring, 7249 unsigned int vmid, 7250 struct amdgpu_fence *timedout_fence) 7251 { 7252 struct amdgpu_device *adev = ring->adev; 7253 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 7254 struct amdgpu_ring *kiq_ring = &kiq->ring; 7255 unsigned long flags; 7256 u32 tmp; 7257 int r; 7258 7259 amdgpu_ring_reset_helper_begin(ring, timedout_fence); 7260 7261 spin_lock_irqsave(&kiq->ring_lock, flags); 7262 7263 if (amdgpu_ring_alloc(kiq_ring, 5)) { 7264 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7265 return -ENOMEM; 7266 } 7267 7268 /* send the reset - 5 */ 7269 tmp = REG_SET_FIELD(0, CP_VMID_RESET, RESET_REQUEST, 1 << vmid); 7270 gfx_v9_0_ring_emit_wreg(kiq_ring, 7271 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), tmp); 7272 amdgpu_ring_commit(kiq_ring); 7273 r = amdgpu_ring_test_ring(kiq_ring); 7274 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7275 if (r) 7276 return r; 7277 7278 if (amdgpu_ring_alloc(ring, 8 + 7 + 5 + 2 + 8 + 7)) 7279 return -ENOMEM; 7280 /* emit the fence to finish the reset - 8 */ 7281 ring->trail_seq++; 7282 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr, 7283 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC); 7284 /* wait for the fence - 7 */ 7285 gfx_v9_0_wait_reg_mem(ring, 0, 1, 0, 7286 lower_32_bits(ring->trail_fence_gpu_addr), 7287 upper_32_bits(ring->trail_fence_gpu_addr), 7288 ring->trail_seq, 0xffffffff, 4); 7289 /* clear mmCP_VMID_RESET - 5 */ 7290 gfx_v9_0_ring_emit_wreg_me(ring, 7291 SOC15_REG_OFFSET(GC, 0, mmCP_VMID_RESET), 0); 7292 /* event write ENABLE_LEGACY_PIPELINE - 2 */ 7293 gfx_v9_0_ring_emit_event_write(ring, ENABLE_LEGACY_PIPELINE, 0); 7294 /* emit a regular fence - 8 */ 7295 ring->trail_seq++; 7296 gfx_v9_0_ring_emit_fence(ring, ring->trail_fence_gpu_addr, 7297 ring->trail_seq, AMDGPU_FENCE_FLAG_EXEC); 7298 /* wait for the fence - 7 */ 7299 gfx_v9_0_wait_reg_mem(ring, 1, 1, 0, 7300 lower_32_bits(ring->trail_fence_gpu_addr), 7301 upper_32_bits(ring->trail_fence_gpu_addr), 7302 ring->trail_seq, 0xffffffff, 4); 7303 amdgpu_ring_commit(ring); 7304 /* wait for the commands to complete */ 7305 r = amdgpu_ring_test_ring(ring); 7306 if (r) 7307 return r; 7308 7309 return amdgpu_ring_reset_helper_end(ring, timedout_fence); 7310 } 7311 7312 static int gfx_v9_0_reset_kcq(struct amdgpu_ring *ring, 7313 unsigned int vmid, 7314 struct amdgpu_fence *timedout_fence) 7315 { 7316 struct amdgpu_device *adev = ring->adev; 7317 struct amdgpu_kiq *kiq = &adev->gfx.kiq[0]; 7318 struct amdgpu_ring *kiq_ring = &kiq->ring; 7319 unsigned long flags; 7320 int i, r; 7321 7322 if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues) 7323 return -EINVAL; 7324 7325 amdgpu_ring_reset_helper_begin(ring, timedout_fence); 7326 7327 spin_lock_irqsave(&kiq->ring_lock, flags); 7328 7329 if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) { 7330 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7331 return -ENOMEM; 7332 } 7333 7334 kiq->pmf->kiq_unmap_queues(kiq_ring, ring, RESET_QUEUES, 7335 0, 0); 7336 amdgpu_ring_commit(kiq_ring); 7337 7338 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7339 7340 r = amdgpu_ring_test_ring(kiq_ring); 7341 if (r) 7342 return r; 7343 7344 /* make sure dequeue is complete*/ 7345 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 7346 mutex_lock(&adev->srbm_mutex); 7347 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, 0); 7348 for (i = 0; i < adev->usec_timeout; i++) { 7349 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1)) 7350 break; 7351 udelay(1); 7352 } 7353 if (i >= adev->usec_timeout) 7354 r = -ETIMEDOUT; 7355 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 7356 mutex_unlock(&adev->srbm_mutex); 7357 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 7358 if (r) { 7359 dev_err(adev->dev, "fail to wait on hqd deactive\n"); 7360 return r; 7361 } 7362 7363 r = gfx_v9_0_kcq_init_queue(ring, true); 7364 if (r) { 7365 dev_err(adev->dev, "fail to init kcq\n"); 7366 return r; 7367 } 7368 spin_lock_irqsave(&kiq->ring_lock, flags); 7369 r = amdgpu_ring_alloc(kiq_ring, kiq->pmf->map_queues_size); 7370 if (r) { 7371 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7372 return -ENOMEM; 7373 } 7374 kiq->pmf->kiq_map_queues(kiq_ring, ring); 7375 amdgpu_ring_commit(kiq_ring); 7376 r = amdgpu_ring_test_ring(kiq_ring); 7377 spin_unlock_irqrestore(&kiq->ring_lock, flags); 7378 if (r) { 7379 DRM_ERROR("fail to remap queue\n"); 7380 return r; 7381 } 7382 return amdgpu_ring_reset_helper_end(ring, timedout_fence); 7383 } 7384 7385 static void gfx_v9_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 7386 { 7387 struct amdgpu_device *adev = ip_block->adev; 7388 uint32_t i, j, k, reg, index = 0; 7389 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9); 7390 7391 if (!adev->gfx.ip_dump_core) 7392 return; 7393 7394 for (i = 0; i < reg_count; i++) 7395 drm_printf(p, "%-50s \t 0x%08x\n", 7396 gc_reg_list_9[i].reg_name, 7397 adev->gfx.ip_dump_core[i]); 7398 7399 /* print compute queue registers for all instances */ 7400 if (!adev->gfx.ip_dump_compute_queues) 7401 return; 7402 7403 reg_count = ARRAY_SIZE(gc_cp_reg_list_9); 7404 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n", 7405 adev->gfx.mec.num_mec, 7406 adev->gfx.mec.num_pipe_per_mec, 7407 adev->gfx.mec.num_queue_per_pipe); 7408 7409 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 7410 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 7411 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 7412 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k); 7413 for (reg = 0; reg < reg_count; reg++) { 7414 if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP) 7415 drm_printf(p, "%-50s \t 0x%08x\n", 7416 "mmCP_MEC_ME2_HEADER_DUMP", 7417 adev->gfx.ip_dump_compute_queues[index + reg]); 7418 else 7419 drm_printf(p, "%-50s \t 0x%08x\n", 7420 gc_cp_reg_list_9[reg].reg_name, 7421 adev->gfx.ip_dump_compute_queues[index + reg]); 7422 } 7423 index += reg_count; 7424 } 7425 } 7426 } 7427 7428 } 7429 7430 static void gfx_v9_ip_dump(struct amdgpu_ip_block *ip_block) 7431 { 7432 struct amdgpu_device *adev = ip_block->adev; 7433 uint32_t i, j, k, reg, index = 0; 7434 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_9); 7435 7436 if (!adev->gfx.ip_dump_core || !adev->gfx.num_gfx_rings) 7437 return; 7438 7439 amdgpu_gfx_off_ctrl(adev, false); 7440 for (i = 0; i < reg_count; i++) 7441 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_9[i])); 7442 amdgpu_gfx_off_ctrl(adev, true); 7443 7444 /* dump compute queue registers for all instances */ 7445 if (!adev->gfx.ip_dump_compute_queues) 7446 return; 7447 7448 reg_count = ARRAY_SIZE(gc_cp_reg_list_9); 7449 amdgpu_gfx_off_ctrl(adev, false); 7450 mutex_lock(&adev->srbm_mutex); 7451 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 7452 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 7453 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 7454 /* ME0 is for GFX so start from 1 for CP */ 7455 soc15_grbm_select(adev, 1 + i, j, k, 0, 0); 7456 7457 for (reg = 0; reg < reg_count; reg++) { 7458 if (i && gc_cp_reg_list_9[reg].reg_offset == mmCP_MEC_ME1_HEADER_DUMP) 7459 adev->gfx.ip_dump_compute_queues[index + reg] = 7460 RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_MEC_ME2_HEADER_DUMP)); 7461 else 7462 adev->gfx.ip_dump_compute_queues[index + reg] = 7463 RREG32(SOC15_REG_ENTRY_OFFSET( 7464 gc_cp_reg_list_9[reg])); 7465 } 7466 index += reg_count; 7467 } 7468 } 7469 } 7470 soc15_grbm_select(adev, 0, 0, 0, 0, 0); 7471 mutex_unlock(&adev->srbm_mutex); 7472 amdgpu_gfx_off_ctrl(adev, true); 7473 7474 } 7475 7476 static void gfx_v9_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring) 7477 { 7478 struct amdgpu_device *adev = ring->adev; 7479 7480 /* Emit the cleaner shader */ 7481 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2)) 7482 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0)); 7483 else 7484 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER_9_0, 0)); 7485 7486 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */ 7487 } 7488 7489 static void gfx_v9_0_ring_begin_use_compute(struct amdgpu_ring *ring) 7490 { 7491 struct amdgpu_device *adev = ring->adev; 7492 struct amdgpu_ip_block *gfx_block = 7493 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 7494 7495 amdgpu_gfx_enforce_isolation_ring_begin_use(ring); 7496 7497 /* Raven and PCO APUs seem to have stability issues 7498 * with compute and gfxoff and gfx pg. Disable gfx pg during 7499 * submission and allow again afterwards. 7500 */ 7501 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0)) 7502 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_UNGATE); 7503 } 7504 7505 static void gfx_v9_0_ring_end_use_compute(struct amdgpu_ring *ring) 7506 { 7507 struct amdgpu_device *adev = ring->adev; 7508 struct amdgpu_ip_block *gfx_block = 7509 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX); 7510 7511 /* Raven and PCO APUs seem to have stability issues 7512 * with compute and gfxoff and gfx pg. Disable gfx pg during 7513 * submission and allow again afterwards. 7514 */ 7515 if (gfx_block && amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 1, 0)) 7516 gfx_v9_0_set_powergating_state(gfx_block, AMD_PG_STATE_GATE); 7517 7518 amdgpu_gfx_enforce_isolation_ring_end_use(ring); 7519 } 7520 7521 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = { 7522 .name = "gfx_v9_0", 7523 .early_init = gfx_v9_0_early_init, 7524 .late_init = gfx_v9_0_late_init, 7525 .sw_init = gfx_v9_0_sw_init, 7526 .sw_fini = gfx_v9_0_sw_fini, 7527 .hw_init = gfx_v9_0_hw_init, 7528 .hw_fini = gfx_v9_0_hw_fini, 7529 .suspend = gfx_v9_0_suspend, 7530 .resume = gfx_v9_0_resume, 7531 .is_idle = gfx_v9_0_is_idle, 7532 .wait_for_idle = gfx_v9_0_wait_for_idle, 7533 .soft_reset = gfx_v9_0_soft_reset, 7534 .set_clockgating_state = gfx_v9_0_set_clockgating_state, 7535 .set_powergating_state = gfx_v9_0_set_powergating_state, 7536 .get_clockgating_state = gfx_v9_0_get_clockgating_state, 7537 .dump_ip_state = gfx_v9_ip_dump, 7538 .print_ip_state = gfx_v9_ip_print, 7539 }; 7540 7541 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = { 7542 .type = AMDGPU_RING_TYPE_GFX, 7543 .align_mask = 0xff, 7544 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7545 .support_64bit_ptrs = true, 7546 .secure_submission_supported = true, 7547 .get_rptr = gfx_v9_0_ring_get_rptr_gfx, 7548 .get_wptr = gfx_v9_0_ring_get_wptr_gfx, 7549 .set_wptr = gfx_v9_0_ring_set_wptr_gfx, 7550 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 7551 5 + /* COND_EXEC */ 7552 13 + /* PIPELINE_SYNC */ 7553 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7554 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7555 2 + /* VM_FLUSH */ 7556 8 + /* FENCE for VM_FLUSH */ 7557 20 + /* GDS switch */ 7558 4 + /* double SWITCH_BUFFER, 7559 the first COND_EXEC jump to the place just 7560 prior to this double SWITCH_BUFFER */ 7561 5 + /* COND_EXEC */ 7562 7 + /* HDP_flush */ 7563 4 + /* VGT_flush */ 7564 14 + /* CE_META */ 7565 31 + /* DE_META */ 7566 3 + /* CNTX_CTRL */ 7567 5 + /* HDP_INVL */ 7568 8 + 8 + /* FENCE x2 */ 7569 2 + /* SWITCH_BUFFER */ 7570 7 + /* gfx_v9_0_emit_mem_sync */ 7571 2, /* gfx_v9_0_ring_emit_cleaner_shader */ 7572 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 7573 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 7574 .emit_fence = gfx_v9_0_ring_emit_fence, 7575 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 7576 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 7577 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 7578 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 7579 .test_ring = gfx_v9_0_ring_test_ring, 7580 .insert_nop = gfx_v9_ring_insert_nop, 7581 .pad_ib = amdgpu_ring_generic_pad_ib, 7582 .emit_switch_buffer = gfx_v9_ring_emit_sb, 7583 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 7584 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 7585 .preempt_ib = gfx_v9_0_ring_preempt_ib, 7586 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 7587 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7588 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7589 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7590 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 7591 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader, 7592 .reset = gfx_v9_0_reset_kgq, 7593 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use, 7594 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use, 7595 }; 7596 7597 static const struct amdgpu_ring_funcs gfx_v9_0_sw_ring_funcs_gfx = { 7598 .type = AMDGPU_RING_TYPE_GFX, 7599 .align_mask = 0xff, 7600 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7601 .support_64bit_ptrs = true, 7602 .secure_submission_supported = true, 7603 .get_rptr = amdgpu_sw_ring_get_rptr_gfx, 7604 .get_wptr = amdgpu_sw_ring_get_wptr_gfx, 7605 .set_wptr = amdgpu_sw_ring_set_wptr_gfx, 7606 .emit_frame_size = /* totally 242 maximum if 16 IBs */ 7607 5 + /* COND_EXEC */ 7608 13 + /* PIPELINE_SYNC */ 7609 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7610 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7611 2 + /* VM_FLUSH */ 7612 8 + /* FENCE for VM_FLUSH */ 7613 20 + /* GDS switch */ 7614 4 + /* double SWITCH_BUFFER, 7615 * the first COND_EXEC jump to the place just 7616 * prior to this double SWITCH_BUFFER 7617 */ 7618 5 + /* COND_EXEC */ 7619 7 + /* HDP_flush */ 7620 4 + /* VGT_flush */ 7621 14 + /* CE_META */ 7622 31 + /* DE_META */ 7623 3 + /* CNTX_CTRL */ 7624 5 + /* HDP_INVL */ 7625 8 + 8 + /* FENCE x2 */ 7626 2 + /* SWITCH_BUFFER */ 7627 7 + /* gfx_v9_0_emit_mem_sync */ 7628 2, /* gfx_v9_0_ring_emit_cleaner_shader */ 7629 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */ 7630 .emit_ib = gfx_v9_0_ring_emit_ib_gfx, 7631 .emit_fence = gfx_v9_0_ring_emit_fence, 7632 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 7633 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 7634 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 7635 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 7636 .test_ring = gfx_v9_0_ring_test_ring, 7637 .test_ib = gfx_v9_0_ring_test_ib, 7638 .insert_nop = gfx_v9_ring_insert_nop, 7639 .pad_ib = amdgpu_ring_generic_pad_ib, 7640 .emit_switch_buffer = gfx_v9_ring_emit_sb, 7641 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl, 7642 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec, 7643 .emit_frame_cntl = gfx_v9_0_ring_emit_frame_cntl, 7644 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7645 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7646 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7647 .soft_recovery = gfx_v9_0_ring_soft_recovery, 7648 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 7649 .patch_cntl = gfx_v9_0_ring_patch_cntl, 7650 .patch_de = gfx_v9_0_ring_patch_de_meta, 7651 .patch_ce = gfx_v9_0_ring_patch_ce_meta, 7652 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader, 7653 .begin_use = amdgpu_gfx_enforce_isolation_ring_begin_use, 7654 .end_use = amdgpu_gfx_enforce_isolation_ring_end_use, 7655 }; 7656 7657 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = { 7658 .type = AMDGPU_RING_TYPE_COMPUTE, 7659 .align_mask = 0xff, 7660 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7661 .support_64bit_ptrs = true, 7662 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 7663 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 7664 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 7665 .emit_frame_size = 7666 20 + /* gfx_v9_0_ring_emit_gds_switch */ 7667 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 7668 5 + /* hdp invalidate */ 7669 9 + /* gfx_v9_0_ring_emit_pipeline_sync */ 7670 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7671 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7672 8 + 8 + 8 + /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */ 7673 7 + /* gfx_v9_0_emit_mem_sync */ 7674 5 + /* gfx_v9_0_emit_wave_limit for updating mmSPI_WCL_PIPE_PERCENT_GFX register */ 7675 15 + /* for updating 3 mmSPI_WCL_PIPE_PERCENT_CS registers */ 7676 2, /* gfx_v9_0_ring_emit_cleaner_shader */ 7677 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 7678 .emit_ib = gfx_v9_0_ring_emit_ib_compute, 7679 .emit_fence = gfx_v9_0_ring_emit_fence, 7680 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync, 7681 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush, 7682 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch, 7683 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 7684 .test_ring = gfx_v9_0_ring_test_ring, 7685 .test_ib = gfx_v9_0_ring_test_ib, 7686 .insert_nop = gfx_v9_ring_insert_nop, 7687 .pad_ib = amdgpu_ring_generic_pad_ib, 7688 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7689 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7690 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7691 .emit_mem_sync = gfx_v9_0_emit_mem_sync, 7692 .emit_wave_limit = gfx_v9_0_emit_wave_limit, 7693 .reset = gfx_v9_0_reset_kcq, 7694 .emit_cleaner_shader = gfx_v9_0_ring_emit_cleaner_shader, 7695 .begin_use = gfx_v9_0_ring_begin_use_compute, 7696 .end_use = gfx_v9_0_ring_end_use_compute, 7697 }; 7698 7699 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = { 7700 .type = AMDGPU_RING_TYPE_KIQ, 7701 .align_mask = 0xff, 7702 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7703 .support_64bit_ptrs = true, 7704 .get_rptr = gfx_v9_0_ring_get_rptr_compute, 7705 .get_wptr = gfx_v9_0_ring_get_wptr_compute, 7706 .set_wptr = gfx_v9_0_ring_set_wptr_compute, 7707 .emit_frame_size = 7708 20 + /* gfx_v9_0_ring_emit_gds_switch */ 7709 7 + /* gfx_v9_0_ring_emit_hdp_flush */ 7710 5 + /* hdp invalidate */ 7711 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7712 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7713 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7714 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */ 7715 .emit_fence = gfx_v9_0_ring_emit_fence_kiq, 7716 .test_ring = gfx_v9_0_ring_test_ring, 7717 .insert_nop = amdgpu_ring_insert_nop, 7718 .pad_ib = amdgpu_ring_generic_pad_ib, 7719 .emit_rreg = gfx_v9_0_ring_emit_rreg, 7720 .emit_wreg = gfx_v9_0_ring_emit_wreg, 7721 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait, 7722 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait, 7723 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush, 7724 }; 7725 7726 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev) 7727 { 7728 int i; 7729 7730 adev->gfx.kiq[0].ring.funcs = &gfx_v9_0_ring_funcs_kiq; 7731 7732 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7733 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx; 7734 7735 if (adev->gfx.mcbp && adev->gfx.num_gfx_rings) { 7736 for (i = 0; i < GFX9_NUM_SW_GFX_RINGS; i++) 7737 adev->gfx.sw_gfx_ring[i].funcs = &gfx_v9_0_sw_ring_funcs_gfx; 7738 } 7739 7740 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7741 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute; 7742 } 7743 7744 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = { 7745 .set = gfx_v9_0_set_eop_interrupt_state, 7746 .process = gfx_v9_0_eop_irq, 7747 }; 7748 7749 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = { 7750 .set = gfx_v9_0_set_priv_reg_fault_state, 7751 .process = gfx_v9_0_priv_reg_irq, 7752 }; 7753 7754 static const struct amdgpu_irq_src_funcs gfx_v9_0_bad_op_irq_funcs = { 7755 .set = gfx_v9_0_set_bad_op_fault_state, 7756 .process = gfx_v9_0_bad_op_irq, 7757 }; 7758 7759 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = { 7760 .set = gfx_v9_0_set_priv_inst_fault_state, 7761 .process = gfx_v9_0_priv_inst_irq, 7762 }; 7763 7764 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = { 7765 .set = gfx_v9_0_set_cp_ecc_error_state, 7766 .process = amdgpu_gfx_cp_ecc_error_irq, 7767 }; 7768 7769 7770 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev) 7771 { 7772 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7773 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs; 7774 7775 adev->gfx.priv_reg_irq.num_types = 1; 7776 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs; 7777 7778 adev->gfx.bad_op_irq.num_types = 1; 7779 adev->gfx.bad_op_irq.funcs = &gfx_v9_0_bad_op_irq_funcs; 7780 7781 adev->gfx.priv_inst_irq.num_types = 1; 7782 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs; 7783 7784 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/ 7785 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs; 7786 } 7787 7788 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev) 7789 { 7790 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7791 case IP_VERSION(9, 0, 1): 7792 case IP_VERSION(9, 2, 1): 7793 case IP_VERSION(9, 4, 0): 7794 case IP_VERSION(9, 2, 2): 7795 case IP_VERSION(9, 1, 0): 7796 case IP_VERSION(9, 4, 1): 7797 case IP_VERSION(9, 3, 0): 7798 case IP_VERSION(9, 4, 2): 7799 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs; 7800 break; 7801 default: 7802 break; 7803 } 7804 } 7805 7806 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev) 7807 { 7808 /* init asci gds info */ 7809 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7810 case IP_VERSION(9, 0, 1): 7811 case IP_VERSION(9, 2, 1): 7812 case IP_VERSION(9, 4, 0): 7813 adev->gds.gds_size = 0x10000; 7814 break; 7815 case IP_VERSION(9, 2, 2): 7816 case IP_VERSION(9, 1, 0): 7817 case IP_VERSION(9, 4, 1): 7818 adev->gds.gds_size = 0x1000; 7819 break; 7820 case IP_VERSION(9, 4, 2): 7821 /* aldebaran removed all the GDS internal memory, 7822 * only support GWS opcode in kernel, like barrier 7823 * semaphore.etc */ 7824 adev->gds.gds_size = 0; 7825 break; 7826 default: 7827 adev->gds.gds_size = 0x10000; 7828 break; 7829 } 7830 7831 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 7832 case IP_VERSION(9, 0, 1): 7833 case IP_VERSION(9, 4, 0): 7834 adev->gds.gds_compute_max_wave_id = 0x7ff; 7835 break; 7836 case IP_VERSION(9, 2, 1): 7837 adev->gds.gds_compute_max_wave_id = 0x27f; 7838 break; 7839 case IP_VERSION(9, 2, 2): 7840 case IP_VERSION(9, 1, 0): 7841 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 7842 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */ 7843 else 7844 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */ 7845 break; 7846 case IP_VERSION(9, 4, 1): 7847 adev->gds.gds_compute_max_wave_id = 0xfff; 7848 break; 7849 case IP_VERSION(9, 4, 2): 7850 /* deprecated for Aldebaran, no usage at all */ 7851 adev->gds.gds_compute_max_wave_id = 0; 7852 break; 7853 default: 7854 /* this really depends on the chip */ 7855 adev->gds.gds_compute_max_wave_id = 0x7ff; 7856 break; 7857 } 7858 7859 adev->gds.gws_size = 64; 7860 adev->gds.oa_size = 16; 7861 } 7862 7863 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev, 7864 u32 bitmap) 7865 { 7866 u32 data; 7867 7868 if (!bitmap) 7869 return; 7870 7871 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7872 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7873 7874 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data); 7875 } 7876 7877 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev) 7878 { 7879 u32 data, mask; 7880 7881 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG); 7882 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG); 7883 7884 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK; 7885 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT; 7886 7887 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 7888 7889 return (~data) & mask; 7890 } 7891 7892 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev, 7893 struct amdgpu_cu_info *cu_info) 7894 { 7895 int i, j, k, counter, active_cu_number = 0; 7896 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0; 7897 unsigned disable_masks[4 * 4]; 7898 7899 if (!adev || !cu_info) 7900 return -EINVAL; 7901 7902 /* 7903 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs 7904 */ 7905 if (adev->gfx.config.max_shader_engines * 7906 adev->gfx.config.max_sh_per_se > 16) 7907 return -EINVAL; 7908 7909 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 7910 adev->gfx.config.max_shader_engines, 7911 adev->gfx.config.max_sh_per_se); 7912 7913 mutex_lock(&adev->grbm_idx_mutex); 7914 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7915 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7916 mask = 1; 7917 ao_bitmap = 0; 7918 counter = 0; 7919 amdgpu_gfx_select_se_sh(adev, i, j, 0xffffffff, 0); 7920 gfx_v9_0_set_user_cu_inactive_bitmap( 7921 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]); 7922 bitmap = gfx_v9_0_get_cu_active_bitmap(adev); 7923 7924 /* 7925 * The bitmap(and ao_cu_bitmap) in cu_info structure is 7926 * 4x4 size array, and it's usually suitable for Vega 7927 * ASICs which has 4*2 SE/SH layout. 7928 * But for Arcturus, SE/SH layout is changed to 8*1. 7929 * To mostly reduce the impact, we make it compatible 7930 * with current bitmap array as below: 7931 * SE4,SH0 --> bitmap[0][1] 7932 * SE5,SH0 --> bitmap[1][1] 7933 * SE6,SH0 --> bitmap[2][1] 7934 * SE7,SH0 --> bitmap[3][1] 7935 */ 7936 cu_info->bitmap[0][i % 4][j + i / 4] = bitmap; 7937 7938 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) { 7939 if (bitmap & mask) { 7940 if (counter < adev->gfx.config.max_cu_per_sh) 7941 ao_bitmap |= mask; 7942 counter ++; 7943 } 7944 mask <<= 1; 7945 } 7946 active_cu_number += counter; 7947 if (i < 2 && j < 2) 7948 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8)); 7949 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap; 7950 } 7951 } 7952 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7953 mutex_unlock(&adev->grbm_idx_mutex); 7954 7955 cu_info->number = active_cu_number; 7956 cu_info->ao_cu_mask = ao_cu_mask; 7957 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7958 7959 return 0; 7960 } 7961 7962 const struct amdgpu_ip_block_version gfx_v9_0_ip_block = 7963 { 7964 .type = AMD_IP_BLOCK_TYPE_GFX, 7965 .major = 9, 7966 .minor = 0, 7967 .rev = 0, 7968 .funcs = &gfx_v9_0_ip_funcs, 7969 }; 7970