1 /* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/delay.h> 24 #include <linux/kernel.h> 25 #include <linux/firmware.h> 26 #include <linux/module.h> 27 #include <linux/pci.h> 28 #include "amdgpu.h" 29 #include "amdgpu_gfx.h" 30 #include "amdgpu_psp.h" 31 #include "amdgpu_smu.h" 32 #include "imu_v11_0.h" 33 #include "soc21.h" 34 #include "nvd.h" 35 36 #include "gc/gc_11_0_0_offset.h" 37 #include "gc/gc_11_0_0_sh_mask.h" 38 #include "smuio/smuio_13_0_6_offset.h" 39 #include "smuio/smuio_13_0_6_sh_mask.h" 40 #include "navi10_enum.h" 41 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h" 42 43 #include "soc15.h" 44 #include "clearstate_gfx11.h" 45 #include "v11_structs.h" 46 #include "gfx_v11_0.h" 47 #include "gfx_v11_0_cleaner_shader.h" 48 #include "gfx_v11_0_3.h" 49 #include "nbio_v4_3.h" 50 #include "mes_v11_0.h" 51 #include "mes_userqueue.h" 52 #include "amdgpu_userq_fence.h" 53 54 #define GFX11_NUM_GFX_RINGS 2 55 #define GFX11_MEC_HPD_SIZE 2048 56 57 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 58 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388 59 60 #define regCGTT_WD_CLK_CTRL 0x5086 61 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1 62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e 63 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1 64 #define regPC_CONFIG_CNTL_1 0x194d 65 #define regPC_CONFIG_CNTL_1_BASE_IDX 1 66 67 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0 0x0030 68 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0_BASE_IDX 1 69 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0 0x0031 70 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0_BASE_IDX 1 71 72 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100 73 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000 74 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000 75 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01 76 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000 77 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000 78 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000 79 80 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006 81 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 82 #define regCP_MQD_CONTROL_DEFAULT 0x00000100 83 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509 84 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 85 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000 86 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501 87 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000 88 89 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin"); 90 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin"); 91 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin"); 92 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin"); 93 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_kicker.bin"); 94 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin"); 95 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin"); 96 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin"); 97 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin"); 98 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin"); 99 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin"); 101 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin"); 102 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin"); 103 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin"); 104 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin"); 105 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin"); 106 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin"); 107 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin"); 109 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin"); 110 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin"); 112 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin"); 113 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin"); 114 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin"); 115 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin"); 116 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin"); 117 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin"); 118 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin"); 119 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin"); 120 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin"); 122 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin"); 124 MODULE_FIRMWARE("amdgpu/gc_11_5_3_pfp.bin"); 125 MODULE_FIRMWARE("amdgpu/gc_11_5_3_me.bin"); 126 MODULE_FIRMWARE("amdgpu/gc_11_5_3_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/gc_11_5_3_rlc.bin"); 128 MODULE_FIRMWARE("amdgpu/gc_11_5_4_pfp.bin"); 129 MODULE_FIRMWARE("amdgpu/gc_11_5_4_me.bin"); 130 MODULE_FIRMWARE("amdgpu/gc_11_5_4_mec.bin"); 131 MODULE_FIRMWARE("amdgpu/gc_11_5_4_rlc.bin"); 132 MODULE_FIRMWARE("amdgpu/gc_11_5_6_pfp.bin"); 133 MODULE_FIRMWARE("amdgpu/gc_11_5_6_me.bin"); 134 MODULE_FIRMWARE("amdgpu/gc_11_5_6_mec.bin"); 135 MODULE_FIRMWARE("amdgpu/gc_11_5_6_rlc.bin"); 136 MODULE_FIRMWARE("amdgpu/gc_11_7_0_pfp.bin"); 137 MODULE_FIRMWARE("amdgpu/gc_11_7_0_me.bin"); 138 MODULE_FIRMWARE("amdgpu/gc_11_7_0_mec.bin"); 139 MODULE_FIRMWARE("amdgpu/gc_11_7_0_rlc.bin"); 140 MODULE_FIRMWARE("amdgpu/gc_11_7_1_pfp.bin"); 141 MODULE_FIRMWARE("amdgpu/gc_11_7_1_me.bin"); 142 MODULE_FIRMWARE("amdgpu/gc_11_7_1_mec.bin"); 143 MODULE_FIRMWARE("amdgpu/gc_11_7_1_rlc.bin"); 144 145 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = { 146 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), 147 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2), 148 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3), 149 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1), 150 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2), 151 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3), 152 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1), 153 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1), 154 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT), 155 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT), 156 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT), 157 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2), 158 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2), 159 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS), 160 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR), 161 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0), 162 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE), 163 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 164 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR), 165 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE), 166 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR), 167 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR), 168 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE), 169 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR), 170 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR), 171 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 172 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ), 173 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 174 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 175 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 176 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO), 177 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI), 178 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ), 179 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS), 180 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS), 181 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS), 182 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT), 183 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT), 184 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS), 185 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2), 186 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS), 187 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS), 188 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS), 189 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES), 190 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS), 191 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS), 192 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL), 193 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS), 194 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG), 195 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL), 196 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL), 197 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR), 198 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR), 199 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR), 200 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR), 201 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR), 202 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS), 203 /* cp header registers */ 204 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 205 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 206 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 207 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 208 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 209 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 210 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 211 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 212 /* SE status registers */ 213 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0), 214 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1), 215 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2), 216 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3), 217 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4), 218 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5) 219 }; 220 221 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = { 222 /* compute registers */ 223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID), 224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE), 225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY), 226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY), 227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM), 228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE), 229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI), 230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR), 231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 232 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 233 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), 234 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL), 235 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR), 236 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI), 237 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR), 238 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL), 239 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST), 240 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR), 241 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI), 242 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL), 243 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR), 244 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR), 245 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS), 246 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO), 247 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI), 248 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL), 249 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET), 250 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE), 251 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET), 252 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE), 253 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE), 254 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR), 255 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM), 256 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO), 257 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI), 258 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET), 259 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT), 260 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET), 261 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS), 262 /* cp header registers */ 263 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 264 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 265 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 266 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 267 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 268 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 269 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 270 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 271 }; 272 273 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = { 274 /* gfx queue registers */ 275 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE), 276 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID), 277 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY), 278 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM), 279 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE), 280 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI), 281 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET), 282 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL), 283 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR), 284 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR), 285 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI), 286 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST), 287 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED), 288 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL), 289 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0), 290 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0), 291 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR), 292 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI), 293 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO), 294 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI), 295 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 296 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 297 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 298 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 299 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 300 /* cp header registers */ 301 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 302 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 303 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 304 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 305 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 306 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 307 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 308 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 309 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 310 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 311 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 312 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 313 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 314 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 315 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 316 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 317 }; 318 319 static const struct soc15_reg_golden golden_settings_gc_11_0[] = { 320 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000) 321 }; 322 323 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] = 324 { 325 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010), 326 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010), 327 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200), 328 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988), 329 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007), 330 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008), 331 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100), 332 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000), 333 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a) 334 }; 335 336 #define DEFAULT_SH_MEM_CONFIG \ 337 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \ 338 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \ 339 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT)) 340 341 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev); 342 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev); 343 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev); 344 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev); 345 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev); 346 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev); 347 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev); 348 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 349 struct amdgpu_cu_info *cu_info); 350 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev); 351 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 352 u32 sh_num, u32 instance, int xcc_id); 353 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev); 354 355 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume); 356 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); 357 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 358 uint32_t val); 359 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); 360 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 361 uint16_t pasid, uint32_t flush_type, 362 bool all_hub, uint8_t dst_sel); 363 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 364 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 365 static void gfx_v11_0_userq_priv_fault_work(struct work_struct *work); 366 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 367 bool enable); 368 369 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask) 370 { 371 struct amdgpu_device *adev = kiq_ring->adev; 372 u64 shader_mc_addr; 373 374 /* Cleaner shader MC address */ 375 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; 376 377 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 378 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | 379 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */ 380 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ 381 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 382 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 383 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */ 384 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */ 385 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 386 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 387 } 388 389 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring, 390 struct amdgpu_ring *ring) 391 { 392 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 393 uint64_t wptr_addr = ring->wptr_gpu_addr; 394 uint32_t me = 0, eng_sel = 0; 395 396 switch (ring->funcs->type) { 397 case AMDGPU_RING_TYPE_COMPUTE: 398 me = 1; 399 eng_sel = 0; 400 break; 401 case AMDGPU_RING_TYPE_GFX: 402 me = 0; 403 eng_sel = 4; 404 break; 405 case AMDGPU_RING_TYPE_MES: 406 me = 2; 407 eng_sel = 5; 408 break; 409 default: 410 WARN_ON(1); 411 } 412 413 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 414 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 415 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 416 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 417 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 418 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 419 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 420 PACKET3_MAP_QUEUES_ME((me)) | 421 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 422 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 423 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 424 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 425 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 426 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 427 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 428 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 429 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 430 } 431 432 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 433 struct amdgpu_ring *ring, 434 enum amdgpu_unmap_queues_action action, 435 u64 gpu_addr, u64 seq) 436 { 437 struct amdgpu_device *adev = kiq_ring->adev; 438 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 439 440 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) { 441 amdgpu_mes_unmap_legacy_queue(adev, ring, action, 442 gpu_addr, seq, 0); 443 return; 444 } 445 446 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 447 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 448 PACKET3_UNMAP_QUEUES_ACTION(action) | 449 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 450 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 451 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 452 amdgpu_ring_write(kiq_ring, 453 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 454 455 if (action == PREEMPT_QUEUES_NO_UNMAP) { 456 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 457 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 458 amdgpu_ring_write(kiq_ring, seq); 459 } else { 460 amdgpu_ring_write(kiq_ring, 0); 461 amdgpu_ring_write(kiq_ring, 0); 462 amdgpu_ring_write(kiq_ring, 0); 463 } 464 } 465 466 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring, 467 struct amdgpu_ring *ring, 468 u64 addr, 469 u64 seq) 470 { 471 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 472 473 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 474 amdgpu_ring_write(kiq_ring, 475 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 476 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 477 PACKET3_QUERY_STATUS_COMMAND(2)); 478 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 479 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 480 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 481 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 482 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 483 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 484 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 485 } 486 487 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 488 uint16_t pasid, uint32_t flush_type, 489 bool all_hub) 490 { 491 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1); 492 } 493 494 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = { 495 .kiq_set_resources = gfx11_kiq_set_resources, 496 .kiq_map_queues = gfx11_kiq_map_queues, 497 .kiq_unmap_queues = gfx11_kiq_unmap_queues, 498 .kiq_query_status = gfx11_kiq_query_status, 499 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs, 500 .set_resources_size = 8, 501 .map_queues_size = 7, 502 .unmap_queues_size = 6, 503 .query_status_size = 7, 504 .invalidate_tlbs_size = 2, 505 }; 506 507 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 508 { 509 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs; 510 } 511 512 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev) 513 { 514 if (amdgpu_sriov_vf(adev)) 515 return; 516 517 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 518 case IP_VERSION(11, 0, 1): 519 case IP_VERSION(11, 0, 4): 520 soc15_program_register_sequence(adev, 521 golden_settings_gc_11_0_1, 522 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1)); 523 break; 524 default: 525 break; 526 } 527 soc15_program_register_sequence(adev, 528 golden_settings_gc_11_0, 529 (const u32)ARRAY_SIZE(golden_settings_gc_11_0)); 530 531 } 532 533 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 534 bool wc, uint32_t reg, uint32_t val) 535 { 536 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 537 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 538 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0)); 539 amdgpu_ring_write(ring, reg); 540 amdgpu_ring_write(ring, 0); 541 amdgpu_ring_write(ring, val); 542 } 543 544 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 545 int mem_space, int opt, uint32_t addr0, 546 uint32_t addr1, uint32_t ref, uint32_t mask, 547 uint32_t inv) 548 { 549 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 550 amdgpu_ring_write(ring, 551 /* memory (1) or register (0) */ 552 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 553 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 554 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 555 WAIT_REG_MEM_ENGINE(eng_sel))); 556 557 if (mem_space) 558 WARN_ON(addr0 & 0x3); /* Dword align */ 559 amdgpu_ring_write(ring, addr0); 560 amdgpu_ring_write(ring, addr1); 561 amdgpu_ring_write(ring, ref); 562 amdgpu_ring_write(ring, mask); 563 amdgpu_ring_write(ring, inv); /* poll interval */ 564 } 565 566 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop) 567 { 568 /* Header itself is a NOP packet */ 569 if (num_nop == 1) { 570 amdgpu_ring_write(ring, ring->funcs->nop); 571 return; 572 } 573 574 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 575 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); 576 577 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ 578 amdgpu_ring_insert_nop(ring, num_nop - 1); 579 } 580 581 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring) 582 { 583 struct amdgpu_device *adev = ring->adev; 584 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 585 uint32_t tmp = 0; 586 unsigned i; 587 int r; 588 589 WREG32(scratch, 0xCAFEDEAD); 590 r = amdgpu_ring_alloc(ring, 5); 591 if (r) { 592 drm_err(adev_to_drm(adev), "cp failed to lock ring %d (%d).\n", 593 ring->idx, r); 594 return r; 595 } 596 597 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) { 598 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF); 599 } else { 600 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 601 amdgpu_ring_write(ring, scratch - 602 PACKET3_SET_UCONFIG_REG_START); 603 amdgpu_ring_write(ring, 0xDEADBEEF); 604 } 605 amdgpu_ring_commit(ring); 606 607 for (i = 0; i < adev->usec_timeout; i++) { 608 tmp = RREG32(scratch); 609 if (tmp == 0xDEADBEEF) 610 break; 611 if (amdgpu_emu_mode == 1) 612 msleep(1); 613 else 614 udelay(1); 615 } 616 617 if (i >= adev->usec_timeout) 618 r = -ETIMEDOUT; 619 return r; 620 } 621 622 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 623 { 624 struct amdgpu_device *adev = ring->adev; 625 struct amdgpu_ib ib; 626 struct dma_fence *f = NULL; 627 unsigned index; 628 uint64_t gpu_addr; 629 uint32_t *cpu_ptr; 630 long r; 631 632 /* MES KIQ fw hasn't indirect buffer support for now */ 633 if (adev->enable_mes_kiq && 634 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 635 return 0; 636 637 memset(&ib, 0, sizeof(ib)); 638 639 r = amdgpu_wb_get(adev, &index); 640 if (r) 641 return r; 642 643 gpu_addr = adev->wb.gpu_addr + (index * 4); 644 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 645 cpu_ptr = &adev->wb.wb[index]; 646 647 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 648 if (r) { 649 drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r); 650 goto err1; 651 } 652 653 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 654 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 655 ib.ptr[2] = lower_32_bits(gpu_addr); 656 ib.ptr[3] = upper_32_bits(gpu_addr); 657 ib.ptr[4] = 0xDEADBEEF; 658 ib.length_dw = 5; 659 660 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 661 if (r) 662 goto err2; 663 664 r = dma_fence_wait_timeout(f, false, timeout); 665 if (r == 0) { 666 r = -ETIMEDOUT; 667 goto err2; 668 } else if (r < 0) { 669 goto err2; 670 } 671 672 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF) 673 r = 0; 674 else 675 r = -EINVAL; 676 err2: 677 amdgpu_ib_free(&ib, NULL); 678 dma_fence_put(f); 679 err1: 680 amdgpu_wb_free(adev, index); 681 return r; 682 } 683 684 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev) 685 { 686 amdgpu_ucode_release(&adev->gfx.pfp_fw); 687 amdgpu_ucode_release(&adev->gfx.me_fw); 688 amdgpu_ucode_release(&adev->gfx.rlc_fw); 689 amdgpu_ucode_release(&adev->gfx.mec_fw); 690 691 kfree(adev->gfx.rlc.register_list_format); 692 } 693 694 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix) 695 { 696 const struct psp_firmware_header_v1_0 *toc_hdr; 697 int err = 0; 698 699 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, 700 AMDGPU_UCODE_REQUIRED, 701 "amdgpu/%s_toc.bin", ucode_prefix); 702 if (err) 703 goto out; 704 705 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data; 706 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version); 707 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version); 708 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes); 709 adev->psp.toc.start_addr = (uint8_t *)toc_hdr + 710 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes); 711 return 0; 712 out: 713 amdgpu_ucode_release(&adev->psp.toc_fw); 714 return err; 715 } 716 717 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev) 718 { 719 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 720 case IP_VERSION(11, 0, 0): 721 case IP_VERSION(11, 0, 2): 722 case IP_VERSION(11, 0, 3): 723 if ((adev->gfx.me_fw_version >= 1505) && 724 (adev->gfx.pfp_fw_version >= 1600) && 725 (adev->gfx.mec_fw_version >= 512)) { 726 if (amdgpu_sriov_vf(adev)) 727 adev->gfx.cp_gfx_shadow = true; 728 else 729 adev->gfx.cp_gfx_shadow = false; 730 } 731 break; 732 default: 733 adev->gfx.cp_gfx_shadow = false; 734 break; 735 } 736 } 737 738 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) 739 { 740 char ucode_prefix[25]; 741 int err; 742 const struct rlc_firmware_header_v2_0 *rlc_hdr; 743 uint16_t version_major; 744 uint16_t version_minor; 745 746 DRM_DEBUG("\n"); 747 748 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 749 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 750 AMDGPU_UCODE_REQUIRED, 751 "amdgpu/%s_pfp.bin", ucode_prefix); 752 if (err) 753 goto out; 754 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/ 755 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version( 756 (union amdgpu_firmware_header *) 757 adev->gfx.pfp_fw->data, 2, 0); 758 if (adev->gfx.rs64_enable) { 759 dev_info(adev->dev, "CP RS64 enable\n"); 760 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP); 761 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK); 762 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK); 763 } else { 764 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 765 } 766 767 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 768 AMDGPU_UCODE_REQUIRED, 769 "amdgpu/%s_me.bin", ucode_prefix); 770 if (err) 771 goto out; 772 if (adev->gfx.rs64_enable) { 773 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME); 774 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK); 775 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK); 776 } else { 777 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 778 } 779 780 if (!amdgpu_sriov_vf(adev)) { 781 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) && 782 adev->pdev->revision == 0xCE) 783 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 784 AMDGPU_UCODE_REQUIRED, 785 "amdgpu/gc_11_0_0_rlc_1.bin"); 786 else if (amdgpu_is_kicker_fw(adev)) 787 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 788 AMDGPU_UCODE_REQUIRED, 789 "amdgpu/%s_rlc_kicker.bin", ucode_prefix); 790 else 791 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 792 AMDGPU_UCODE_REQUIRED, 793 "amdgpu/%s_rlc.bin", ucode_prefix); 794 if (err) 795 goto out; 796 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 797 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 798 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 799 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 800 if (err) 801 goto out; 802 } 803 804 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 805 AMDGPU_UCODE_REQUIRED, 806 "amdgpu/%s_mec.bin", ucode_prefix); 807 if (err) 808 goto out; 809 if (adev->gfx.rs64_enable) { 810 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC); 811 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK); 812 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK); 813 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK); 814 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK); 815 } else { 816 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 817 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 818 } 819 820 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 821 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix); 822 823 /* only one MEC for gfx 11.0.0. */ 824 adev->gfx.mec2_fw = NULL; 825 826 gfx_v11_0_check_fw_cp_gfx_shadow(adev); 827 828 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) { 829 err = adev->gfx.imu.funcs->init_microcode(adev); 830 if (err) 831 DRM_ERROR("Failed to init imu firmware!\n"); 832 return err; 833 } 834 835 out: 836 if (err) { 837 amdgpu_ucode_release(&adev->gfx.pfp_fw); 838 amdgpu_ucode_release(&adev->gfx.me_fw); 839 amdgpu_ucode_release(&adev->gfx.rlc_fw); 840 amdgpu_ucode_release(&adev->gfx.mec_fw); 841 } 842 843 return err; 844 } 845 846 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev) 847 { 848 u32 count = 0; 849 const struct cs_section_def *sect = NULL; 850 const struct cs_extent_def *ext = NULL; 851 852 /* begin clear state */ 853 count += 2; 854 /* context control state */ 855 count += 3; 856 857 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 858 for (ext = sect->section; ext->extent != NULL; ++ext) { 859 if (sect->id == SECT_CONTEXT) 860 count += 2 + ext->reg_count; 861 else 862 return 0; 863 } 864 } 865 866 /* set PA_SC_TILE_STEERING_OVERRIDE */ 867 count += 3; 868 /* end clear state */ 869 count += 2; 870 /* clear state */ 871 count += 2; 872 873 return count; 874 } 875 876 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 877 { 878 u32 count = 0; 879 int ctx_reg_offset; 880 881 if (adev->gfx.rlc.cs_data == NULL) 882 return; 883 if (buffer == NULL) 884 return; 885 886 count = amdgpu_gfx_csb_preamble_start(buffer); 887 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 888 889 ctx_reg_offset = SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 890 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 891 buffer[count++] = cpu_to_le32(ctx_reg_offset); 892 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override); 893 894 amdgpu_gfx_csb_preamble_end(buffer, count); 895 } 896 897 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev) 898 { 899 /* clear state block */ 900 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 901 &adev->gfx.rlc.clear_state_gpu_addr, 902 (void **)&adev->gfx.rlc.cs_ptr); 903 904 /* jump table block */ 905 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 906 &adev->gfx.rlc.cp_table_gpu_addr, 907 (void **)&adev->gfx.rlc.cp_table_ptr); 908 } 909 910 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 911 { 912 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 913 914 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 915 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 916 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1); 917 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2); 918 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3); 919 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL); 920 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX); 921 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0); 922 adev->gfx.rlc.rlcg_reg_access_supported = true; 923 } 924 925 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev) 926 { 927 const struct cs_section_def *cs_data; 928 int r; 929 930 adev->gfx.rlc.cs_data = gfx11_cs_data; 931 932 cs_data = adev->gfx.rlc.cs_data; 933 934 if (cs_data) { 935 /* init clear state block */ 936 r = amdgpu_gfx_rlc_init_csb(adev); 937 if (r) 938 return r; 939 } 940 941 /* init spm vmid with 0xf */ 942 if (adev->gfx.rlc.funcs->update_spm_vmid) 943 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf); 944 945 return 0; 946 } 947 948 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev) 949 { 950 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 951 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 952 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL); 953 } 954 955 static void gfx_v11_0_me_init(struct amdgpu_device *adev) 956 { 957 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 958 959 amdgpu_gfx_graphics_queue_acquire(adev); 960 } 961 962 static int gfx_v11_0_mec_init(struct amdgpu_device *adev) 963 { 964 int r; 965 u32 *hpd; 966 size_t mec_hpd_size; 967 968 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 969 970 /* take ownership of the relevant compute queues */ 971 amdgpu_gfx_compute_queue_acquire(adev); 972 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE; 973 974 if (mec_hpd_size) { 975 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 976 AMDGPU_GEM_DOMAIN_GTT, 977 &adev->gfx.mec.hpd_eop_obj, 978 &adev->gfx.mec.hpd_eop_gpu_addr, 979 (void **)&hpd); 980 if (r) { 981 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 982 gfx_v11_0_mec_fini(adev); 983 return r; 984 } 985 986 memset(hpd, 0, mec_hpd_size); 987 988 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 989 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 990 } 991 992 return 0; 993 } 994 995 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address) 996 { 997 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 998 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 999 (address << SQ_IND_INDEX__INDEX__SHIFT)); 1000 return RREG32_SOC15(GC, 0, regSQ_IND_DATA); 1001 } 1002 1003 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave, 1004 uint32_t thread, uint32_t regno, 1005 uint32_t num, uint32_t *out) 1006 { 1007 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 1008 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1009 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1010 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) | 1011 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1012 while (num--) 1013 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA); 1014 } 1015 1016 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1017 { 1018 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE 1019 * field when performing a select_se_sh so it should be 1020 * zero here */ 1021 WARN_ON(simd != 0); 1022 1023 /* type 3 wave data */ 1024 dst[(*no_fields)++] = 3; 1025 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS); 1026 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO); 1027 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI); 1028 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO); 1029 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI); 1030 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1); 1031 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2); 1032 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC); 1033 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC); 1034 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS); 1035 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS); 1036 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2); 1037 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1); 1038 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0); 1039 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE); 1040 } 1041 1042 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1043 uint32_t wave, uint32_t start, 1044 uint32_t size, uint32_t *dst) 1045 { 1046 WARN_ON(simd != 0); 1047 1048 wave_read_regs( 1049 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size, 1050 dst); 1051 } 1052 1053 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1054 uint32_t wave, uint32_t thread, 1055 uint32_t start, uint32_t size, 1056 uint32_t *dst) 1057 { 1058 wave_read_regs( 1059 adev, wave, thread, 1060 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1061 } 1062 1063 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev, 1064 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1065 { 1066 soc21_grbm_select(adev, me, pipe, q, vm); 1067 } 1068 1069 /* all sizes are in bytes */ 1070 #define MQD_SHADOW_BASE_SIZE 73728 1071 #define MQD_SHADOW_BASE_ALIGNMENT 256 1072 #define MQD_FWWORKAREA_SIZE 484 1073 #define MQD_FWWORKAREA_ALIGNMENT 256 1074 1075 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev, 1076 struct amdgpu_gfx_shadow_info *shadow_info) 1077 { 1078 /* for gfx */ 1079 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE; 1080 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT; 1081 shadow_info->csa_size = MQD_FWWORKAREA_SIZE; 1082 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT; 1083 /* for compute */ 1084 shadow_info->eop_size = GFX11_MEC_HPD_SIZE; 1085 shadow_info->eop_alignment = 256; 1086 } 1087 1088 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev, 1089 struct amdgpu_gfx_shadow_info *shadow_info, 1090 bool skip_check) 1091 { 1092 if (adev->gfx.cp_gfx_shadow || skip_check) { 1093 gfx_v11_0_get_gfx_shadow_info_nocheck(adev, shadow_info); 1094 return 0; 1095 } else { 1096 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info)); 1097 return -ENOTSUPP; 1098 } 1099 } 1100 1101 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = { 1102 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter, 1103 .select_se_sh = &gfx_v11_0_select_se_sh, 1104 .read_wave_data = &gfx_v11_0_read_wave_data, 1105 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs, 1106 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs, 1107 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q, 1108 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk, 1109 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info, 1110 .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask, 1111 }; 1112 1113 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) 1114 { 1115 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1116 case IP_VERSION(11, 0, 0): 1117 case IP_VERSION(11, 0, 2): 1118 adev->gfx.config.max_hw_contexts = 8; 1119 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1120 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1121 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1122 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1123 break; 1124 case IP_VERSION(11, 0, 3): 1125 adev->gfx.ras = &gfx_v11_0_3_ras; 1126 adev->gfx.config.max_hw_contexts = 8; 1127 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1128 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1129 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1130 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1131 break; 1132 case IP_VERSION(11, 0, 1): 1133 case IP_VERSION(11, 0, 4): 1134 case IP_VERSION(11, 5, 0): 1135 case IP_VERSION(11, 5, 1): 1136 case IP_VERSION(11, 5, 2): 1137 case IP_VERSION(11, 5, 3): 1138 case IP_VERSION(11, 5, 4): 1139 case IP_VERSION(11, 5, 6): 1140 case IP_VERSION(11, 7, 0): 1141 case IP_VERSION(11, 7, 1): 1142 adev->gfx.config.max_hw_contexts = 8; 1143 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1144 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1145 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 1146 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300; 1147 break; 1148 default: 1149 dev_warn(adev->dev, "Unsupported GC version 0x%08x\n", 1150 amdgpu_ip_version(adev, GC_HWIP, 0)); 1151 return -EINVAL; 1152 } 1153 1154 return 0; 1155 } 1156 1157 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id, 1158 int me, int pipe, int queue) 1159 { 1160 struct amdgpu_ring *ring; 1161 unsigned int irq_type; 1162 unsigned int hw_prio; 1163 1164 ring = &adev->gfx.gfx_ring[ring_id]; 1165 1166 ring->me = me; 1167 ring->pipe = pipe; 1168 ring->queue = queue; 1169 1170 ring->ring_obj = NULL; 1171 ring->use_doorbell = true; 1172 if (adev->gfx.disable_kq) { 1173 ring->no_scheduler = true; 1174 ring->no_user_submission = true; 1175 } 1176 1177 if (!ring_id) 1178 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 1179 else 1180 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1; 1181 ring->vm_hub = AMDGPU_GFXHUB(0); 1182 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1183 1184 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe; 1185 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ? 1186 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1187 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1188 hw_prio, NULL); 1189 } 1190 1191 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1192 int mec, int pipe, int queue) 1193 { 1194 int r; 1195 unsigned irq_type; 1196 struct amdgpu_ring *ring; 1197 unsigned int hw_prio; 1198 1199 ring = &adev->gfx.compute_ring[ring_id]; 1200 1201 /* mec0 is me1 */ 1202 ring->me = mec + 1; 1203 ring->pipe = pipe; 1204 ring->queue = queue; 1205 1206 ring->ring_obj = NULL; 1207 ring->use_doorbell = true; 1208 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 1209 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1210 + (ring_id * GFX11_MEC_HPD_SIZE); 1211 ring->vm_hub = AMDGPU_GFXHUB(0); 1212 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1213 1214 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1215 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1216 + ring->pipe; 1217 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1218 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1219 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1220 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1221 hw_prio, NULL); 1222 if (r) 1223 return r; 1224 1225 return 0; 1226 } 1227 1228 static struct { 1229 SOC21_FIRMWARE_ID id; 1230 unsigned int offset; 1231 unsigned int size; 1232 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX]; 1233 1234 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc) 1235 { 1236 RLC_TABLE_OF_CONTENT *ucode = rlc_toc; 1237 1238 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) && 1239 (ucode->id < SOC21_FIRMWARE_ID_MAX)) { 1240 rlc_autoload_info[ucode->id].id = ucode->id; 1241 rlc_autoload_info[ucode->id].offset = ucode->offset * 4; 1242 rlc_autoload_info[ucode->id].size = ucode->size * 4; 1243 1244 ucode++; 1245 } 1246 } 1247 1248 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev) 1249 { 1250 uint32_t total_size = 0; 1251 SOC21_FIRMWARE_ID id; 1252 1253 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr); 1254 1255 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++) 1256 total_size += rlc_autoload_info[id].size; 1257 1258 /* In case the offset in rlc toc ucode is aligned */ 1259 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset) 1260 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset + 1261 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size; 1262 1263 return total_size; 1264 } 1265 1266 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev) 1267 { 1268 int r; 1269 uint32_t total_size; 1270 1271 total_size = gfx_v11_0_calc_toc_total_size(adev); 1272 1273 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024, 1274 AMDGPU_GEM_DOMAIN_VRAM | 1275 AMDGPU_GEM_DOMAIN_GTT, 1276 &adev->gfx.rlc.rlc_autoload_bo, 1277 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1278 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1279 1280 if (r) { 1281 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r); 1282 return r; 1283 } 1284 1285 return 0; 1286 } 1287 1288 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev, 1289 SOC21_FIRMWARE_ID id, 1290 const void *fw_data, 1291 uint32_t fw_size, 1292 uint32_t *fw_autoload_mask) 1293 { 1294 uint32_t toc_offset; 1295 uint32_t toc_fw_size; 1296 char *ptr = adev->gfx.rlc.rlc_autoload_ptr; 1297 1298 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX) 1299 return; 1300 1301 toc_offset = rlc_autoload_info[id].offset; 1302 toc_fw_size = rlc_autoload_info[id].size; 1303 1304 if (fw_size == 0) 1305 fw_size = toc_fw_size; 1306 1307 if (fw_size > toc_fw_size) 1308 fw_size = toc_fw_size; 1309 1310 memcpy(ptr + toc_offset, fw_data, fw_size); 1311 1312 if (fw_size < toc_fw_size) 1313 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); 1314 1315 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME)) 1316 *(uint64_t *)fw_autoload_mask |= 1ULL << id; 1317 } 1318 1319 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev, 1320 uint32_t *fw_autoload_mask) 1321 { 1322 void *data; 1323 uint32_t size; 1324 uint64_t *toc_ptr; 1325 1326 *(uint64_t *)fw_autoload_mask |= 0x1; 1327 1328 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask); 1329 1330 data = adev->psp.toc.start_addr; 1331 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size; 1332 1333 toc_ptr = (uint64_t *)data + size / 8 - 1; 1334 *toc_ptr = *(uint64_t *)fw_autoload_mask; 1335 1336 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC, 1337 data, size, fw_autoload_mask); 1338 } 1339 1340 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev, 1341 uint32_t *fw_autoload_mask) 1342 { 1343 const __le32 *fw_data; 1344 uint32_t fw_size; 1345 const struct gfx_firmware_header_v1_0 *cp_hdr; 1346 const struct gfx_firmware_header_v2_0 *cpv2_hdr; 1347 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1348 const struct rlc_firmware_header_v2_2 *rlcv22_hdr; 1349 uint16_t version_major, version_minor; 1350 1351 if (adev->gfx.rs64_enable) { 1352 /* pfp ucode */ 1353 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1354 adev->gfx.pfp_fw->data; 1355 /* instruction */ 1356 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1357 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1358 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1359 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP, 1360 fw_data, fw_size, fw_autoload_mask); 1361 /* data */ 1362 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1363 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1364 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1365 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK, 1366 fw_data, fw_size, fw_autoload_mask); 1367 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK, 1368 fw_data, fw_size, fw_autoload_mask); 1369 /* me ucode */ 1370 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1371 adev->gfx.me_fw->data; 1372 /* instruction */ 1373 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1374 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1375 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1376 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME, 1377 fw_data, fw_size, fw_autoload_mask); 1378 /* data */ 1379 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1380 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1381 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1382 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK, 1383 fw_data, fw_size, fw_autoload_mask); 1384 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK, 1385 fw_data, fw_size, fw_autoload_mask); 1386 /* mec ucode */ 1387 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1388 adev->gfx.mec_fw->data; 1389 /* instruction */ 1390 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1391 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1392 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1393 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC, 1394 fw_data, fw_size, fw_autoload_mask); 1395 /* data */ 1396 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1397 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1398 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1399 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK, 1400 fw_data, fw_size, fw_autoload_mask); 1401 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK, 1402 fw_data, fw_size, fw_autoload_mask); 1403 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK, 1404 fw_data, fw_size, fw_autoload_mask); 1405 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK, 1406 fw_data, fw_size, fw_autoload_mask); 1407 } else { 1408 /* pfp ucode */ 1409 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1410 adev->gfx.pfp_fw->data; 1411 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1412 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1413 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1414 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP, 1415 fw_data, fw_size, fw_autoload_mask); 1416 1417 /* me ucode */ 1418 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1419 adev->gfx.me_fw->data; 1420 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1421 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1422 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1423 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME, 1424 fw_data, fw_size, fw_autoload_mask); 1425 1426 /* mec ucode */ 1427 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1428 adev->gfx.mec_fw->data; 1429 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1430 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1431 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1432 cp_hdr->jt_size * 4; 1433 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC, 1434 fw_data, fw_size, fw_autoload_mask); 1435 } 1436 1437 /* rlc ucode */ 1438 rlc_hdr = (const struct rlc_firmware_header_v2_0 *) 1439 adev->gfx.rlc_fw->data; 1440 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1441 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes)); 1442 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes); 1443 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE, 1444 fw_data, fw_size, fw_autoload_mask); 1445 1446 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1447 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1448 if (version_major == 2) { 1449 if (version_minor >= 2) { 1450 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 1451 1452 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1453 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes)); 1454 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes); 1455 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE, 1456 fw_data, fw_size, fw_autoload_mask); 1457 1458 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1459 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes)); 1460 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes); 1461 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT, 1462 fw_data, fw_size, fw_autoload_mask); 1463 } 1464 } 1465 } 1466 1467 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev, 1468 uint32_t *fw_autoload_mask) 1469 { 1470 const __le32 *fw_data; 1471 uint32_t fw_size; 1472 const struct sdma_firmware_header_v2_0 *sdma_hdr; 1473 1474 sdma_hdr = (const struct sdma_firmware_header_v2_0 *) 1475 adev->sdma.instance[0].fw->data; 1476 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1477 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes)); 1478 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes); 1479 1480 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1481 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask); 1482 1483 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1484 le32_to_cpu(sdma_hdr->ctl_ucode_offset)); 1485 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes); 1486 1487 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1488 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask); 1489 } 1490 1491 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev, 1492 uint32_t *fw_autoload_mask) 1493 { 1494 const __le32 *fw_data; 1495 unsigned fw_size; 1496 const struct mes_firmware_header_v1_0 *mes_hdr; 1497 int pipe, ucode_id, data_id; 1498 1499 for (pipe = 0; pipe < 2; pipe++) { 1500 if (pipe==0) { 1501 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0; 1502 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK; 1503 } else { 1504 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1; 1505 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK; 1506 } 1507 1508 mes_hdr = (const struct mes_firmware_header_v1_0 *) 1509 adev->mes.fw[pipe]->data; 1510 1511 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1512 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); 1513 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); 1514 1515 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1516 ucode_id, fw_data, fw_size, fw_autoload_mask); 1517 1518 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1519 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); 1520 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); 1521 1522 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1523 data_id, fw_data, fw_size, fw_autoload_mask); 1524 } 1525 } 1526 1527 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) 1528 { 1529 uint32_t rlc_g_offset, rlc_g_size; 1530 uint64_t gpu_addr; 1531 uint32_t autoload_fw_id[2]; 1532 1533 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2); 1534 1535 /* RLC autoload sequence 2: copy ucode */ 1536 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id); 1537 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id); 1538 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id); 1539 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id); 1540 1541 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset; 1542 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size; 1543 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset; 1544 1545 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr)); 1546 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr)); 1547 1548 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size); 1549 1550 /* RLC autoload sequence 3: load IMU fw */ 1551 if (adev->gfx.imu.funcs->load_microcode) 1552 adev->gfx.imu.funcs->load_microcode(adev); 1553 /* RLC autoload sequence 4 init IMU fw */ 1554 if (adev->gfx.imu.funcs->setup_imu) 1555 adev->gfx.imu.funcs->setup_imu(adev); 1556 if (adev->gfx.imu.funcs->start_imu) 1557 adev->gfx.imu.funcs->start_imu(adev); 1558 1559 /* RLC autoload sequence 5 disable gpa mode */ 1560 gfx_v11_0_disable_gpa_mode(adev); 1561 1562 return 0; 1563 } 1564 1565 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev) 1566 { 1567 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 1568 uint32_t *ptr; 1569 uint32_t inst; 1570 1571 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL); 1572 if (!ptr) { 1573 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 1574 adev->gfx.ip_dump_core = NULL; 1575 } else { 1576 adev->gfx.ip_dump_core = ptr; 1577 } 1578 1579 /* Allocate memory for compute queue registers for all the instances */ 1580 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 1581 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1582 adev->gfx.mec.num_queue_per_pipe; 1583 1584 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1585 if (!ptr) { 1586 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 1587 adev->gfx.ip_dump_compute_queues = NULL; 1588 } else { 1589 adev->gfx.ip_dump_compute_queues = ptr; 1590 } 1591 1592 /* Allocate memory for gfx queue registers for all the instances */ 1593 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 1594 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me * 1595 adev->gfx.me.num_queue_per_pipe; 1596 1597 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1598 if (!ptr) { 1599 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n"); 1600 adev->gfx.ip_dump_gfx_queues = NULL; 1601 } else { 1602 adev->gfx.ip_dump_gfx_queues = ptr; 1603 } 1604 } 1605 1606 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) 1607 { 1608 int i, j, k, r, ring_id; 1609 int xcc_id = 0; 1610 struct amdgpu_device *adev = ip_block->adev; 1611 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */ 1612 1613 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler); 1614 1615 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1616 case IP_VERSION(11, 0, 0): 1617 case IP_VERSION(11, 0, 1): 1618 case IP_VERSION(11, 0, 2): 1619 case IP_VERSION(11, 0, 3): 1620 case IP_VERSION(11, 0, 4): 1621 case IP_VERSION(11, 5, 0): 1622 case IP_VERSION(11, 5, 1): 1623 case IP_VERSION(11, 5, 2): 1624 case IP_VERSION(11, 5, 3): 1625 case IP_VERSION(11, 5, 4): 1626 case IP_VERSION(11, 5, 6): 1627 case IP_VERSION(11, 7, 0): 1628 case IP_VERSION(11, 7, 1): 1629 adev->gfx.me.num_me = 1; 1630 if (adev->gfx.rs64_enable) 1631 adev->gfx.me.num_pipe_per_me = 1; 1632 else 1633 adev->gfx.me.num_pipe_per_me = 2; 1634 adev->gfx.me.num_queue_per_pipe = 2; 1635 adev->gfx.mec.num_mec = 1; 1636 adev->gfx.mec.num_pipe_per_mec = 4; 1637 adev->gfx.mec.num_queue_per_pipe = 4; 1638 break; 1639 default: 1640 adev->gfx.me.num_me = 1; 1641 adev->gfx.me.num_pipe_per_me = 1; 1642 adev->gfx.me.num_queue_per_pipe = 1; 1643 adev->gfx.mec.num_mec = 1; 1644 adev->gfx.mec.num_pipe_per_mec = 4; 1645 adev->gfx.mec.num_queue_per_pipe = 8; 1646 break; 1647 } 1648 1649 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1650 case IP_VERSION(11, 0, 0): 1651 case IP_VERSION(11, 0, 2): 1652 case IP_VERSION(11, 0, 3): 1653 if (!adev->gfx.disable_uq && 1654 adev->gfx.me_fw_version >= 3090 && 1655 adev->gfx.pfp_fw_version >= 3190 && 1656 adev->gfx.mec_fw_version >= 3450 && 1657 adev->mes.fw_version[0] >= 147) { 1658 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1659 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1660 } 1661 break; 1662 case IP_VERSION(11, 0, 1): 1663 case IP_VERSION(11, 0, 4): 1664 case IP_VERSION(11, 5, 0): 1665 case IP_VERSION(11, 5, 1): 1666 case IP_VERSION(11, 5, 2): 1667 case IP_VERSION(11, 5, 3): 1668 /* add firmware version checks here */ 1669 if (0 && !adev->gfx.disable_uq) { 1670 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1671 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1672 } 1673 break; 1674 default: 1675 break; 1676 } 1677 1678 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1679 case IP_VERSION(11, 0, 0): 1680 case IP_VERSION(11, 0, 2): 1681 case IP_VERSION(11, 0, 3): 1682 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1683 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1684 if (adev->gfx.me_fw_version >= 2280 && 1685 adev->gfx.pfp_fw_version >= 2370 && 1686 adev->gfx.mec_fw_version >= 2450 && 1687 adev->mes.fw_version[0] >= 99) { 1688 adev->gfx.enable_cleaner_shader = true; 1689 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1690 if (r) { 1691 adev->gfx.enable_cleaner_shader = false; 1692 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1693 } 1694 } 1695 break; 1696 case IP_VERSION(11, 0, 1): 1697 case IP_VERSION(11, 0, 4): 1698 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1699 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1700 if (adev->gfx.pfp_fw_version >= 102 && 1701 adev->gfx.mec_fw_version >= 66 && 1702 adev->mes.fw_version[0] >= 128) { 1703 adev->gfx.enable_cleaner_shader = true; 1704 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1705 if (r) { 1706 adev->gfx.enable_cleaner_shader = false; 1707 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1708 } 1709 } 1710 break; 1711 case IP_VERSION(11, 5, 0): 1712 case IP_VERSION(11, 5, 1): 1713 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1714 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1715 if (adev->gfx.mec_fw_version >= 26 && 1716 adev->mes.fw_version[0] >= 114) { 1717 adev->gfx.enable_cleaner_shader = true; 1718 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1719 if (r) { 1720 adev->gfx.enable_cleaner_shader = false; 1721 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1722 } 1723 } 1724 break; 1725 case IP_VERSION(11, 5, 2): 1726 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1727 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1728 if (adev->gfx.me_fw_version >= 12 && 1729 adev->gfx.pfp_fw_version >= 15 && 1730 adev->gfx.mec_fw_version >= 15) { 1731 adev->gfx.enable_cleaner_shader = true; 1732 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1733 if (r) { 1734 adev->gfx.enable_cleaner_shader = false; 1735 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1736 } 1737 } 1738 break; 1739 case IP_VERSION(11, 5, 3): 1740 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1741 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1742 if (adev->gfx.me_fw_version >= 7 && 1743 adev->gfx.pfp_fw_version >= 8 && 1744 adev->gfx.mec_fw_version >= 8) { 1745 adev->gfx.enable_cleaner_shader = true; 1746 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1747 if (r) { 1748 adev->gfx.enable_cleaner_shader = false; 1749 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1750 } 1751 } 1752 break; 1753 case IP_VERSION(11, 5, 4): 1754 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1755 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1756 if (adev->gfx.me_fw_version >= 4 && 1757 adev->gfx.pfp_fw_version >= 7 && 1758 adev->gfx.mec_fw_version >= 5) { 1759 adev->gfx.enable_cleaner_shader = true; 1760 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1761 if (r) { 1762 adev->gfx.enable_cleaner_shader = false; 1763 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1764 } 1765 } 1766 break; 1767 default: 1768 adev->gfx.enable_cleaner_shader = false; 1769 break; 1770 } 1771 1772 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */ 1773 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) && 1774 amdgpu_sriov_is_pp_one_vf(adev)) 1775 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG; 1776 1777 /* EOP Event */ 1778 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1779 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT, 1780 &adev->gfx.eop_irq); 1781 if (r) 1782 return r; 1783 1784 /* Bad opcode Event */ 1785 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1786 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR, 1787 &adev->gfx.bad_op_irq); 1788 if (r) 1789 return r; 1790 1791 /* Privileged reg */ 1792 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1793 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT, 1794 &adev->gfx.priv_reg_irq); 1795 if (r) 1796 return r; 1797 1798 /* Privileged inst */ 1799 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1800 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT, 1801 &adev->gfx.priv_inst_irq); 1802 if (r) 1803 return r; 1804 1805 /* FED error */ 1806 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX, 1807 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT, 1808 &adev->gfx.rlc_gc_fed_irq); 1809 if (r) 1810 return r; 1811 1812 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1813 1814 gfx_v11_0_me_init(adev); 1815 1816 r = gfx_v11_0_rlc_init(adev); 1817 if (r) { 1818 DRM_ERROR("Failed to init rlc BOs!\n"); 1819 return r; 1820 } 1821 1822 r = gfx_v11_0_mec_init(adev); 1823 if (r) { 1824 DRM_ERROR("Failed to init MEC BOs!\n"); 1825 return r; 1826 } 1827 1828 if (adev->gfx.num_gfx_rings) { 1829 ring_id = 0; 1830 /* set up the gfx ring */ 1831 for (i = 0; i < adev->gfx.me.num_me; i++) { 1832 for (j = 0; j < num_queue_per_pipe; j++) { 1833 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 1834 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j)) 1835 continue; 1836 1837 r = gfx_v11_0_gfx_ring_init(adev, ring_id, 1838 i, k, j); 1839 if (r) 1840 return r; 1841 ring_id++; 1842 } 1843 } 1844 } 1845 } 1846 1847 if (adev->gfx.num_compute_rings) { 1848 ring_id = 0; 1849 /* set up the compute queues - allocate horizontally across pipes */ 1850 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1851 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1852 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1853 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 1854 k, j)) 1855 continue; 1856 1857 r = gfx_v11_0_compute_ring_init(adev, ring_id, 1858 i, k, j); 1859 if (r) 1860 return r; 1861 1862 ring_id++; 1863 } 1864 } 1865 } 1866 } 1867 1868 adev->gfx.gfx_supported_reset = 1869 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 1870 adev->gfx.compute_supported_reset = 1871 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 1872 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1873 case IP_VERSION(11, 0, 0): 1874 case IP_VERSION(11, 0, 2): 1875 case IP_VERSION(11, 0, 3): 1876 if ((adev->gfx.me_fw_version >= 2280) && 1877 (adev->gfx.mec_fw_version >= 2410) && 1878 !amdgpu_sriov_vf(adev) && 1879 !adev->debug_disable_gpu_ring_reset) { 1880 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1881 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1882 } 1883 break; 1884 default: 1885 if (!amdgpu_sriov_vf(adev) && 1886 !adev->debug_disable_gpu_ring_reset) { 1887 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1888 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1889 } 1890 break; 1891 } 1892 1893 if (!adev->enable_mes_kiq) { 1894 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0); 1895 if (r) { 1896 DRM_ERROR("Failed to init KIQ BOs!\n"); 1897 return r; 1898 } 1899 1900 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 1901 if (r) 1902 return r; 1903 } 1904 1905 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0); 1906 if (r) 1907 return r; 1908 1909 /* allocate visible FB for rlc auto-loading fw */ 1910 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 1911 r = gfx_v11_0_rlc_autoload_buffer_init(adev); 1912 if (r) 1913 return r; 1914 } 1915 1916 r = gfx_v11_0_gpu_early_init(adev); 1917 if (r) 1918 return r; 1919 1920 if (amdgpu_gfx_ras_sw_init(adev)) { 1921 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 1922 return -EINVAL; 1923 } 1924 1925 r = amdgpu_gfx_sysfs_init(adev); 1926 if (r) 1927 return r; 1928 1929 gfx_v11_0_alloc_ip_dump(adev); 1930 1931 adev->gfx.me.use_mmio_for_reset = false; 1932 adev->gfx.mec.use_mmio_for_reset = true; 1933 1934 mutex_init(&adev->gfx.mec.reset_mutex); 1935 1936 INIT_WORK(&adev->gfx.userq_priv_fault_work, gfx_v11_0_userq_priv_fault_work); 1937 1938 return 0; 1939 } 1940 1941 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev) 1942 { 1943 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj, 1944 &adev->gfx.pfp.pfp_fw_gpu_addr, 1945 (void **)&adev->gfx.pfp.pfp_fw_ptr); 1946 1947 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj, 1948 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 1949 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 1950 } 1951 1952 static void gfx_v11_0_me_fini(struct amdgpu_device *adev) 1953 { 1954 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj, 1955 &adev->gfx.me.me_fw_gpu_addr, 1956 (void **)&adev->gfx.me.me_fw_ptr); 1957 1958 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj, 1959 &adev->gfx.me.me_fw_data_gpu_addr, 1960 (void **)&adev->gfx.me.me_fw_data_ptr); 1961 } 1962 1963 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev) 1964 { 1965 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo, 1966 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1967 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1968 } 1969 1970 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block) 1971 { 1972 int i; 1973 struct amdgpu_device *adev = ip_block->adev; 1974 1975 cancel_work_sync(&adev->gfx.userq_priv_fault_work); 1976 1977 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 1978 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 1979 for (i = 0; i < adev->gfx.num_compute_rings; i++) 1980 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 1981 1982 amdgpu_gfx_mqd_sw_fini(adev, 0); 1983 1984 if (!adev->enable_mes_kiq) { 1985 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 1986 amdgpu_gfx_kiq_fini(adev, 0); 1987 } 1988 1989 amdgpu_gfx_cleaner_shader_sw_fini(adev); 1990 1991 gfx_v11_0_pfp_fini(adev); 1992 gfx_v11_0_me_fini(adev); 1993 gfx_v11_0_rlc_fini(adev); 1994 gfx_v11_0_mec_fini(adev); 1995 1996 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 1997 gfx_v11_0_rlc_autoload_buffer_fini(adev); 1998 1999 gfx_v11_0_free_microcode(adev); 2000 2001 amdgpu_gfx_sysfs_fini(adev); 2002 2003 kfree(adev->gfx.ip_dump_core); 2004 kfree(adev->gfx.ip_dump_compute_queues); 2005 kfree(adev->gfx.ip_dump_gfx_queues); 2006 2007 return 0; 2008 } 2009 2010 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 2011 u32 sh_num, u32 instance, int xcc_id) 2012 { 2013 u32 data; 2014 2015 if (instance == 0xffffffff) 2016 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, 2017 INSTANCE_BROADCAST_WRITES, 1); 2018 else 2019 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, 2020 instance); 2021 2022 if (se_num == 0xffffffff) 2023 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 2024 1); 2025 else 2026 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2027 2028 if (sh_num == 0xffffffff) 2029 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES, 2030 1); 2031 else 2032 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num); 2033 2034 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data); 2035 } 2036 2037 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev) 2038 { 2039 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask; 2040 2041 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE); 2042 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask, 2043 CC_GC_SA_UNIT_DISABLE, 2044 SA_DISABLE); 2045 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE); 2046 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask, 2047 GC_USER_SA_UNIT_DISABLE, 2048 SA_DISABLE); 2049 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * 2050 adev->gfx.config.max_shader_engines); 2051 2052 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask)); 2053 } 2054 2055 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2056 { 2057 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask; 2058 u32 rb_mask; 2059 2060 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE); 2061 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask, 2062 CC_RB_BACKEND_DISABLE, 2063 BACKEND_DISABLE); 2064 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE); 2065 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask, 2066 GC_USER_RB_BACKEND_DISABLE, 2067 BACKEND_DISABLE); 2068 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se * 2069 adev->gfx.config.max_shader_engines); 2070 2071 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask)); 2072 } 2073 2074 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev) 2075 { 2076 u32 rb_bitmap_per_sa; 2077 u32 rb_bitmap_width_per_sa; 2078 u32 max_sa; 2079 u32 active_sa_bitmap; 2080 u32 global_active_rb_bitmap; 2081 u32 active_rb_bitmap = 0; 2082 u32 i; 2083 2084 /* query sa bitmap from SA_UNIT_DISABLE registers */ 2085 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev); 2086 /* query rb bitmap from RB_BACKEND_DISABLE registers */ 2087 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev); 2088 2089 /* generate active rb bitmap according to active sa bitmap */ 2090 max_sa = adev->gfx.config.max_shader_engines * 2091 adev->gfx.config.max_sh_per_se; 2092 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se / 2093 adev->gfx.config.max_sh_per_se; 2094 rb_bitmap_per_sa = amdgpu_gfx_create_bitmask(rb_bitmap_width_per_sa); 2095 2096 for (i = 0; i < max_sa; i++) { 2097 if (active_sa_bitmap & (1 << i)) 2098 active_rb_bitmap |= (rb_bitmap_per_sa << (i * rb_bitmap_width_per_sa)); 2099 } 2100 2101 active_rb_bitmap &= global_active_rb_bitmap; 2102 adev->gfx.config.backend_enable_mask = active_rb_bitmap; 2103 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap); 2104 } 2105 2106 #define DEFAULT_SH_MEM_BASES (0x6000) 2107 #define LDS_APP_BASE 0x1 2108 #define SCRATCH_APP_BASE 0x2 2109 2110 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev) 2111 { 2112 int i; 2113 uint32_t sh_mem_bases; 2114 uint32_t data; 2115 2116 /* 2117 * Configure apertures: 2118 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2119 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2120 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2121 */ 2122 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) | 2123 SCRATCH_APP_BASE; 2124 2125 mutex_lock(&adev->srbm_mutex); 2126 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2127 soc21_grbm_select(adev, 0, 0, 0, i); 2128 /* CP and shaders */ 2129 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2130 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases); 2131 2132 /* Enable trap for each kfd vmid. */ 2133 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL); 2134 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 2135 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data); 2136 } 2137 soc21_grbm_select(adev, 0, 0, 0, 0); 2138 mutex_unlock(&adev->srbm_mutex); 2139 2140 /* 2141 * Initialize all compute VMIDs to have no GDS, GWS, or OA 2142 * access. These should be enabled by FW for target VMIDs. 2143 */ 2144 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2145 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0); 2146 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0); 2147 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0); 2148 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0); 2149 } 2150 } 2151 2152 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev) 2153 { 2154 int vmid; 2155 2156 /* 2157 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2158 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2159 * the driver can enable them for graphics. VMID0 should maintain 2160 * access so that HWS firmware can save/restore entries. 2161 */ 2162 for (vmid = 1; vmid < 16; vmid++) { 2163 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0); 2164 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0); 2165 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0); 2166 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0); 2167 } 2168 } 2169 2170 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev) 2171 { 2172 /* TODO: harvest feature to be added later. */ 2173 } 2174 2175 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev) 2176 { 2177 /* TCCs are global (not instanced). */ 2178 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) | 2179 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE); 2180 2181 adev->gfx.config.tcc_disabled_mask = 2182 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) | 2183 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16); 2184 } 2185 2186 static void gfx_v11_0_constants_init(struct amdgpu_device *adev) 2187 { 2188 u32 tmp; 2189 int i; 2190 2191 if (!amdgpu_sriov_vf(adev)) 2192 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2193 2194 gfx_v11_0_setup_rb(adev); 2195 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info); 2196 gfx_v11_0_get_tcc_info(adev); 2197 adev->gfx.config.pa_sc_tile_steering_override = 0; 2198 2199 /* Set whether texture coordinate truncation is conformant. */ 2200 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); 2201 adev->gfx.config.ta_cntl2_truncate_coord_mode = 2202 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); 2203 /* Program DB_RING_CONTROL for multiple GFX pipes 2204 * Default power up value is 1. 2205 * Possible values: 2206 * 0 - split occlusion counters between gfx pipes 2207 * 1 - all occlusion counters to pipe 0 2208 * 2 - all occlusion counters to pipe 1 2209 */ 2210 WREG32_FIELD15_PREREG(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL, 2211 (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1); 2212 2213 /* XXX SH_MEM regs */ 2214 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2215 mutex_lock(&adev->srbm_mutex); 2216 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2217 soc21_grbm_select(adev, 0, 0, 0, i); 2218 /* CP and shaders */ 2219 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2220 if (i != 0) { 2221 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2222 (adev->gmc.private_aperture_start >> 48)); 2223 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2224 (adev->gmc.shared_aperture_start >> 48)); 2225 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp); 2226 } 2227 } 2228 soc21_grbm_select(adev, 0, 0, 0, 0); 2229 2230 mutex_unlock(&adev->srbm_mutex); 2231 2232 gfx_v11_0_init_compute_vmid(adev); 2233 gfx_v11_0_init_gds_vmid(adev); 2234 } 2235 2236 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev, 2237 int me, int pipe) 2238 { 2239 if (me != 0) 2240 return 0; 2241 2242 switch (pipe) { 2243 case 0: 2244 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 2245 case 1: 2246 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 2247 default: 2248 return 0; 2249 } 2250 } 2251 2252 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev, 2253 int me, int pipe) 2254 { 2255 /* 2256 * amdgpu controls only the first MEC. That's why this function only 2257 * handles the setting of interrupts for this specific MEC. All other 2258 * pipes' interrupts are set by amdkfd. 2259 */ 2260 if (me != 1) 2261 return 0; 2262 2263 switch (pipe) { 2264 case 0: 2265 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 2266 case 1: 2267 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 2268 case 2: 2269 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 2270 case 3: 2271 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 2272 default: 2273 return 0; 2274 } 2275 } 2276 2277 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2278 bool enable) 2279 { 2280 u32 tmp, cp_int_cntl_reg; 2281 int i, j; 2282 2283 if (amdgpu_sriov_vf(adev)) 2284 return; 2285 2286 for (i = 0; i < adev->gfx.me.num_me; i++) { 2287 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 2288 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 2289 2290 if (cp_int_cntl_reg) { 2291 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 2292 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 2293 enable ? 1 : 0); 2294 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 2295 enable ? 1 : 0); 2296 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 2297 enable ? 1 : 0); 2298 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 2299 enable ? 1 : 0); 2300 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp); 2301 } 2302 } 2303 } 2304 } 2305 2306 static int gfx_v11_0_init_csb(struct amdgpu_device *adev) 2307 { 2308 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2309 2310 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI, 2311 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2312 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO, 2313 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2314 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); 2315 2316 return 0; 2317 } 2318 2319 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev) 2320 { 2321 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL); 2322 2323 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); 2324 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp); 2325 } 2326 2327 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev) 2328 { 2329 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2330 udelay(50); 2331 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2332 udelay(50); 2333 } 2334 2335 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev, 2336 bool enable) 2337 { 2338 uint32_t rlc_pg_cntl; 2339 2340 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 2341 2342 if (!enable) { 2343 /* RLC_PG_CNTL[23] = 0 (default) 2344 * RLC will wait for handshake acks with SMU 2345 * GFXOFF will be enabled 2346 * RLC_PG_CNTL[23] = 1 2347 * RLC will not issue any message to SMU 2348 * hence no handshake between SMU & RLC 2349 * GFXOFF will be disabled 2350 */ 2351 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2352 } else 2353 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2354 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl); 2355 } 2356 2357 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev) 2358 { 2359 /* TODO: enable rlc & smu handshake until smu 2360 * and gfxoff feature works as expected */ 2361 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK)) 2362 gfx_v11_0_rlc_smu_handshake_cntl(adev, false); 2363 2364 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 2365 udelay(50); 2366 } 2367 2368 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev) 2369 { 2370 uint32_t tmp; 2371 2372 /* enable Save Restore Machine */ 2373 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL)); 2374 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2375 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK; 2376 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp); 2377 } 2378 2379 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev) 2380 { 2381 const struct rlc_firmware_header_v2_0 *hdr; 2382 const __le32 *fw_data; 2383 unsigned i, fw_size; 2384 2385 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2386 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2387 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2388 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 2389 2390 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, 2391 RLCG_UCODE_LOADING_START_ADDRESS); 2392 2393 for (i = 0; i < fw_size; i++) 2394 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA, 2395 le32_to_cpup(fw_data++)); 2396 2397 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 2398 } 2399 2400 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev) 2401 { 2402 const struct rlc_firmware_header_v2_2 *hdr; 2403 const __le32 *fw_data; 2404 unsigned i, fw_size; 2405 u32 tmp; 2406 2407 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 2408 2409 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2410 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes)); 2411 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4; 2412 2413 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0); 2414 2415 for (i = 0; i < fw_size; i++) { 2416 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2417 msleep(1); 2418 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA, 2419 le32_to_cpup(fw_data++)); 2420 } 2421 2422 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2423 2424 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2425 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes)); 2426 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4; 2427 2428 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0); 2429 for (i = 0; i < fw_size; i++) { 2430 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2431 msleep(1); 2432 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA, 2433 le32_to_cpup(fw_data++)); 2434 } 2435 2436 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2437 2438 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL); 2439 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1); 2440 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0); 2441 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp); 2442 } 2443 2444 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev) 2445 { 2446 const struct rlc_firmware_header_v2_3 *hdr; 2447 const __le32 *fw_data; 2448 unsigned i, fw_size; 2449 u32 tmp; 2450 2451 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data; 2452 2453 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2454 le32_to_cpu(hdr->rlcp_ucode_offset_bytes)); 2455 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4; 2456 2457 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0); 2458 2459 for (i = 0; i < fw_size; i++) { 2460 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2461 msleep(1); 2462 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA, 2463 le32_to_cpup(fw_data++)); 2464 } 2465 2466 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version); 2467 2468 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE); 2469 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1); 2470 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp); 2471 2472 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2473 le32_to_cpu(hdr->rlcv_ucode_offset_bytes)); 2474 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4; 2475 2476 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0); 2477 2478 for (i = 0; i < fw_size; i++) { 2479 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2480 msleep(1); 2481 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA, 2482 le32_to_cpup(fw_data++)); 2483 } 2484 2485 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version); 2486 2487 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL); 2488 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1); 2489 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp); 2490 } 2491 2492 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev) 2493 { 2494 const struct rlc_firmware_header_v2_0 *hdr; 2495 uint16_t version_major; 2496 uint16_t version_minor; 2497 2498 if (!adev->gfx.rlc_fw) 2499 return -EINVAL; 2500 2501 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2502 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2503 2504 version_major = le16_to_cpu(hdr->header.header_version_major); 2505 version_minor = le16_to_cpu(hdr->header.header_version_minor); 2506 2507 if (version_major == 2) { 2508 gfx_v11_0_load_rlcg_microcode(adev); 2509 if (amdgpu_dpm == 1) { 2510 if (version_minor >= 2) 2511 gfx_v11_0_load_rlc_iram_dram_microcode(adev); 2512 if (version_minor == 3) 2513 gfx_v11_0_load_rlcp_rlcv_microcode(adev); 2514 } 2515 2516 return 0; 2517 } 2518 2519 return -EINVAL; 2520 } 2521 2522 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev) 2523 { 2524 int r; 2525 2526 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 2527 gfx_v11_0_init_csb(adev); 2528 2529 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */ 2530 gfx_v11_0_rlc_enable_srm(adev); 2531 } else { 2532 if (amdgpu_sriov_vf(adev)) { 2533 gfx_v11_0_init_csb(adev); 2534 return 0; 2535 } 2536 2537 adev->gfx.rlc.funcs->stop(adev); 2538 2539 /* disable CG */ 2540 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0); 2541 2542 /* disable PG */ 2543 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0); 2544 2545 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 2546 /* legacy rlc firmware loading */ 2547 r = gfx_v11_0_rlc_load_microcode(adev); 2548 if (r) 2549 return r; 2550 } 2551 2552 gfx_v11_0_init_csb(adev); 2553 2554 adev->gfx.rlc.funcs->start(adev); 2555 } 2556 return 0; 2557 } 2558 2559 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr) 2560 { 2561 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2562 uint32_t tmp; 2563 int i; 2564 2565 /* Trigger an invalidation of the L1 instruction caches */ 2566 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2567 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2568 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2569 2570 /* Wait for invalidation complete */ 2571 for (i = 0; i < usec_timeout; i++) { 2572 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2573 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2574 INVALIDATE_CACHE_COMPLETE)) 2575 break; 2576 udelay(1); 2577 } 2578 2579 if (i >= usec_timeout) { 2580 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2581 return -EINVAL; 2582 } 2583 2584 if (amdgpu_emu_mode == 1) 2585 amdgpu_device_flush_hdp(adev, NULL); 2586 2587 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2588 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2589 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2590 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2591 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2592 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2593 2594 /* Program me ucode address into intruction cache address register */ 2595 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2596 lower_32_bits(addr) & 0xFFFFF000); 2597 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2598 upper_32_bits(addr)); 2599 2600 return 0; 2601 } 2602 2603 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr) 2604 { 2605 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2606 uint32_t tmp; 2607 int i; 2608 2609 /* Trigger an invalidation of the L1 instruction caches */ 2610 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2611 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2612 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2613 2614 /* Wait for invalidation complete */ 2615 for (i = 0; i < usec_timeout; i++) { 2616 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2617 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2618 INVALIDATE_CACHE_COMPLETE)) 2619 break; 2620 udelay(1); 2621 } 2622 2623 if (i >= usec_timeout) { 2624 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2625 return -EINVAL; 2626 } 2627 2628 if (amdgpu_emu_mode == 1) 2629 amdgpu_device_flush_hdp(adev, NULL); 2630 2631 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2632 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2633 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2634 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2635 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2636 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2637 2638 /* Program pfp ucode address into intruction cache address register */ 2639 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2640 lower_32_bits(addr) & 0xFFFFF000); 2641 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2642 upper_32_bits(addr)); 2643 2644 return 0; 2645 } 2646 2647 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr) 2648 { 2649 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2650 uint32_t tmp; 2651 int i; 2652 2653 /* Trigger an invalidation of the L1 instruction caches */ 2654 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2655 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2656 2657 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2658 2659 /* Wait for invalidation complete */ 2660 for (i = 0; i < usec_timeout; i++) { 2661 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2662 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2663 INVALIDATE_CACHE_COMPLETE)) 2664 break; 2665 udelay(1); 2666 } 2667 2668 if (i >= usec_timeout) { 2669 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2670 return -EINVAL; 2671 } 2672 2673 if (amdgpu_emu_mode == 1) 2674 amdgpu_device_flush_hdp(adev, NULL); 2675 2676 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2677 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2678 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2679 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2680 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2681 2682 /* Program mec1 ucode address into intruction cache address register */ 2683 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, 2684 lower_32_bits(addr) & 0xFFFFF000); 2685 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2686 upper_32_bits(addr)); 2687 2688 return 0; 2689 } 2690 2691 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2692 { 2693 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2694 uint32_t tmp; 2695 unsigned i, pipe_id; 2696 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2697 2698 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 2699 adev->gfx.pfp_fw->data; 2700 2701 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2702 lower_32_bits(addr)); 2703 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2704 upper_32_bits(addr)); 2705 2706 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2707 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2708 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2709 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2710 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2711 2712 /* 2713 * Programming any of the CP_PFP_IC_BASE registers 2714 * forces invalidation of the ME L1 I$. Wait for the 2715 * invalidation complete 2716 */ 2717 for (i = 0; i < usec_timeout; i++) { 2718 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2719 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2720 INVALIDATE_CACHE_COMPLETE)) 2721 break; 2722 udelay(1); 2723 } 2724 2725 if (i >= usec_timeout) { 2726 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2727 return -EINVAL; 2728 } 2729 2730 /* Prime the L1 instruction caches */ 2731 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2732 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 2733 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2734 /* Waiting for cache primed*/ 2735 for (i = 0; i < usec_timeout; i++) { 2736 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2737 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2738 ICACHE_PRIMED)) 2739 break; 2740 udelay(1); 2741 } 2742 2743 if (i >= usec_timeout) { 2744 dev_err(adev->dev, "failed to prime instruction cache\n"); 2745 return -EINVAL; 2746 } 2747 2748 mutex_lock(&adev->srbm_mutex); 2749 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2750 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2751 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 2752 (pfp_hdr->ucode_start_addr_hi << 30) | 2753 (pfp_hdr->ucode_start_addr_lo >> 2)); 2754 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 2755 pfp_hdr->ucode_start_addr_hi >> 2); 2756 2757 /* 2758 * Program CP_ME_CNTL to reset given PIPE to take 2759 * effect of CP_PFP_PRGRM_CNTR_START. 2760 */ 2761 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2762 if (pipe_id == 0) 2763 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2764 PFP_PIPE0_RESET, 1); 2765 else 2766 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2767 PFP_PIPE1_RESET, 1); 2768 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2769 2770 /* Clear pfp pipe0 reset bit. */ 2771 if (pipe_id == 0) 2772 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2773 PFP_PIPE0_RESET, 0); 2774 else 2775 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2776 PFP_PIPE1_RESET, 0); 2777 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2778 2779 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 2780 lower_32_bits(addr2)); 2781 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 2782 upper_32_bits(addr2)); 2783 } 2784 soc21_grbm_select(adev, 0, 0, 0, 0); 2785 mutex_unlock(&adev->srbm_mutex); 2786 2787 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2788 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2789 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2790 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2791 2792 /* Invalidate the data caches */ 2793 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2794 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2795 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2796 2797 for (i = 0; i < usec_timeout; i++) { 2798 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2799 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2800 INVALIDATE_DCACHE_COMPLETE)) 2801 break; 2802 udelay(1); 2803 } 2804 2805 if (i >= usec_timeout) { 2806 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2807 return -EINVAL; 2808 } 2809 2810 return 0; 2811 } 2812 2813 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2814 { 2815 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2816 uint32_t tmp; 2817 unsigned i, pipe_id; 2818 const struct gfx_firmware_header_v2_0 *me_hdr; 2819 2820 me_hdr = (const struct gfx_firmware_header_v2_0 *) 2821 adev->gfx.me_fw->data; 2822 2823 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2824 lower_32_bits(addr)); 2825 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2826 upper_32_bits(addr)); 2827 2828 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2829 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2830 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2831 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2832 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2833 2834 /* 2835 * Programming any of the CP_ME_IC_BASE registers 2836 * forces invalidation of the ME L1 I$. Wait for the 2837 * invalidation complete 2838 */ 2839 for (i = 0; i < usec_timeout; i++) { 2840 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2841 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2842 INVALIDATE_CACHE_COMPLETE)) 2843 break; 2844 udelay(1); 2845 } 2846 2847 if (i >= usec_timeout) { 2848 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2849 return -EINVAL; 2850 } 2851 2852 /* Prime the instruction caches */ 2853 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2854 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 2855 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2856 2857 /* Waiting for instruction cache primed*/ 2858 for (i = 0; i < usec_timeout; i++) { 2859 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2860 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2861 ICACHE_PRIMED)) 2862 break; 2863 udelay(1); 2864 } 2865 2866 if (i >= usec_timeout) { 2867 dev_err(adev->dev, "failed to prime instruction cache\n"); 2868 return -EINVAL; 2869 } 2870 2871 mutex_lock(&adev->srbm_mutex); 2872 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2873 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2874 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 2875 (me_hdr->ucode_start_addr_hi << 30) | 2876 (me_hdr->ucode_start_addr_lo >> 2) ); 2877 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 2878 me_hdr->ucode_start_addr_hi>>2); 2879 2880 /* 2881 * Program CP_ME_CNTL to reset given PIPE to take 2882 * effect of CP_PFP_PRGRM_CNTR_START. 2883 */ 2884 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2885 if (pipe_id == 0) 2886 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2887 ME_PIPE0_RESET, 1); 2888 else 2889 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2890 ME_PIPE1_RESET, 1); 2891 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2892 2893 /* Clear pfp pipe0 reset bit. */ 2894 if (pipe_id == 0) 2895 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2896 ME_PIPE0_RESET, 0); 2897 else 2898 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2899 ME_PIPE1_RESET, 0); 2900 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2901 2902 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 2903 lower_32_bits(addr2)); 2904 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 2905 upper_32_bits(addr2)); 2906 } 2907 soc21_grbm_select(adev, 0, 0, 0, 0); 2908 mutex_unlock(&adev->srbm_mutex); 2909 2910 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2911 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2912 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2913 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2914 2915 /* Invalidate the data caches */ 2916 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2917 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2918 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2919 2920 for (i = 0; i < usec_timeout; i++) { 2921 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2922 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2923 INVALIDATE_DCACHE_COMPLETE)) 2924 break; 2925 udelay(1); 2926 } 2927 2928 if (i >= usec_timeout) { 2929 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2930 return -EINVAL; 2931 } 2932 2933 return 0; 2934 } 2935 2936 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2937 { 2938 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2939 uint32_t tmp; 2940 unsigned i; 2941 const struct gfx_firmware_header_v2_0 *mec_hdr; 2942 2943 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2944 adev->gfx.mec_fw->data; 2945 2946 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2947 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 2948 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2949 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2950 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2951 2952 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 2953 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 2954 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 2955 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 2956 2957 mutex_lock(&adev->srbm_mutex); 2958 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 2959 soc21_grbm_select(adev, 1, i, 0, 0); 2960 2961 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2); 2962 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 2963 upper_32_bits(addr2)); 2964 2965 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 2966 mec_hdr->ucode_start_addr_lo >> 2 | 2967 mec_hdr->ucode_start_addr_hi << 30); 2968 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2969 mec_hdr->ucode_start_addr_hi >> 2); 2970 2971 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr); 2972 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2973 upper_32_bits(addr)); 2974 } 2975 mutex_unlock(&adev->srbm_mutex); 2976 soc21_grbm_select(adev, 0, 0, 0, 0); 2977 2978 /* Trigger an invalidation of the L1 instruction caches */ 2979 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2980 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2981 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 2982 2983 /* Wait for invalidation complete */ 2984 for (i = 0; i < usec_timeout; i++) { 2985 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2986 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 2987 INVALIDATE_DCACHE_COMPLETE)) 2988 break; 2989 udelay(1); 2990 } 2991 2992 if (i >= usec_timeout) { 2993 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2994 return -EINVAL; 2995 } 2996 2997 /* Trigger an invalidation of the L1 instruction caches */ 2998 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2999 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 3000 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 3001 3002 /* Wait for invalidation complete */ 3003 for (i = 0; i < usec_timeout; i++) { 3004 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 3005 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 3006 INVALIDATE_CACHE_COMPLETE)) 3007 break; 3008 udelay(1); 3009 } 3010 3011 if (i >= usec_timeout) { 3012 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3013 return -EINVAL; 3014 } 3015 3016 return 0; 3017 } 3018 3019 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev) 3020 { 3021 const struct gfx_firmware_header_v2_0 *pfp_hdr; 3022 const struct gfx_firmware_header_v2_0 *me_hdr; 3023 const struct gfx_firmware_header_v2_0 *mec_hdr; 3024 uint32_t pipe_id, tmp; 3025 3026 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 3027 adev->gfx.mec_fw->data; 3028 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3029 adev->gfx.me_fw->data; 3030 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3031 adev->gfx.pfp_fw->data; 3032 3033 /* config pfp program start addr */ 3034 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3035 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3036 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3037 (pfp_hdr->ucode_start_addr_hi << 30) | 3038 (pfp_hdr->ucode_start_addr_lo >> 2)); 3039 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3040 pfp_hdr->ucode_start_addr_hi >> 2); 3041 } 3042 soc21_grbm_select(adev, 0, 0, 0, 0); 3043 3044 /* reset pfp pipe */ 3045 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3046 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1); 3047 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1); 3048 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3049 3050 /* clear pfp pipe reset */ 3051 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0); 3052 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0); 3053 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3054 3055 /* config me program start addr */ 3056 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3057 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3058 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3059 (me_hdr->ucode_start_addr_hi << 30) | 3060 (me_hdr->ucode_start_addr_lo >> 2) ); 3061 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3062 me_hdr->ucode_start_addr_hi>>2); 3063 } 3064 soc21_grbm_select(adev, 0, 0, 0, 0); 3065 3066 /* reset me pipe */ 3067 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3068 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1); 3069 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1); 3070 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3071 3072 /* clear me pipe reset */ 3073 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0); 3074 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0); 3075 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3076 3077 /* config mec program start addr */ 3078 for (pipe_id = 0; pipe_id < 4; pipe_id++) { 3079 soc21_grbm_select(adev, 1, pipe_id, 0, 0); 3080 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 3081 mec_hdr->ucode_start_addr_lo >> 2 | 3082 mec_hdr->ucode_start_addr_hi << 30); 3083 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 3084 mec_hdr->ucode_start_addr_hi >> 2); 3085 } 3086 soc21_grbm_select(adev, 0, 0, 0, 0); 3087 3088 /* reset mec pipe */ 3089 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3090 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1); 3091 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1); 3092 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1); 3093 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1); 3094 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3095 3096 /* clear mec pipe reset */ 3097 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0); 3098 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0); 3099 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0); 3100 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0); 3101 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3102 } 3103 3104 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) 3105 { 3106 uint32_t cp_status; 3107 uint32_t bootload_status; 3108 int i, r; 3109 uint64_t addr, addr2; 3110 3111 for (i = 0; i < adev->usec_timeout; i++) { 3112 cp_status = RREG32_SOC15(GC, 0, regCP_STAT); 3113 3114 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 3115 IP_VERSION(11, 0, 1) || 3116 amdgpu_ip_version(adev, GC_HWIP, 0) == 3117 IP_VERSION(11, 0, 4) || 3118 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) || 3119 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) || 3120 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) || 3121 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3) || 3122 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4) || 3123 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6) || 3124 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 0) || 3125 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 1)) 3126 bootload_status = RREG32_SOC15(GC, 0, 3127 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1); 3128 else 3129 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS); 3130 3131 if ((cp_status == 0) && 3132 (REG_GET_FIELD(bootload_status, 3133 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) { 3134 break; 3135 } 3136 udelay(1); 3137 } 3138 3139 if (i >= adev->usec_timeout) { 3140 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n"); 3141 return -ETIMEDOUT; 3142 } 3143 3144 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 3145 if (adev->gfx.rs64_enable) { 3146 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3147 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset; 3148 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3149 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset; 3150 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2); 3151 if (r) 3152 return r; 3153 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3154 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset; 3155 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3156 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset; 3157 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2); 3158 if (r) 3159 return r; 3160 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3161 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset; 3162 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3163 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset; 3164 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2); 3165 if (r) 3166 return r; 3167 } else { 3168 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3169 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset; 3170 r = gfx_v11_0_config_me_cache(adev, addr); 3171 if (r) 3172 return r; 3173 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3174 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset; 3175 r = gfx_v11_0_config_pfp_cache(adev, addr); 3176 if (r) 3177 return r; 3178 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3179 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset; 3180 r = gfx_v11_0_config_mec_cache(adev, addr); 3181 if (r) 3182 return r; 3183 } 3184 } 3185 3186 return 0; 3187 } 3188 3189 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3190 { 3191 int i; 3192 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3193 3194 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3195 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3196 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3197 3198 for (i = 0; i < adev->usec_timeout; i++) { 3199 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0) 3200 break; 3201 udelay(1); 3202 } 3203 3204 if (i >= adev->usec_timeout) 3205 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt"); 3206 3207 return 0; 3208 } 3209 3210 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev) 3211 { 3212 int r; 3213 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3214 const __le32 *fw_data; 3215 unsigned i, fw_size; 3216 3217 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3218 adev->gfx.pfp_fw->data; 3219 3220 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3221 3222 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3223 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3224 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes); 3225 3226 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes, 3227 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3228 &adev->gfx.pfp.pfp_fw_obj, 3229 &adev->gfx.pfp.pfp_fw_gpu_addr, 3230 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3231 if (r) { 3232 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r); 3233 gfx_v11_0_pfp_fini(adev); 3234 return r; 3235 } 3236 3237 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size); 3238 3239 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3240 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3241 3242 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr); 3243 3244 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0); 3245 3246 for (i = 0; i < pfp_hdr->jt_size; i++) 3247 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA, 3248 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i)); 3249 3250 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3251 3252 return 0; 3253 } 3254 3255 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) 3256 { 3257 int r; 3258 const struct gfx_firmware_header_v2_0 *pfp_hdr; 3259 const __le32 *fw_ucode, *fw_data; 3260 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3261 uint32_t tmp; 3262 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3263 3264 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3265 adev->gfx.pfp_fw->data; 3266 3267 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3268 3269 /* instruction */ 3270 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data + 3271 le32_to_cpu(pfp_hdr->ucode_offset_bytes)); 3272 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes); 3273 /* data */ 3274 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3275 le32_to_cpu(pfp_hdr->data_offset_bytes)); 3276 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes); 3277 3278 /* 64kb align */ 3279 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3280 64 * 1024, 3281 AMDGPU_GEM_DOMAIN_VRAM | 3282 AMDGPU_GEM_DOMAIN_GTT, 3283 &adev->gfx.pfp.pfp_fw_obj, 3284 &adev->gfx.pfp.pfp_fw_gpu_addr, 3285 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3286 if (r) { 3287 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r); 3288 gfx_v11_0_pfp_fini(adev); 3289 return r; 3290 } 3291 3292 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3293 64 * 1024, 3294 AMDGPU_GEM_DOMAIN_VRAM | 3295 AMDGPU_GEM_DOMAIN_GTT, 3296 &adev->gfx.pfp.pfp_fw_data_obj, 3297 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 3298 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 3299 if (r) { 3300 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r); 3301 gfx_v11_0_pfp_fini(adev); 3302 return r; 3303 } 3304 3305 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size); 3306 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); 3307 3308 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3309 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj); 3310 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3311 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj); 3312 3313 if (amdgpu_emu_mode == 1) 3314 amdgpu_device_flush_hdp(adev, NULL); 3315 3316 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 3317 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3318 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 3319 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3320 3321 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 3322 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 3323 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 3324 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 3325 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 3326 3327 /* 3328 * Programming any of the CP_PFP_IC_BASE registers 3329 * forces invalidation of the ME L1 I$. Wait for the 3330 * invalidation complete 3331 */ 3332 for (i = 0; i < usec_timeout; i++) { 3333 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3334 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3335 INVALIDATE_CACHE_COMPLETE)) 3336 break; 3337 udelay(1); 3338 } 3339 3340 if (i >= usec_timeout) { 3341 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3342 return -EINVAL; 3343 } 3344 3345 /* Prime the L1 instruction caches */ 3346 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3347 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 3348 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 3349 /* Waiting for cache primed*/ 3350 for (i = 0; i < usec_timeout; i++) { 3351 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3352 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3353 ICACHE_PRIMED)) 3354 break; 3355 udelay(1); 3356 } 3357 3358 if (i >= usec_timeout) { 3359 dev_err(adev->dev, "failed to prime instruction cache\n"); 3360 return -EINVAL; 3361 } 3362 3363 mutex_lock(&adev->srbm_mutex); 3364 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3365 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3366 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3367 (pfp_hdr->ucode_start_addr_hi << 30) | 3368 (pfp_hdr->ucode_start_addr_lo >> 2) ); 3369 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3370 pfp_hdr->ucode_start_addr_hi>>2); 3371 3372 /* 3373 * Program CP_ME_CNTL to reset given PIPE to take 3374 * effect of CP_PFP_PRGRM_CNTR_START. 3375 */ 3376 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3377 if (pipe_id == 0) 3378 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3379 PFP_PIPE0_RESET, 1); 3380 else 3381 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3382 PFP_PIPE1_RESET, 1); 3383 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3384 3385 /* Clear pfp pipe0 reset bit. */ 3386 if (pipe_id == 0) 3387 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3388 PFP_PIPE0_RESET, 0); 3389 else 3390 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3391 PFP_PIPE1_RESET, 0); 3392 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3393 3394 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 3395 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3396 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 3397 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3398 } 3399 soc21_grbm_select(adev, 0, 0, 0, 0); 3400 mutex_unlock(&adev->srbm_mutex); 3401 3402 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3403 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3404 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3405 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3406 3407 /* Invalidate the data caches */ 3408 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3409 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3410 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3411 3412 for (i = 0; i < usec_timeout; i++) { 3413 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3414 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3415 INVALIDATE_DCACHE_COMPLETE)) 3416 break; 3417 udelay(1); 3418 } 3419 3420 if (i >= usec_timeout) { 3421 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3422 return -EINVAL; 3423 } 3424 3425 return 0; 3426 } 3427 3428 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev) 3429 { 3430 int r; 3431 const struct gfx_firmware_header_v1_0 *me_hdr; 3432 const __le32 *fw_data; 3433 unsigned i, fw_size; 3434 3435 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3436 adev->gfx.me_fw->data; 3437 3438 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3439 3440 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3441 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3442 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes); 3443 3444 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes, 3445 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3446 &adev->gfx.me.me_fw_obj, 3447 &adev->gfx.me.me_fw_gpu_addr, 3448 (void **)&adev->gfx.me.me_fw_ptr); 3449 if (r) { 3450 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r); 3451 gfx_v11_0_me_fini(adev); 3452 return r; 3453 } 3454 3455 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size); 3456 3457 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3458 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3459 3460 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr); 3461 3462 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0); 3463 3464 for (i = 0; i < me_hdr->jt_size; i++) 3465 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA, 3466 le32_to_cpup(fw_data + me_hdr->jt_offset + i)); 3467 3468 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version); 3469 3470 return 0; 3471 } 3472 3473 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) 3474 { 3475 int r; 3476 const struct gfx_firmware_header_v2_0 *me_hdr; 3477 const __le32 *fw_ucode, *fw_data; 3478 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3479 uint32_t tmp; 3480 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3481 3482 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3483 adev->gfx.me_fw->data; 3484 3485 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3486 3487 /* instruction */ 3488 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data + 3489 le32_to_cpu(me_hdr->ucode_offset_bytes)); 3490 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes); 3491 /* data */ 3492 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3493 le32_to_cpu(me_hdr->data_offset_bytes)); 3494 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes); 3495 3496 /* 64kb align*/ 3497 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3498 64 * 1024, 3499 AMDGPU_GEM_DOMAIN_VRAM | 3500 AMDGPU_GEM_DOMAIN_GTT, 3501 &adev->gfx.me.me_fw_obj, 3502 &adev->gfx.me.me_fw_gpu_addr, 3503 (void **)&adev->gfx.me.me_fw_ptr); 3504 if (r) { 3505 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r); 3506 gfx_v11_0_me_fini(adev); 3507 return r; 3508 } 3509 3510 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3511 64 * 1024, 3512 AMDGPU_GEM_DOMAIN_VRAM | 3513 AMDGPU_GEM_DOMAIN_GTT, 3514 &adev->gfx.me.me_fw_data_obj, 3515 &adev->gfx.me.me_fw_data_gpu_addr, 3516 (void **)&adev->gfx.me.me_fw_data_ptr); 3517 if (r) { 3518 dev_err(adev->dev, "(%d) failed to create me data bo\n", r); 3519 gfx_v11_0_pfp_fini(adev); 3520 return r; 3521 } 3522 3523 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size); 3524 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); 3525 3526 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3527 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj); 3528 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3529 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj); 3530 3531 if (amdgpu_emu_mode == 1) 3532 amdgpu_device_flush_hdp(adev, NULL); 3533 3534 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 3535 lower_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3536 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 3537 upper_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3538 3539 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 3540 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 3541 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 3542 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 3543 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 3544 3545 /* 3546 * Programming any of the CP_ME_IC_BASE registers 3547 * forces invalidation of the ME L1 I$. Wait for the 3548 * invalidation complete 3549 */ 3550 for (i = 0; i < usec_timeout; i++) { 3551 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3552 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3553 INVALIDATE_CACHE_COMPLETE)) 3554 break; 3555 udelay(1); 3556 } 3557 3558 if (i >= usec_timeout) { 3559 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3560 return -EINVAL; 3561 } 3562 3563 /* Prime the instruction caches */ 3564 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3565 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 3566 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 3567 3568 /* Waiting for instruction cache primed*/ 3569 for (i = 0; i < usec_timeout; i++) { 3570 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3571 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3572 ICACHE_PRIMED)) 3573 break; 3574 udelay(1); 3575 } 3576 3577 if (i >= usec_timeout) { 3578 dev_err(adev->dev, "failed to prime instruction cache\n"); 3579 return -EINVAL; 3580 } 3581 3582 mutex_lock(&adev->srbm_mutex); 3583 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3584 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3585 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3586 (me_hdr->ucode_start_addr_hi << 30) | 3587 (me_hdr->ucode_start_addr_lo >> 2) ); 3588 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3589 me_hdr->ucode_start_addr_hi>>2); 3590 3591 /* 3592 * Program CP_ME_CNTL to reset given PIPE to take 3593 * effect of CP_PFP_PRGRM_CNTR_START. 3594 */ 3595 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3596 if (pipe_id == 0) 3597 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3598 ME_PIPE0_RESET, 1); 3599 else 3600 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3601 ME_PIPE1_RESET, 1); 3602 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3603 3604 /* Clear pfp pipe0 reset bit. */ 3605 if (pipe_id == 0) 3606 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3607 ME_PIPE0_RESET, 0); 3608 else 3609 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3610 ME_PIPE1_RESET, 0); 3611 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3612 3613 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 3614 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3615 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 3616 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3617 } 3618 soc21_grbm_select(adev, 0, 0, 0, 0); 3619 mutex_unlock(&adev->srbm_mutex); 3620 3621 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3622 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3623 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3624 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3625 3626 /* Invalidate the data caches */ 3627 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3628 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3629 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3630 3631 for (i = 0; i < usec_timeout; i++) { 3632 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3633 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3634 INVALIDATE_DCACHE_COMPLETE)) 3635 break; 3636 udelay(1); 3637 } 3638 3639 if (i >= usec_timeout) { 3640 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3641 return -EINVAL; 3642 } 3643 3644 return 0; 3645 } 3646 3647 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3648 { 3649 int r; 3650 3651 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw) 3652 return -EINVAL; 3653 3654 gfx_v11_0_cp_gfx_enable(adev, false); 3655 3656 if (adev->gfx.rs64_enable) 3657 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev); 3658 else 3659 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev); 3660 if (r) { 3661 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r); 3662 return r; 3663 } 3664 3665 if (adev->gfx.rs64_enable) 3666 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev); 3667 else 3668 r = gfx_v11_0_cp_gfx_load_me_microcode(adev); 3669 if (r) { 3670 dev_err(adev->dev, "(%d) failed to load me fw\n", r); 3671 return r; 3672 } 3673 3674 return 0; 3675 } 3676 3677 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev) 3678 { 3679 struct amdgpu_ring *ring; 3680 const struct cs_section_def *sect = NULL; 3681 const struct cs_extent_def *ext = NULL; 3682 int r, i; 3683 int ctx_reg_offset; 3684 3685 /* init the CP */ 3686 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT, 3687 adev->gfx.config.max_hw_contexts - 1); 3688 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1); 3689 3690 if (!amdgpu_async_gfx_ring) 3691 gfx_v11_0_cp_gfx_enable(adev, true); 3692 3693 ring = &adev->gfx.gfx_ring[0]; 3694 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev)); 3695 if (r) { 3696 drm_err(&adev->ddev, "cp failed to lock ring (%d).\n", r); 3697 return r; 3698 } 3699 3700 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3701 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3702 3703 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3704 amdgpu_ring_write(ring, 0x80000000); 3705 amdgpu_ring_write(ring, 0x80000000); 3706 3707 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 3708 for (ext = sect->section; ext->extent != NULL; ++ext) { 3709 if (sect->id == SECT_CONTEXT) { 3710 amdgpu_ring_write(ring, 3711 PACKET3(PACKET3_SET_CONTEXT_REG, 3712 ext->reg_count)); 3713 amdgpu_ring_write(ring, ext->reg_index - 3714 PACKET3_SET_CONTEXT_REG_START); 3715 for (i = 0; i < ext->reg_count; i++) 3716 amdgpu_ring_write(ring, ext->extent[i]); 3717 } 3718 } 3719 } 3720 3721 ctx_reg_offset = 3722 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 3723 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 3724 amdgpu_ring_write(ring, ctx_reg_offset); 3725 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override); 3726 3727 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3728 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3729 3730 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3731 amdgpu_ring_write(ring, 0); 3732 3733 amdgpu_ring_commit(ring); 3734 3735 /* submit cs packet to copy state 0 to next available state */ 3736 if (adev->gfx.num_gfx_rings > 1) { 3737 /* maximum supported gfx ring is 2 */ 3738 ring = &adev->gfx.gfx_ring[1]; 3739 r = amdgpu_ring_alloc(ring, 2); 3740 if (r) { 3741 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 3742 return r; 3743 } 3744 3745 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3746 amdgpu_ring_write(ring, 0); 3747 3748 amdgpu_ring_commit(ring); 3749 } 3750 return 0; 3751 } 3752 3753 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev, 3754 CP_PIPE_ID pipe) 3755 { 3756 u32 tmp; 3757 3758 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); 3759 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe); 3760 3761 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); 3762 } 3763 3764 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev, 3765 struct amdgpu_ring *ring) 3766 { 3767 u32 tmp; 3768 3769 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); 3770 if (ring->use_doorbell) { 3771 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3772 DOORBELL_OFFSET, ring->doorbell_index); 3773 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3774 DOORBELL_EN, 1); 3775 } else { 3776 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3777 DOORBELL_EN, 0); 3778 } 3779 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp); 3780 3781 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3782 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3783 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp); 3784 3785 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 3786 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3787 } 3788 3789 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev) 3790 { 3791 struct amdgpu_ring *ring; 3792 u32 tmp; 3793 u32 rb_bufsz; 3794 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3795 3796 /* Set the write pointer delay */ 3797 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0); 3798 3799 /* set the RB to use vmid 0 */ 3800 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0); 3801 3802 /* Init gfx ring 0 for pipe 0 */ 3803 mutex_lock(&adev->srbm_mutex); 3804 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3805 3806 /* Set ring buffer size */ 3807 ring = &adev->gfx.gfx_ring[0]; 3808 rb_bufsz = order_base_2(ring->ring_size / 8); 3809 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3810 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3811 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, KMD_QUEUE, 1); 3812 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3813 3814 /* Initialize the ring buffer's write pointers */ 3815 ring->wptr = 0; 3816 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3817 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3818 3819 /* set the wb address whether it's enabled or not */ 3820 rptr_addr = ring->rptr_gpu_addr; 3821 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3822 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3823 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3824 3825 wptr_gpu_addr = ring->wptr_gpu_addr; 3826 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3827 lower_32_bits(wptr_gpu_addr)); 3828 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3829 upper_32_bits(wptr_gpu_addr)); 3830 3831 mdelay(1); 3832 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3833 3834 rb_addr = ring->gpu_addr >> 8; 3835 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr); 3836 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3837 3838 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1); 3839 3840 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3841 mutex_unlock(&adev->srbm_mutex); 3842 3843 /* Init gfx ring 1 for pipe 1 */ 3844 if (adev->gfx.num_gfx_rings > 1) { 3845 mutex_lock(&adev->srbm_mutex); 3846 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1); 3847 /* maximum supported gfx ring is 2 */ 3848 ring = &adev->gfx.gfx_ring[1]; 3849 rb_bufsz = order_base_2(ring->ring_size / 8); 3850 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz); 3851 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2); 3852 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3853 /* Initialize the ring buffer's write pointers */ 3854 ring->wptr = 0; 3855 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr)); 3856 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr)); 3857 /* Set the wb address whether it's enabled or not */ 3858 rptr_addr = ring->rptr_gpu_addr; 3859 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr)); 3860 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3861 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3862 wptr_gpu_addr = ring->wptr_gpu_addr; 3863 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3864 lower_32_bits(wptr_gpu_addr)); 3865 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3866 upper_32_bits(wptr_gpu_addr)); 3867 3868 mdelay(1); 3869 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3870 3871 rb_addr = ring->gpu_addr >> 8; 3872 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr); 3873 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr)); 3874 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1); 3875 3876 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3877 mutex_unlock(&adev->srbm_mutex); 3878 } 3879 /* Switch to pipe 0 */ 3880 mutex_lock(&adev->srbm_mutex); 3881 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3882 mutex_unlock(&adev->srbm_mutex); 3883 3884 /* start the ring */ 3885 gfx_v11_0_cp_gfx_start(adev); 3886 3887 return 0; 3888 } 3889 3890 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3891 { 3892 u32 data; 3893 3894 if (adev->gfx.rs64_enable) { 3895 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3896 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE, 3897 enable ? 0 : 1); 3898 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 3899 enable ? 0 : 1); 3900 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 3901 enable ? 0 : 1); 3902 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 3903 enable ? 0 : 1); 3904 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 3905 enable ? 0 : 1); 3906 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE, 3907 enable ? 1 : 0); 3908 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE, 3909 enable ? 1 : 0); 3910 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE, 3911 enable ? 1 : 0); 3912 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE, 3913 enable ? 1 : 0); 3914 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT, 3915 enable ? 0 : 1); 3916 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data); 3917 } else { 3918 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); 3919 3920 if (enable) { 3921 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0); 3922 if (!adev->enable_mes_kiq) 3923 data = REG_SET_FIELD(data, CP_MEC_CNTL, 3924 MEC_ME2_HALT, 0); 3925 } else { 3926 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1); 3927 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1); 3928 } 3929 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data); 3930 } 3931 3932 udelay(50); 3933 } 3934 3935 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3936 { 3937 const struct gfx_firmware_header_v1_0 *mec_hdr; 3938 const __le32 *fw_data; 3939 unsigned i, fw_size; 3940 u32 *fw = NULL; 3941 int r; 3942 3943 if (!adev->gfx.mec_fw) 3944 return -EINVAL; 3945 3946 gfx_v11_0_cp_compute_enable(adev, false); 3947 3948 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3949 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3950 3951 fw_data = (const __le32 *) 3952 (adev->gfx.mec_fw->data + 3953 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3954 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 3955 3956 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 3957 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3958 &adev->gfx.mec.mec_fw_obj, 3959 &adev->gfx.mec.mec_fw_gpu_addr, 3960 (void **)&fw); 3961 if (r) { 3962 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r); 3963 gfx_v11_0_mec_fini(adev); 3964 return r; 3965 } 3966 3967 memcpy(fw, fw_data, fw_size); 3968 3969 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 3970 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 3971 3972 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr); 3973 3974 /* MEC1 */ 3975 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0); 3976 3977 for (i = 0; i < mec_hdr->jt_size; i++) 3978 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA, 3979 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3980 3981 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version); 3982 3983 return 0; 3984 } 3985 3986 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) 3987 { 3988 const struct gfx_firmware_header_v2_0 *mec_hdr; 3989 const __le32 *fw_ucode, *fw_data; 3990 u32 tmp, fw_ucode_size, fw_data_size; 3991 u32 i, usec_timeout = 50000; /* Wait for 50 ms */ 3992 u32 *fw_ucode_ptr, *fw_data_ptr; 3993 int r; 3994 3995 if (!adev->gfx.mec_fw) 3996 return -EINVAL; 3997 3998 gfx_v11_0_cp_compute_enable(adev, false); 3999 4000 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; 4001 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 4002 4003 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data + 4004 le32_to_cpu(mec_hdr->ucode_offset_bytes)); 4005 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes); 4006 4007 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 4008 le32_to_cpu(mec_hdr->data_offset_bytes)); 4009 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes); 4010 4011 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 4012 64 * 1024, 4013 AMDGPU_GEM_DOMAIN_VRAM | 4014 AMDGPU_GEM_DOMAIN_GTT, 4015 &adev->gfx.mec.mec_fw_obj, 4016 &adev->gfx.mec.mec_fw_gpu_addr, 4017 (void **)&fw_ucode_ptr); 4018 if (r) { 4019 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 4020 gfx_v11_0_mec_fini(adev); 4021 return r; 4022 } 4023 4024 r = amdgpu_bo_create_reserved(adev, fw_data_size, 4025 64 * 1024, 4026 AMDGPU_GEM_DOMAIN_VRAM | 4027 AMDGPU_GEM_DOMAIN_GTT, 4028 &adev->gfx.mec.mec_fw_data_obj, 4029 &adev->gfx.mec.mec_fw_data_gpu_addr, 4030 (void **)&fw_data_ptr); 4031 if (r) { 4032 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 4033 gfx_v11_0_mec_fini(adev); 4034 return r; 4035 } 4036 4037 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); 4038 memcpy(fw_data_ptr, fw_data, fw_data_size); 4039 4040 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 4041 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj); 4042 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 4043 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj); 4044 4045 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 4046 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 4047 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 4048 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 4049 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 4050 4051 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 4052 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 4053 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 4054 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 4055 4056 mutex_lock(&adev->srbm_mutex); 4057 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 4058 soc21_grbm_select(adev, 1, i, 0, 0); 4059 4060 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr); 4061 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 4062 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr)); 4063 4064 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 4065 mec_hdr->ucode_start_addr_lo >> 2 | 4066 mec_hdr->ucode_start_addr_hi << 30); 4067 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 4068 mec_hdr->ucode_start_addr_hi >> 2); 4069 4070 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr); 4071 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 4072 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 4073 } 4074 mutex_unlock(&adev->srbm_mutex); 4075 soc21_grbm_select(adev, 0, 0, 0, 0); 4076 4077 /* Trigger an invalidation of the L1 instruction caches */ 4078 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4079 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 4080 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 4081 4082 /* Wait for invalidation complete */ 4083 for (i = 0; i < usec_timeout; i++) { 4084 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4085 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 4086 INVALIDATE_DCACHE_COMPLETE)) 4087 break; 4088 udelay(1); 4089 } 4090 4091 if (i >= usec_timeout) { 4092 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4093 return -EINVAL; 4094 } 4095 4096 /* Trigger an invalidation of the L1 instruction caches */ 4097 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4098 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 4099 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 4100 4101 /* Wait for invalidation complete */ 4102 for (i = 0; i < usec_timeout; i++) { 4103 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4104 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 4105 INVALIDATE_CACHE_COMPLETE)) 4106 break; 4107 udelay(1); 4108 } 4109 4110 if (i >= usec_timeout) { 4111 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4112 return -EINVAL; 4113 } 4114 4115 return 0; 4116 } 4117 4118 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring) 4119 { 4120 uint32_t tmp; 4121 struct amdgpu_device *adev = ring->adev; 4122 4123 /* tell RLC which is KIQ queue */ 4124 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); 4125 tmp &= 0xffffff00; 4126 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4127 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80); 4128 } 4129 4130 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev) 4131 { 4132 /* set graphics engine doorbell range */ 4133 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, 4134 (adev->doorbell_index.gfx_ring0 * 2) << 2); 4135 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 4136 (adev->doorbell_index.gfx_userqueue_end * 2) << 2); 4137 4138 /* set compute engine doorbell range */ 4139 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4140 (adev->doorbell_index.kiq * 2) << 2); 4141 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4142 (adev->doorbell_index.userqueue_end * 2) << 2); 4143 } 4144 4145 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev, 4146 struct v11_gfx_mqd *mqd, 4147 struct amdgpu_mqd_prop *prop) 4148 { 4149 bool priority = 0; 4150 u32 tmp; 4151 4152 /* set up default queue priority level 4153 * 0x0 = low priority, 0x1 = high priority 4154 */ 4155 if (prop->hqd_queue_priority == AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM) 4156 priority = 1; 4157 4158 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT; 4159 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority); 4160 mqd->cp_gfx_hqd_queue_priority = tmp; 4161 } 4162 4163 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, 4164 struct amdgpu_mqd_prop *prop) 4165 { 4166 struct v11_gfx_mqd *mqd = m; 4167 uint64_t hqd_gpu_addr, wb_gpu_addr; 4168 uint32_t tmp; 4169 uint32_t rb_bufsz; 4170 4171 /* set up gfx hqd wptr */ 4172 mqd->cp_gfx_hqd_wptr = 0; 4173 mqd->cp_gfx_hqd_wptr_hi = 0; 4174 4175 /* set the pointer to the MQD */ 4176 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc; 4177 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4178 4179 /* set up mqd control */ 4180 tmp = regCP_GFX_MQD_CONTROL_DEFAULT; 4181 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0); 4182 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1); 4183 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0); 4184 mqd->cp_gfx_mqd_control = tmp; 4185 4186 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */ 4187 tmp = regCP_GFX_HQD_VMID_DEFAULT; 4188 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0); 4189 mqd->cp_gfx_hqd_vmid = 0; 4190 4191 /* set up gfx queue priority */ 4192 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop); 4193 4194 /* set up time quantum */ 4195 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT; 4196 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1); 4197 mqd->cp_gfx_hqd_quantum = tmp; 4198 4199 /* set up gfx hqd base. this is similar as CP_RB_BASE */ 4200 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4201 mqd->cp_gfx_hqd_base = hqd_gpu_addr; 4202 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr); 4203 4204 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */ 4205 wb_gpu_addr = prop->rptr_gpu_addr; 4206 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc; 4207 mqd->cp_gfx_hqd_rptr_addr_hi = 4208 upper_32_bits(wb_gpu_addr) & 0xffff; 4209 4210 /* set up rb_wptr_poll addr */ 4211 wb_gpu_addr = prop->wptr_gpu_addr; 4212 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4213 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4214 4215 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */ 4216 rb_bufsz = order_base_2(prop->queue_size / 4) - 1; 4217 tmp = regCP_GFX_HQD_CNTL_DEFAULT; 4218 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz); 4219 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2); 4220 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, KMD_QUEUE, prop->kernel_queue); 4221 #ifdef __BIG_ENDIAN 4222 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1); 4223 #endif 4224 if (prop->tmz_queue) 4225 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, TMZ_MATCH, 1); 4226 if (!prop->kernel_queue) 4227 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_NON_PRIV, 1); 4228 mqd->cp_gfx_hqd_cntl = tmp; 4229 4230 /* set up cp_doorbell_control */ 4231 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT; 4232 if (prop->use_doorbell) { 4233 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4234 DOORBELL_OFFSET, prop->doorbell_index); 4235 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4236 DOORBELL_EN, 1); 4237 } else 4238 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4239 DOORBELL_EN, 0); 4240 mqd->cp_rb_doorbell_control = tmp; 4241 4242 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4243 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT; 4244 4245 /* active the queue */ 4246 mqd->cp_gfx_hqd_active = 1; 4247 4248 /* set gfx UQ items */ 4249 mqd->shadow_base_lo = lower_32_bits(prop->shadow_addr); 4250 mqd->shadow_base_hi = upper_32_bits(prop->shadow_addr); 4251 mqd->gds_bkup_base_lo = lower_32_bits(prop->gds_bkup_addr); 4252 mqd->gds_bkup_base_hi = upper_32_bits(prop->gds_bkup_addr); 4253 mqd->fw_work_area_base_lo = lower_32_bits(prop->csa_addr); 4254 mqd->fw_work_area_base_hi = upper_32_bits(prop->csa_addr); 4255 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4256 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4257 4258 return 0; 4259 } 4260 4261 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring) 4262 { 4263 struct amdgpu_device *adev = ring->adev; 4264 struct v11_gfx_mqd *mqd = ring->mqd_ptr; 4265 int mqd_idx = ring - &adev->gfx.gfx_ring[0]; 4266 4267 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4268 memset((void *)mqd, 0, sizeof(*mqd)); 4269 mutex_lock(&adev->srbm_mutex); 4270 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4271 amdgpu_ring_init_mqd(ring); 4272 soc21_grbm_select(adev, 0, 0, 0, 0); 4273 mutex_unlock(&adev->srbm_mutex); 4274 if (adev->gfx.me.mqd_backup[mqd_idx]) 4275 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4276 } else { 4277 /* restore mqd with the backup copy */ 4278 if (adev->gfx.me.mqd_backup[mqd_idx]) 4279 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); 4280 /* reset the ring */ 4281 ring->wptr = 0; 4282 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4283 amdgpu_ring_clear_ring(ring); 4284 } 4285 4286 return 0; 4287 } 4288 4289 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) 4290 { 4291 int r, i; 4292 4293 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4294 r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i]); 4295 if (r) 4296 return r; 4297 } 4298 4299 r = amdgpu_gfx_enable_kgq(adev, 0); 4300 if (r) 4301 return r; 4302 4303 return gfx_v11_0_cp_gfx_start(adev); 4304 } 4305 4306 static void gfx_v11_0_compute_mqd_set_cu_mask(struct amdgpu_device *adev, 4307 struct v11_compute_mqd *mqd, 4308 struct amdgpu_mqd_prop *prop) 4309 { 4310 uint32_t se_mask[8] = {0}; 4311 uint32_t wa_mask; 4312 bool has_wa_flag = prop->cu_flags & (AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE | 4313 AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE); 4314 4315 if (!has_wa_flag && (!prop->cu_mask || !prop->cu_mask_count)) 4316 return; 4317 4318 if (has_wa_flag) { 4319 wa_mask = (prop->cu_flags & AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE) ? 4320 0xffff : 0xffffffff; 4321 mqd->compute_static_thread_mgmt_se0 = wa_mask; 4322 mqd->compute_static_thread_mgmt_se1 = wa_mask; 4323 mqd->compute_static_thread_mgmt_se2 = wa_mask; 4324 mqd->compute_static_thread_mgmt_se3 = wa_mask; 4325 return; 4326 } 4327 4328 amdgpu_gfx_mqd_symmetrically_map_cu_mask(adev, prop->cu_mask, 4329 prop->cu_mask_count, se_mask); 4330 4331 mqd->compute_static_thread_mgmt_se0 = se_mask[0]; 4332 mqd->compute_static_thread_mgmt_se1 = se_mask[1]; 4333 mqd->compute_static_thread_mgmt_se2 = se_mask[2]; 4334 mqd->compute_static_thread_mgmt_se3 = se_mask[3]; 4335 } 4336 4337 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m, 4338 struct amdgpu_mqd_prop *prop) 4339 { 4340 struct v11_compute_mqd *mqd = m; 4341 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4342 uint32_t tmp; 4343 4344 mqd->header = 0xC0310800; 4345 mqd->compute_pipelinestat_enable = 0x00000001; 4346 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4347 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4348 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4349 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4350 mqd->compute_misc_reserved = 0x00000007; 4351 4352 eop_base_addr = prop->eop_gpu_addr >> 8; 4353 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4354 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4355 4356 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4357 tmp = regCP_HQD_EOP_CONTROL_DEFAULT; 4358 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4359 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1)); 4360 4361 mqd->cp_hqd_eop_control = tmp; 4362 4363 /* enable doorbell? */ 4364 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4365 4366 if (prop->use_doorbell) { 4367 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4368 DOORBELL_OFFSET, prop->doorbell_index); 4369 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4370 DOORBELL_EN, 1); 4371 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4372 DOORBELL_SOURCE, 0); 4373 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4374 DOORBELL_HIT, 0); 4375 } else { 4376 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4377 DOORBELL_EN, 0); 4378 } 4379 4380 mqd->cp_hqd_pq_doorbell_control = tmp; 4381 4382 /* disable the queue if it's active */ 4383 mqd->cp_hqd_dequeue_request = 0; 4384 mqd->cp_hqd_pq_rptr = 0; 4385 mqd->cp_hqd_pq_wptr_lo = 0; 4386 mqd->cp_hqd_pq_wptr_hi = 0; 4387 4388 /* set the pointer to the MQD */ 4389 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc; 4390 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4391 4392 /* set MQD vmid to 0 */ 4393 tmp = regCP_MQD_CONTROL_DEFAULT; 4394 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4395 mqd->cp_mqd_control = tmp; 4396 4397 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4398 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4399 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4400 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4401 4402 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4403 tmp = regCP_HQD_PQ_CONTROL_DEFAULT; 4404 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4405 (order_base_2(prop->queue_size / 4) - 1)); 4406 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4407 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4408 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1); 4409 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 4410 prop->allow_tunneling); 4411 if (prop->kernel_queue) { 4412 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4413 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4414 } 4415 if (prop->tmz_queue) 4416 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TMZ, 1); 4417 mqd->cp_hqd_pq_control = tmp; 4418 4419 /* set the wb address whether it's enabled or not */ 4420 wb_gpu_addr = prop->rptr_gpu_addr; 4421 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4422 mqd->cp_hqd_pq_rptr_report_addr_hi = 4423 upper_32_bits(wb_gpu_addr) & 0xffff; 4424 4425 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4426 wb_gpu_addr = prop->wptr_gpu_addr; 4427 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4428 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4429 4430 tmp = 0; 4431 /* enable the doorbell if requested */ 4432 if (prop->use_doorbell) { 4433 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4434 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4435 DOORBELL_OFFSET, prop->doorbell_index); 4436 4437 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4438 DOORBELL_EN, 1); 4439 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4440 DOORBELL_SOURCE, 0); 4441 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4442 DOORBELL_HIT, 0); 4443 } 4444 4445 mqd->cp_hqd_pq_doorbell_control = tmp; 4446 4447 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4448 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT; 4449 4450 /* set the vmid for the queue */ 4451 mqd->cp_hqd_vmid = 0; 4452 4453 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT; 4454 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55); 4455 mqd->cp_hqd_persistent_state = tmp; 4456 4457 /* set MIN_IB_AVAIL_SIZE */ 4458 tmp = regCP_HQD_IB_CONTROL_DEFAULT; 4459 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4460 mqd->cp_hqd_ib_control = tmp; 4461 4462 /* set static priority for a compute queue/ring */ 4463 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; 4464 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; 4465 4466 tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); 4467 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 4468 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); 4469 mqd->cp_hqd_quantum = tmp; 4470 4471 mqd->cp_hqd_active = prop->hqd_active; 4472 4473 /* set UQ fenceaddress */ 4474 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4475 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4476 /* set CU mask */ 4477 gfx_v11_0_compute_mqd_set_cu_mask(adev, mqd, prop); 4478 4479 return 0; 4480 } 4481 4482 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring) 4483 { 4484 struct amdgpu_device *adev = ring->adev; 4485 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4486 int j; 4487 4488 /* inactivate the queue */ 4489 if (amdgpu_sriov_vf(adev)) 4490 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); 4491 4492 /* disable wptr polling */ 4493 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4494 4495 /* write the EOP addr */ 4496 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR, 4497 mqd->cp_hqd_eop_base_addr_lo); 4498 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI, 4499 mqd->cp_hqd_eop_base_addr_hi); 4500 4501 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4502 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL, 4503 mqd->cp_hqd_eop_control); 4504 4505 /* enable doorbell? */ 4506 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4507 mqd->cp_hqd_pq_doorbell_control); 4508 4509 /* disable the queue if it's active */ 4510 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { 4511 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); 4512 for (j = 0; j < adev->usec_timeout; j++) { 4513 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) 4514 break; 4515 udelay(1); 4516 } 4517 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 4518 mqd->cp_hqd_dequeue_request); 4519 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 4520 mqd->cp_hqd_pq_rptr); 4521 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4522 mqd->cp_hqd_pq_wptr_lo); 4523 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4524 mqd->cp_hqd_pq_wptr_hi); 4525 } 4526 4527 /* set the pointer to the MQD */ 4528 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, 4529 mqd->cp_mqd_base_addr_lo); 4530 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, 4531 mqd->cp_mqd_base_addr_hi); 4532 4533 /* set MQD vmid to 0 */ 4534 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 4535 mqd->cp_mqd_control); 4536 4537 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4538 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, 4539 mqd->cp_hqd_pq_base_lo); 4540 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, 4541 mqd->cp_hqd_pq_base_hi); 4542 4543 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4544 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, 4545 mqd->cp_hqd_pq_control); 4546 4547 /* set the wb address whether it's enabled or not */ 4548 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR, 4549 mqd->cp_hqd_pq_rptr_report_addr_lo); 4550 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4551 mqd->cp_hqd_pq_rptr_report_addr_hi); 4552 4553 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4554 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR, 4555 mqd->cp_hqd_pq_wptr_poll_addr_lo); 4556 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI, 4557 mqd->cp_hqd_pq_wptr_poll_addr_hi); 4558 4559 /* enable the doorbell if requested */ 4560 if (ring->use_doorbell) { 4561 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4562 (adev->doorbell_index.kiq * 2) << 2); 4563 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4564 (adev->doorbell_index.userqueue_end * 2) << 2); 4565 } 4566 4567 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4568 mqd->cp_hqd_pq_doorbell_control); 4569 4570 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4571 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4572 mqd->cp_hqd_pq_wptr_lo); 4573 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4574 mqd->cp_hqd_pq_wptr_hi); 4575 4576 /* set the vmid for the queue */ 4577 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid); 4578 4579 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, 4580 mqd->cp_hqd_persistent_state); 4581 4582 /* activate the queue */ 4583 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 4584 mqd->cp_hqd_active); 4585 4586 if (ring->use_doorbell) 4587 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4588 4589 return 0; 4590 } 4591 4592 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) 4593 { 4594 struct amdgpu_device *adev = ring->adev; 4595 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4596 4597 gfx_v11_0_kiq_setting(ring); 4598 4599 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4600 /* reset MQD to a clean status */ 4601 if (adev->gfx.kiq[0].mqd_backup) 4602 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd)); 4603 4604 /* reset ring buffer */ 4605 ring->wptr = 0; 4606 amdgpu_ring_clear_ring(ring); 4607 4608 mutex_lock(&adev->srbm_mutex); 4609 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4610 gfx_v11_0_kiq_init_register(ring); 4611 soc21_grbm_select(adev, 0, 0, 0, 0); 4612 mutex_unlock(&adev->srbm_mutex); 4613 } else { 4614 memset((void *)mqd, 0, sizeof(*mqd)); 4615 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4616 amdgpu_ring_clear_ring(ring); 4617 mutex_lock(&adev->srbm_mutex); 4618 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4619 amdgpu_ring_init_mqd(ring); 4620 gfx_v11_0_kiq_init_register(ring); 4621 soc21_grbm_select(adev, 0, 0, 0, 0); 4622 mutex_unlock(&adev->srbm_mutex); 4623 4624 if (adev->gfx.kiq[0].mqd_backup) 4625 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd)); 4626 } 4627 4628 return 0; 4629 } 4630 4631 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring) 4632 { 4633 struct amdgpu_device *adev = ring->adev; 4634 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4635 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4636 4637 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4638 memset((void *)mqd, 0, sizeof(*mqd)); 4639 mutex_lock(&adev->srbm_mutex); 4640 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4641 amdgpu_ring_init_mqd(ring); 4642 soc21_grbm_select(adev, 0, 0, 0, 0); 4643 mutex_unlock(&adev->srbm_mutex); 4644 4645 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4646 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4647 } else { 4648 /* restore MQD to a clean status */ 4649 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4650 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); 4651 /* reset ring buffer */ 4652 ring->wptr = 0; 4653 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4654 amdgpu_ring_clear_ring(ring); 4655 } 4656 4657 return 0; 4658 } 4659 4660 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev) 4661 { 4662 gfx_v11_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 4663 return 0; 4664 } 4665 4666 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev) 4667 { 4668 int i, r; 4669 4670 if (!amdgpu_async_gfx_ring) 4671 gfx_v11_0_cp_compute_enable(adev, true); 4672 4673 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4674 r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i]); 4675 if (r) 4676 return r; 4677 } 4678 4679 return amdgpu_gfx_enable_kcq(adev, 0); 4680 } 4681 4682 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev) 4683 { 4684 int r, i; 4685 struct amdgpu_ring *ring; 4686 4687 if (!(adev->flags & AMD_IS_APU)) 4688 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 4689 4690 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4691 /* legacy firmware loading */ 4692 r = gfx_v11_0_cp_gfx_load_microcode(adev); 4693 if (r) 4694 return r; 4695 4696 if (adev->gfx.rs64_enable) 4697 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev); 4698 else 4699 r = gfx_v11_0_cp_compute_load_microcode(adev); 4700 if (r) 4701 return r; 4702 } 4703 4704 gfx_v11_0_cp_set_doorbell_range(adev); 4705 4706 if (amdgpu_async_gfx_ring) { 4707 gfx_v11_0_cp_compute_enable(adev, true); 4708 gfx_v11_0_cp_gfx_enable(adev, true); 4709 } 4710 4711 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) 4712 r = amdgpu_mes_kiq_hw_init(adev, 0); 4713 else 4714 r = gfx_v11_0_kiq_resume(adev); 4715 if (r) 4716 return r; 4717 4718 r = gfx_v11_0_kcq_resume(adev); 4719 if (r) 4720 return r; 4721 4722 if (!amdgpu_async_gfx_ring) { 4723 r = gfx_v11_0_cp_gfx_resume(adev); 4724 if (r) 4725 return r; 4726 } else { 4727 r = gfx_v11_0_cp_async_gfx_ring_resume(adev); 4728 if (r) 4729 return r; 4730 } 4731 4732 if (adev->gfx.disable_kq) { 4733 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4734 ring = &adev->gfx.gfx_ring[i]; 4735 /* we don't want to set ring->ready */ 4736 r = amdgpu_ring_test_ring(ring); 4737 if (r) 4738 return r; 4739 } 4740 if (amdgpu_async_gfx_ring) 4741 amdgpu_gfx_disable_kgq(adev, 0); 4742 } else { 4743 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4744 ring = &adev->gfx.gfx_ring[i]; 4745 r = amdgpu_ring_test_helper(ring); 4746 if (r) 4747 return r; 4748 } 4749 } 4750 4751 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4752 ring = &adev->gfx.compute_ring[i]; 4753 r = amdgpu_ring_test_helper(ring); 4754 if (r) 4755 return r; 4756 } 4757 4758 return 0; 4759 } 4760 4761 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable) 4762 { 4763 gfx_v11_0_cp_gfx_enable(adev, enable); 4764 gfx_v11_0_cp_compute_enable(adev, enable); 4765 } 4766 4767 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev) 4768 { 4769 int r; 4770 bool value; 4771 4772 r = adev->gfxhub.funcs->gart_enable(adev); 4773 if (r) 4774 return r; 4775 4776 amdgpu_device_flush_hdp(adev, NULL); 4777 4778 value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS; 4779 4780 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 4781 /* TODO investigate why this and the hdp flush above is needed, 4782 * are we missing a flush somewhere else? */ 4783 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0); 4784 4785 return 0; 4786 } 4787 4788 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev) 4789 { 4790 u32 tmp; 4791 4792 /* select RS64 */ 4793 if (adev->gfx.rs64_enable) { 4794 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL); 4795 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1); 4796 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp); 4797 4798 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL); 4799 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1); 4800 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp); 4801 } 4802 4803 if (amdgpu_emu_mode == 1) 4804 msleep(100); 4805 } 4806 4807 static int get_gb_addr_config(struct amdgpu_device * adev) 4808 { 4809 u32 gb_addr_config; 4810 4811 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG); 4812 if (gb_addr_config == 0) 4813 return -EINVAL; 4814 4815 adev->gfx.config.gb_addr_config_fields.num_pkrs = 4816 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS); 4817 4818 adev->gfx.config.gb_addr_config = gb_addr_config; 4819 4820 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 4821 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4822 GB_ADDR_CONFIG, NUM_PIPES); 4823 4824 adev->gfx.config.max_tile_pipes = 4825 adev->gfx.config.gb_addr_config_fields.num_pipes; 4826 4827 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 4828 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4829 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS); 4830 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 4831 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4832 GB_ADDR_CONFIG, NUM_RB_PER_SE); 4833 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 4834 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4835 GB_ADDR_CONFIG, NUM_SHADER_ENGINES); 4836 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 4837 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4838 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE)); 4839 4840 return 0; 4841 } 4842 4843 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev) 4844 { 4845 uint32_t data; 4846 4847 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG); 4848 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK; 4849 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data); 4850 4851 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG); 4852 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK; 4853 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); 4854 } 4855 4856 static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, 4857 bool enable) 4858 { 4859 unsigned int irq_type; 4860 int m, p, r; 4861 4862 if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { 4863 for (m = 0; m < adev->gfx.me.num_me; m++) { 4864 for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { 4865 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4866 if (enable) 4867 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4868 else 4869 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4870 if (r) { 4871 if (!enable) 4872 return r; 4873 goto err_gfx; 4874 } 4875 } 4876 } 4877 } 4878 4879 if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { 4880 for (m = 0; m < adev->gfx.mec.num_mec; ++m) { 4881 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { 4882 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4883 + (m * adev->gfx.mec.num_pipe_per_mec) 4884 + p; 4885 if (enable) 4886 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4887 else 4888 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4889 if (r) { 4890 if (!enable) 4891 return r; 4892 goto err_compute; 4893 } 4894 } 4895 } 4896 } 4897 4898 return 0; 4899 4900 err_compute: 4901 for (p--; p >= 0; p--) { 4902 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4903 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4904 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4905 } 4906 for (m--; m >= 0; m--) { 4907 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { 4908 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4909 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4910 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4911 } 4912 } 4913 m = adev->gfx.me.num_me; 4914 err_gfx: 4915 for (p--; p >= 0; p--) { 4916 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4917 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4918 } 4919 for (m--; m >= 0; m--) { 4920 for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { 4921 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4922 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4923 } 4924 } 4925 return r; 4926 } 4927 4928 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) 4929 { 4930 int r; 4931 struct amdgpu_device *adev = ip_block->adev; 4932 4933 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, 4934 adev->gfx.cleaner_shader_ptr); 4935 4936 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 4937 if (adev->gfx.imu.funcs) { 4938 /* RLC autoload sequence 1: Program rlc ram */ 4939 if (adev->gfx.imu.funcs->program_rlc_ram) 4940 adev->gfx.imu.funcs->program_rlc_ram(adev); 4941 /* rlc autoload firmware */ 4942 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev); 4943 if (r) 4944 return r; 4945 } 4946 } else { 4947 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4948 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) { 4949 if (adev->gfx.imu.funcs->load_microcode) 4950 adev->gfx.imu.funcs->load_microcode(adev); 4951 if (adev->gfx.imu.funcs->setup_imu) 4952 adev->gfx.imu.funcs->setup_imu(adev); 4953 if (adev->gfx.imu.funcs->start_imu) 4954 adev->gfx.imu.funcs->start_imu(adev); 4955 } 4956 4957 /* disable gpa mode in backdoor loading */ 4958 gfx_v11_0_disable_gpa_mode(adev); 4959 } 4960 } 4961 4962 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) || 4963 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 4964 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev); 4965 if (r) { 4966 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r); 4967 return r; 4968 } 4969 } 4970 4971 adev->gfx.is_poweron = true; 4972 4973 if(get_gb_addr_config(adev)) 4974 drm_warn(adev_to_drm(adev), "Invalid gb_addr_config !\n"); 4975 4976 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP && 4977 adev->gfx.rs64_enable) 4978 gfx_v11_0_config_gfx_rs64(adev); 4979 4980 r = gfx_v11_0_gfxhub_enable(adev); 4981 if (r) 4982 return r; 4983 4984 if (!amdgpu_emu_mode) 4985 gfx_v11_0_init_golden_registers(adev); 4986 4987 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || 4988 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) { 4989 /** 4990 * For gfx 11, rlc firmware loading relies on smu firmware is 4991 * loaded firstly, so in direct type, it has to load smc ucode 4992 * here before rlc. 4993 */ 4994 r = amdgpu_pm_load_smu_firmware(adev, NULL); 4995 if (r) 4996 return r; 4997 } 4998 4999 gfx_v11_0_constants_init(adev); 5000 5001 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 5002 gfx_v11_0_select_cp_fw_arch(adev); 5003 5004 if (adev->nbio.funcs->gc_doorbell_init) 5005 adev->nbio.funcs->gc_doorbell_init(adev); 5006 5007 r = gfx_v11_0_rlc_resume(adev); 5008 if (r) 5009 return r; 5010 5011 /* 5012 * init golden registers and rlc resume may override some registers, 5013 * reconfig them here 5014 */ 5015 gfx_v11_0_tcp_harvest(adev); 5016 5017 r = gfx_v11_0_cp_resume(adev); 5018 if (r) 5019 return r; 5020 5021 /* get IMU version from HW if it's not set */ 5022 if (!adev->gfx.imu_fw_version) 5023 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0); 5024 5025 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5026 if (r) 5027 return r; 5028 5029 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5030 if (r) 5031 goto err_priv_inst; 5032 5033 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); 5034 if (r) 5035 goto err_bad_op; 5036 5037 r = gfx_v11_0_set_userq_eop_interrupts(adev, true); 5038 if (r) 5039 goto err_userq_eop; 5040 5041 return 0; 5042 5043 err_userq_eop: 5044 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5045 err_bad_op: 5046 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5047 err_priv_inst: 5048 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5049 return r; 5050 } 5051 5052 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) 5053 { 5054 struct amdgpu_device *adev = ip_block->adev; 5055 5056 cancel_delayed_work_sync(&adev->gfx.idle_work); 5057 5058 gfx_v11_0_set_userq_eop_interrupts(adev, false); 5059 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5060 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5061 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5062 5063 if (!adev->no_hw_access) { 5064 if (amdgpu_async_gfx_ring && 5065 !adev->gfx.disable_kq) { 5066 if (amdgpu_gfx_disable_kgq(adev, 0)) 5067 DRM_ERROR("KGQ disable failed\n"); 5068 } 5069 5070 if (amdgpu_gfx_disable_kcq(adev, 0)) 5071 DRM_ERROR("KCQ disable failed\n"); 5072 5073 amdgpu_mes_kiq_hw_fini(adev, 0); 5074 } 5075 5076 if (amdgpu_sriov_vf(adev)) 5077 /* Remove the steps disabling CPG and clearing KIQ position, 5078 * so that CP could perform IDLE-SAVE during switch. Those 5079 * steps are necessary to avoid a DMAR error in gfx9 but it is 5080 * not reproduced on gfx11. 5081 */ 5082 return 0; 5083 5084 gfx_v11_0_cp_enable(adev, false); 5085 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 5086 5087 adev->gfxhub.funcs->gart_disable(adev); 5088 5089 adev->gfx.is_poweron = false; 5090 5091 return 0; 5092 } 5093 5094 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block) 5095 { 5096 return gfx_v11_0_hw_fini(ip_block); 5097 } 5098 5099 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block) 5100 { 5101 return gfx_v11_0_hw_init(ip_block); 5102 } 5103 5104 static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block) 5105 { 5106 struct amdgpu_device *adev = ip_block->adev; 5107 5108 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS), 5109 GRBM_STATUS, GUI_ACTIVE)) 5110 return false; 5111 else 5112 return true; 5113 } 5114 5115 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 5116 { 5117 unsigned i; 5118 u32 tmp; 5119 struct amdgpu_device *adev = ip_block->adev; 5120 5121 for (i = 0; i < adev->usec_timeout; i++) { 5122 /* read MC_STATUS */ 5123 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) & 5124 GRBM_STATUS__GUI_ACTIVE_MASK; 5125 5126 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE)) 5127 return 0; 5128 udelay(1); 5129 } 5130 return -ETIMEDOUT; 5131 } 5132 5133 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev, 5134 bool req) 5135 { 5136 u32 i, tmp, val; 5137 5138 for (i = 0; i < adev->usec_timeout; i++) { 5139 /* Request with MeId=2, PipeId=0 */ 5140 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req); 5141 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4); 5142 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp); 5143 5144 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX); 5145 if (req) { 5146 if (val == tmp) 5147 break; 5148 } else { 5149 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, 5150 REQUEST, 1); 5151 5152 /* unlocked or locked by firmware */ 5153 if (val != tmp) 5154 break; 5155 } 5156 udelay(1); 5157 } 5158 5159 if (i >= adev->usec_timeout) 5160 return -EINVAL; 5161 5162 return 0; 5163 } 5164 5165 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block) 5166 { 5167 u32 grbm_soft_reset = 0; 5168 u32 tmp; 5169 int r, i, j, k; 5170 struct amdgpu_device *adev = ip_block->adev; 5171 5172 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5173 5174 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5175 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0); 5176 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0); 5177 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0); 5178 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0); 5179 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5180 5181 mutex_lock(&adev->srbm_mutex); 5182 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 5183 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 5184 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 5185 soc21_grbm_select(adev, i, k, j, 0); 5186 5187 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2); 5188 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1); 5189 } 5190 } 5191 } 5192 for (i = 0; i < adev->gfx.me.num_me; ++i) { 5193 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { 5194 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 5195 soc21_grbm_select(adev, i, k, j, 0); 5196 5197 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1); 5198 } 5199 } 5200 } 5201 soc21_grbm_select(adev, 0, 0, 0, 0); 5202 mutex_unlock(&adev->srbm_mutex); 5203 5204 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */ 5205 mutex_lock(&adev->gfx.reset_sem_mutex); 5206 r = gfx_v11_0_request_gfx_index_mutex(adev, true); 5207 if (r) { 5208 mutex_unlock(&adev->gfx.reset_sem_mutex); 5209 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n"); 5210 return r; 5211 } 5212 5213 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe); 5214 5215 // Read CP_VMID_RESET register three times. 5216 // to get sufficient time for GFX_HQD_ACTIVE reach 0 5217 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5218 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5219 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5220 5221 /* release the gfx mutex */ 5222 r = gfx_v11_0_request_gfx_index_mutex(adev, false); 5223 mutex_unlock(&adev->gfx.reset_sem_mutex); 5224 if (r) { 5225 DRM_ERROR("Failed to release the gfx mutex during soft reset\n"); 5226 return r; 5227 } 5228 5229 for (i = 0; i < adev->usec_timeout; i++) { 5230 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) && 5231 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE)) 5232 break; 5233 udelay(1); 5234 } 5235 if (i >= adev->usec_timeout) { 5236 printk("Failed to wait all pipes clean\n"); 5237 return -EINVAL; 5238 } 5239 5240 /********** trigger soft reset ***********/ 5241 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5242 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5243 SOFT_RESET_CP, 1); 5244 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5245 SOFT_RESET_GFX, 1); 5246 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5247 SOFT_RESET_CPF, 1); 5248 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5249 SOFT_RESET_CPC, 1); 5250 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5251 SOFT_RESET_CPG, 1); 5252 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5253 /********** exit soft reset ***********/ 5254 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5255 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5256 SOFT_RESET_CP, 0); 5257 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5258 SOFT_RESET_GFX, 0); 5259 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5260 SOFT_RESET_CPF, 0); 5261 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5262 SOFT_RESET_CPC, 0); 5263 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5264 SOFT_RESET_CPG, 0); 5265 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5266 5267 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL); 5268 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1); 5269 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp); 5270 5271 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0); 5272 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0); 5273 5274 for (i = 0; i < adev->usec_timeout; i++) { 5275 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET)) 5276 break; 5277 udelay(1); 5278 } 5279 if (i >= adev->usec_timeout) { 5280 printk("Failed to wait CP_VMID_RESET to 0\n"); 5281 return -EINVAL; 5282 } 5283 5284 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5285 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5286 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5287 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5288 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5289 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5290 5291 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5292 5293 r = gfx_v11_0_cp_resume(adev); 5294 if (r) 5295 return r; 5296 5297 /** 5298 * GFX soft reset impacts MES, resume MES after GFX soft reset is finished 5299 */ 5300 return amdgpu_mes_resume(adev, 0); 5301 } 5302 5303 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5304 { 5305 uint64_t clock; 5306 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after; 5307 5308 if (amdgpu_sriov_vf(adev)) { 5309 amdgpu_gfx_off_ctrl(adev, false); 5310 mutex_lock(&adev->gfx.gpu_clock_mutex); 5311 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5312 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5313 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5314 if (clock_counter_hi_pre != clock_counter_hi_after) 5315 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5316 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5317 amdgpu_gfx_off_ctrl(adev, true); 5318 } else { 5319 preempt_disable(); 5320 if (amdgpu_ip_version(adev, SMUIO_HWIP, 0) < IP_VERSION(15, 0, 0)) { 5321 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5322 regGOLDEN_TSC_COUNT_UPPER); 5323 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5324 regGOLDEN_TSC_COUNT_LOWER); 5325 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5326 regGOLDEN_TSC_COUNT_UPPER); 5327 if (clock_counter_hi_pre != clock_counter_hi_after) 5328 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5329 regGOLDEN_TSC_COUNT_LOWER); 5330 } else { 5331 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5332 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5333 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5334 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5335 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5336 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5337 if (clock_counter_hi_pre != clock_counter_hi_after) 5338 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5339 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5340 } 5341 preempt_enable(); 5342 } 5343 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL); 5344 5345 return clock; 5346 } 5347 5348 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5349 uint32_t vmid, 5350 uint32_t gds_base, uint32_t gds_size, 5351 uint32_t gws_base, uint32_t gws_size, 5352 uint32_t oa_base, uint32_t oa_size) 5353 { 5354 struct amdgpu_device *adev = ring->adev; 5355 5356 /* GDS Base */ 5357 gfx_v11_0_write_data_to_reg(ring, 0, false, 5358 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid, 5359 gds_base); 5360 5361 /* GDS Size */ 5362 gfx_v11_0_write_data_to_reg(ring, 0, false, 5363 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid, 5364 gds_size); 5365 5366 /* GWS */ 5367 gfx_v11_0_write_data_to_reg(ring, 0, false, 5368 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid, 5369 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5370 5371 /* OA */ 5372 gfx_v11_0_write_data_to_reg(ring, 0, false, 5373 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid, 5374 (1 << (oa_size + oa_base)) - (1 << oa_base)); 5375 } 5376 5377 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) 5378 { 5379 struct amdgpu_device *adev = ip_block->adev; 5380 int r; 5381 5382 switch (amdgpu_user_queue) { 5383 case -1: 5384 case 0: 5385 default: 5386 adev->gfx.disable_kq = false; 5387 adev->gfx.disable_uq = true; 5388 break; 5389 case 1: 5390 adev->gfx.disable_kq = false; 5391 adev->gfx.disable_uq = false; 5392 break; 5393 case 2: 5394 adev->gfx.disable_kq = true; 5395 adev->gfx.disable_uq = false; 5396 break; 5397 } 5398 5399 adev->gfx.funcs = &gfx_v11_0_gfx_funcs; 5400 5401 gfx_v11_0_set_imu_funcs(adev); 5402 r = gfx_v11_0_init_microcode(adev); 5403 if (r) 5404 return r; 5405 5406 if (adev->gfx.disable_kq) { 5407 /* We need one GFX ring temporarily to set up 5408 * the clear state. 5409 */ 5410 adev->gfx.num_gfx_rings = 1; 5411 adev->gfx.num_compute_rings = 0; 5412 } else { 5413 /* rs64 only supports one gfx pipe */ 5414 if (adev->gfx.rs64_enable) 5415 adev->gfx.num_gfx_rings = 1; 5416 else 5417 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS; 5418 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5419 AMDGPU_MAX_COMPUTE_RINGS); 5420 } 5421 5422 gfx_v11_0_set_kiq_pm4_funcs(adev); 5423 gfx_v11_0_set_ring_funcs(adev); 5424 gfx_v11_0_set_irq_funcs(adev); 5425 gfx_v11_0_set_gds_init(adev); 5426 gfx_v11_0_set_rlc_funcs(adev); 5427 gfx_v11_0_set_mqd_funcs(adev); 5428 5429 gfx_v11_0_init_rlcg_reg_access_ctrl(adev); 5430 5431 amdgpu_init_rlc_reg_funcs(adev); 5432 5433 return 0; 5434 } 5435 5436 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev) 5437 { 5438 uint32_t rlc_cntl; 5439 5440 /* if RLC is not enabled, do nothing */ 5441 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); 5442 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; 5443 } 5444 5445 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5446 { 5447 uint32_t data; 5448 unsigned i; 5449 5450 data = RLC_SAFE_MODE__CMD_MASK; 5451 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5452 5453 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data); 5454 5455 /* wait for RLC_SAFE_MODE */ 5456 for (i = 0; i < adev->usec_timeout; i++) { 5457 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE), 5458 RLC_SAFE_MODE, CMD)) 5459 break; 5460 udelay(1); 5461 } 5462 } 5463 5464 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5465 { 5466 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK); 5467 } 5468 5469 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 5470 bool enable) 5471 { 5472 uint32_t def, data; 5473 5474 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK)) 5475 return; 5476 5477 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5478 5479 if (enable) 5480 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5481 else 5482 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5483 5484 if (def != data) 5485 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5486 } 5487 5488 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev, 5489 bool enable) 5490 { 5491 uint32_t def, data; 5492 5493 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG)) 5494 return; 5495 5496 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5497 5498 if (enable) 5499 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5500 else 5501 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5502 5503 if (def != data) 5504 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5505 } 5506 5507 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev, 5508 bool enable) 5509 { 5510 uint32_t def, data; 5511 5512 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG)) 5513 return; 5514 5515 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5516 5517 if (enable) 5518 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5519 else 5520 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5521 5522 if (def != data) 5523 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5524 } 5525 5526 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5527 bool enable) 5528 { 5529 uint32_t data, def; 5530 5531 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS))) 5532 return; 5533 5534 /* It is disabled by HW by default */ 5535 if (enable) { 5536 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5537 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 5538 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5539 5540 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5541 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5542 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5543 5544 if (def != data) 5545 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5546 } 5547 } else { 5548 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5549 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5550 5551 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5552 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5553 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5554 5555 if (def != data) 5556 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5557 } 5558 } 5559 } 5560 5561 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5562 bool enable) 5563 { 5564 uint32_t def, data; 5565 5566 if (!(adev->cg_flags & 5567 (AMD_CG_SUPPORT_GFX_CGCG | 5568 AMD_CG_SUPPORT_GFX_CGLS | 5569 AMD_CG_SUPPORT_GFX_3D_CGCG | 5570 AMD_CG_SUPPORT_GFX_3D_CGLS))) 5571 return; 5572 5573 if (enable) { 5574 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5575 5576 /* unset CGCG override */ 5577 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5578 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5579 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5580 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5581 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG || 5582 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5583 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 5584 5585 /* update CGCG override bits */ 5586 if (def != data) 5587 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5588 5589 /* enable cgcg FSM(0x0000363F) */ 5590 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5591 5592 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5593 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK; 5594 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5595 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5596 } 5597 5598 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5599 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK; 5600 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5601 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5602 } 5603 5604 if (def != data) 5605 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5606 5607 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5608 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5609 5610 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5611 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK; 5612 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5613 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5614 } 5615 5616 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5617 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK; 5618 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5619 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5620 } 5621 5622 if (def != data) 5623 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5624 5625 /* set IDLE_POLL_COUNT(0x00900100) */ 5626 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL); 5627 5628 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK); 5629 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5630 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5631 5632 if (def != data) 5633 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data); 5634 5635 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5636 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5637 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5638 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5639 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5640 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data); 5641 5642 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5643 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5644 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5645 5646 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5647 if (adev->sdma.num_instances > 1) { 5648 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5649 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5650 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5651 } 5652 } else { 5653 /* Program RLC_CGCG_CGLS_CTRL */ 5654 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5655 5656 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5657 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5658 5659 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5660 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5661 5662 if (def != data) 5663 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5664 5665 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5666 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5667 5668 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 5669 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5670 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5671 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5672 5673 if (def != data) 5674 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5675 5676 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5677 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5678 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5679 5680 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5681 if (adev->sdma.num_instances > 1) { 5682 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5683 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5684 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5685 } 5686 } 5687 } 5688 5689 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5690 bool enable) 5691 { 5692 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5693 5694 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable); 5695 5696 gfx_v11_0_update_medium_grain_clock_gating(adev, enable); 5697 5698 gfx_v11_0_update_repeater_fgcg(adev, enable); 5699 5700 gfx_v11_0_update_sram_fgcg(adev, enable); 5701 5702 gfx_v11_0_update_perf_clk(adev, enable); 5703 5704 if (adev->cg_flags & 5705 (AMD_CG_SUPPORT_GFX_MGCG | 5706 AMD_CG_SUPPORT_GFX_CGLS | 5707 AMD_CG_SUPPORT_GFX_CGCG | 5708 AMD_CG_SUPPORT_GFX_3D_CGCG | 5709 AMD_CG_SUPPORT_GFX_3D_CGLS)) 5710 gfx_v11_0_enable_gui_idle_interrupt(adev, enable); 5711 5712 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5713 5714 return 0; 5715 } 5716 5717 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5718 struct amdgpu_ring *ring, unsigned vmid) 5719 { 5720 u32 reg, pre_data, data; 5721 5722 amdgpu_gfx_off_ctrl(adev, false); 5723 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); 5724 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) 5725 pre_data = RREG32_NO_KIQ(reg); 5726 else 5727 pre_data = RREG32(reg); 5728 5729 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK); 5730 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5731 5732 if (pre_data != data) { 5733 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) { 5734 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data); 5735 } else 5736 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data); 5737 } 5738 amdgpu_gfx_off_ctrl(adev, true); 5739 5740 if (ring 5741 && amdgpu_sriov_is_pp_one_vf(adev) 5742 && (pre_data != data) 5743 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) 5744 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) { 5745 amdgpu_ring_emit_wreg(ring, reg, data); 5746 } 5747 } 5748 5749 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = { 5750 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled, 5751 .set_safe_mode = gfx_v11_0_set_safe_mode, 5752 .unset_safe_mode = gfx_v11_0_unset_safe_mode, 5753 .init = gfx_v11_0_rlc_init, 5754 .get_csb_size = gfx_v11_0_get_csb_size, 5755 .get_csb_buffer = gfx_v11_0_get_csb_buffer, 5756 .resume = gfx_v11_0_rlc_resume, 5757 .stop = gfx_v11_0_rlc_stop, 5758 .reset = gfx_v11_0_rlc_reset, 5759 .start = gfx_v11_0_rlc_start, 5760 .update_spm_vmid = gfx_v11_0_update_spm_vmid, 5761 }; 5762 5763 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable) 5764 { 5765 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 5766 5767 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) 5768 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5769 else 5770 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5771 5772 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data); 5773 5774 // Program RLC_PG_DELAY3 for CGPG hysteresis 5775 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { 5776 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5777 case IP_VERSION(11, 0, 1): 5778 case IP_VERSION(11, 0, 4): 5779 case IP_VERSION(11, 5, 0): 5780 case IP_VERSION(11, 5, 1): 5781 case IP_VERSION(11, 5, 2): 5782 case IP_VERSION(11, 5, 3): 5783 case IP_VERSION(11, 5, 4): 5784 case IP_VERSION(11, 5, 6): 5785 case IP_VERSION(11, 7, 0): 5786 case IP_VERSION(11, 7, 1): 5787 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); 5788 break; 5789 default: 5790 break; 5791 } 5792 } 5793 } 5794 5795 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable) 5796 { 5797 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5798 5799 gfx_v11_cntl_power_gating(adev, enable); 5800 5801 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5802 } 5803 5804 static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5805 enum amd_powergating_state state) 5806 { 5807 struct amdgpu_device *adev = ip_block->adev; 5808 bool enable = (state == AMD_PG_STATE_GATE); 5809 5810 if (amdgpu_sriov_vf(adev)) 5811 return 0; 5812 5813 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5814 case IP_VERSION(11, 0, 0): 5815 case IP_VERSION(11, 0, 2): 5816 case IP_VERSION(11, 0, 3): 5817 amdgpu_gfx_off_ctrl(adev, enable); 5818 break; 5819 case IP_VERSION(11, 0, 1): 5820 case IP_VERSION(11, 0, 4): 5821 case IP_VERSION(11, 5, 0): 5822 case IP_VERSION(11, 5, 1): 5823 case IP_VERSION(11, 5, 2): 5824 case IP_VERSION(11, 5, 3): 5825 case IP_VERSION(11, 5, 4): 5826 case IP_VERSION(11, 5, 6): 5827 case IP_VERSION(11, 7, 0): 5828 case IP_VERSION(11, 7, 1): 5829 if (!enable) 5830 amdgpu_gfx_off_ctrl(adev, false); 5831 5832 gfx_v11_cntl_pg(adev, enable); 5833 5834 if (enable) 5835 amdgpu_gfx_off_ctrl(adev, true); 5836 5837 break; 5838 default: 5839 break; 5840 } 5841 5842 return 0; 5843 } 5844 5845 static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5846 enum amd_clockgating_state state) 5847 { 5848 struct amdgpu_device *adev = ip_block->adev; 5849 5850 if (amdgpu_sriov_vf(adev)) 5851 return 0; 5852 5853 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5854 case IP_VERSION(11, 0, 0): 5855 case IP_VERSION(11, 0, 1): 5856 case IP_VERSION(11, 0, 2): 5857 case IP_VERSION(11, 0, 3): 5858 case IP_VERSION(11, 0, 4): 5859 case IP_VERSION(11, 5, 0): 5860 case IP_VERSION(11, 5, 1): 5861 case IP_VERSION(11, 5, 2): 5862 case IP_VERSION(11, 5, 3): 5863 case IP_VERSION(11, 5, 4): 5864 case IP_VERSION(11, 5, 6): 5865 case IP_VERSION(11, 7, 0): 5866 case IP_VERSION(11, 7, 1): 5867 gfx_v11_0_update_gfx_clock_gating(adev, 5868 state == AMD_CG_STATE_GATE); 5869 break; 5870 default: 5871 break; 5872 } 5873 5874 return 0; 5875 } 5876 5877 static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5878 { 5879 struct amdgpu_device *adev = ip_block->adev; 5880 int data; 5881 5882 /* AMD_CG_SUPPORT_GFX_MGCG */ 5883 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5884 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5885 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5886 5887 /* AMD_CG_SUPPORT_REPEATER_FGCG */ 5888 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK)) 5889 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG; 5890 5891 /* AMD_CG_SUPPORT_GFX_FGCG */ 5892 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK)) 5893 *flags |= AMD_CG_SUPPORT_GFX_FGCG; 5894 5895 /* AMD_CG_SUPPORT_GFX_PERF_CLK */ 5896 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK)) 5897 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK; 5898 5899 /* AMD_CG_SUPPORT_GFX_CGCG */ 5900 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5901 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5902 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5903 5904 /* AMD_CG_SUPPORT_GFX_CGLS */ 5905 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5906 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5907 5908 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5909 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5910 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5911 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5912 5913 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5914 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5915 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5916 } 5917 5918 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5919 { 5920 /* gfx11 is 32bit rptr*/ 5921 return *(uint32_t *)ring->rptr_cpu_addr; 5922 } 5923 5924 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5925 { 5926 struct amdgpu_device *adev = ring->adev; 5927 u64 wptr; 5928 5929 /* XXX check if swapping is necessary on BE */ 5930 if (ring->use_doorbell) { 5931 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5932 } else { 5933 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR); 5934 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32; 5935 } 5936 5937 return wptr; 5938 } 5939 5940 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5941 { 5942 struct amdgpu_device *adev = ring->adev; 5943 5944 if (ring->use_doorbell) { 5945 /* XXX check if swapping is necessary on BE */ 5946 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5947 ring->wptr); 5948 WDOORBELL64(ring->doorbell_index, ring->wptr); 5949 } else { 5950 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, 5951 lower_32_bits(ring->wptr)); 5952 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, 5953 upper_32_bits(ring->wptr)); 5954 } 5955 } 5956 5957 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5958 { 5959 /* gfx11 hardware is 32bit rptr */ 5960 return *(uint32_t *)ring->rptr_cpu_addr; 5961 } 5962 5963 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5964 { 5965 struct amdgpu_device *adev = ring->adev; 5966 u64 wptr; 5967 5968 /* XXX check if swapping is necessary on BE */ 5969 if (ring->use_doorbell) { 5970 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5971 } else { 5972 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__); 5973 wptr = 0; 5974 } 5975 return wptr; 5976 } 5977 5978 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5979 { 5980 struct amdgpu_device *adev = ring->adev; 5981 5982 /* XXX check if swapping is necessary on BE */ 5983 if (ring->use_doorbell) { 5984 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5985 ring->wptr); 5986 WDOORBELL64(ring->doorbell_index, ring->wptr); 5987 } else { 5988 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__); 5989 } 5990 } 5991 5992 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5993 { 5994 struct amdgpu_device *adev = ring->adev; 5995 u32 ref_and_mask, reg_mem_engine; 5996 5997 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5998 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5999 return; 6000 } 6001 6002 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 6003 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 6004 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 6005 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 6006 ref_and_mask, ref_and_mask, 0x20); 6007 } 6008 6009 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 6010 struct amdgpu_job *job, 6011 struct amdgpu_ib *ib, 6012 uint32_t flags) 6013 { 6014 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6015 u32 header, control = 0; 6016 6017 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 6018 6019 control |= ib->length_dw | (vmid << 24); 6020 6021 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 6022 control |= INDIRECT_BUFFER_PRE_ENB(1); 6023 6024 if (flags & AMDGPU_IB_PREEMPTED) 6025 control |= INDIRECT_BUFFER_PRE_RESUME(1); 6026 6027 if (vmid && !ring->adev->gfx.rs64_enable) 6028 gfx_v11_0_ring_emit_de_meta(ring, 6029 !amdgpu_sriov_vf(ring->adev) && (flags & AMDGPU_IB_PREEMPTED)); 6030 } 6031 6032 amdgpu_ring_write(ring, header); 6033 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 6034 amdgpu_ring_write(ring, 6035 #ifdef __BIG_ENDIAN 6036 (2 << 0) | 6037 #endif 6038 lower_32_bits(ib->gpu_addr)); 6039 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 6040 amdgpu_ring_write(ring, control); 6041 } 6042 6043 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 6044 struct amdgpu_job *job, 6045 struct amdgpu_ib *ib, 6046 uint32_t flags) 6047 { 6048 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6049 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 6050 6051 /* Currently, there is a high possibility to get wave ID mismatch 6052 * between ME and GDS, leading to a hw deadlock, because ME generates 6053 * different wave IDs than the GDS expects. This situation happens 6054 * randomly when at least 5 compute pipes use GDS ordered append. 6055 * The wave IDs generated by ME are also wrong after suspend/resume. 6056 * Those are probably bugs somewhere else in the kernel driver. 6057 * 6058 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6059 * GDS to 0 for this ring (me/pipe). 6060 */ 6061 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6062 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6063 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID); 6064 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6065 } 6066 6067 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6068 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 6069 amdgpu_ring_write(ring, 6070 #ifdef __BIG_ENDIAN 6071 (2 << 0) | 6072 #endif 6073 lower_32_bits(ib->gpu_addr)); 6074 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 6075 amdgpu_ring_write(ring, control); 6076 } 6077 6078 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 6079 u64 seq, unsigned flags) 6080 { 6081 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6082 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6083 6084 /* RELEASE_MEM - flush caches, send int */ 6085 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 6086 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ | 6087 PACKET3_RELEASE_MEM_GCR_GL2_WB | 6088 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */ 6089 PACKET3_RELEASE_MEM_GCR_GLM_WB | 6090 PACKET3_RELEASE_MEM_CACHE_POLICY(3) | 6091 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6092 PACKET3_RELEASE_MEM_EVENT_INDEX(5))); 6093 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) | 6094 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0))); 6095 6096 /* 6097 * the address should be Qword aligned if 64bit write, Dword 6098 * aligned if only send 32bit data low (discard data high) 6099 */ 6100 if (write64bit) 6101 WARN_ON(addr & 0x7); 6102 else 6103 WARN_ON(addr & 0x3); 6104 amdgpu_ring_write(ring, lower_32_bits(addr)); 6105 amdgpu_ring_write(ring, upper_32_bits(addr)); 6106 amdgpu_ring_write(ring, lower_32_bits(seq)); 6107 amdgpu_ring_write(ring, upper_32_bits(seq)); 6108 amdgpu_ring_write(ring, 0); 6109 } 6110 6111 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6112 { 6113 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6114 uint32_t seq = ring->fence_drv.sync_seq; 6115 uint64_t addr = ring->fence_drv.gpu_addr; 6116 6117 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr), 6118 upper_32_bits(addr), seq, 0xffffffff, 4); 6119 } 6120 6121 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 6122 uint16_t pasid, uint32_t flush_type, 6123 bool all_hub, uint8_t dst_sel) 6124 { 6125 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 6126 amdgpu_ring_write(ring, 6127 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) | 6128 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 6129 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 6130 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 6131 } 6132 6133 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6134 unsigned vmid, uint64_t pd_addr) 6135 { 6136 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6137 6138 /* compute doesn't have PFP */ 6139 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 6140 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6141 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6142 amdgpu_ring_write(ring, 0x0); 6143 } 6144 6145 /* Make sure that we can't skip the SET_Q_MODE packets when the VM 6146 * changed in any way. 6147 */ 6148 ring->set_q_mode_offs = 0; 6149 ring->set_q_mode_ptr = NULL; 6150 } 6151 6152 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6153 u64 seq, unsigned int flags) 6154 { 6155 struct amdgpu_device *adev = ring->adev; 6156 6157 /* write fence seq to the "addr" */ 6158 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6159 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6160 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6161 amdgpu_ring_write(ring, lower_32_bits(addr)); 6162 amdgpu_ring_write(ring, upper_32_bits(addr)); 6163 amdgpu_ring_write(ring, lower_32_bits(seq)); 6164 6165 if (flags & AMDGPU_FENCE_FLAG_INT) { 6166 /* set register to trigger INT */ 6167 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6168 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6169 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6170 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS)); 6171 amdgpu_ring_write(ring, 0); 6172 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6173 } 6174 } 6175 6176 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, 6177 uint32_t flags) 6178 { 6179 uint32_t dw2 = 0; 6180 6181 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6182 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6183 /* set load_global_config & load_global_uconfig */ 6184 dw2 |= 0x8001; 6185 /* set load_cs_sh_regs */ 6186 dw2 |= 0x01000000; 6187 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6188 dw2 |= 0x10002; 6189 } 6190 6191 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6192 amdgpu_ring_write(ring, dw2); 6193 amdgpu_ring_write(ring, 0); 6194 } 6195 6196 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 6197 uint64_t addr) 6198 { 6199 unsigned ret; 6200 6201 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6202 amdgpu_ring_write(ring, lower_32_bits(addr)); 6203 amdgpu_ring_write(ring, upper_32_bits(addr)); 6204 /* discard following DWs if *cond_exec_gpu_addr==0 */ 6205 amdgpu_ring_write(ring, 0); 6206 ret = ring->wptr & ring->buf_mask; 6207 /* patch dummy value later */ 6208 amdgpu_ring_write(ring, 0); 6209 6210 return ret; 6211 } 6212 6213 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring, 6214 u64 shadow_va, u64 csa_va, 6215 u64 gds_va, bool init_shadow, 6216 int vmid) 6217 { 6218 struct amdgpu_device *adev = ring->adev; 6219 unsigned int offs, end; 6220 6221 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj) 6222 return; 6223 6224 /* 6225 * The logic here isn't easy to understand because we need to keep state 6226 * accross multiple executions of the function as well as between the 6227 * CPU and GPU. The general idea is that the newly written GPU command 6228 * has a condition on the previous one and only executed if really 6229 * necessary. 6230 */ 6231 6232 /* 6233 * The dw in the NOP controls if the next SET_Q_MODE packet should be 6234 * executed or not. Reserve 64bits just to be on the save side. 6235 */ 6236 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1)); 6237 offs = ring->wptr & ring->buf_mask; 6238 6239 /* 6240 * We start with skipping the prefix SET_Q_MODE and always executing 6241 * the postfix SET_Q_MODE packet. This is changed below with a 6242 * WRITE_DATA command when the postfix executed. 6243 */ 6244 amdgpu_ring_write(ring, shadow_va ? 1 : 0); 6245 amdgpu_ring_write(ring, 0); 6246 6247 if (ring->set_q_mode_offs) { 6248 uint64_t addr; 6249 6250 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6251 addr += ring->set_q_mode_offs << 2; 6252 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr); 6253 } 6254 6255 /* 6256 * When the postfix SET_Q_MODE packet executes we need to make sure that the 6257 * next prefix SET_Q_MODE packet executes as well. 6258 */ 6259 if (!shadow_va) { 6260 uint64_t addr; 6261 6262 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6263 addr += offs << 2; 6264 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6265 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); 6266 amdgpu_ring_write(ring, lower_32_bits(addr)); 6267 amdgpu_ring_write(ring, upper_32_bits(addr)); 6268 amdgpu_ring_write(ring, 0x1); 6269 } 6270 6271 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7)); 6272 amdgpu_ring_write(ring, lower_32_bits(shadow_va)); 6273 amdgpu_ring_write(ring, upper_32_bits(shadow_va)); 6274 amdgpu_ring_write(ring, lower_32_bits(gds_va)); 6275 amdgpu_ring_write(ring, upper_32_bits(gds_va)); 6276 amdgpu_ring_write(ring, lower_32_bits(csa_va)); 6277 amdgpu_ring_write(ring, upper_32_bits(csa_va)); 6278 amdgpu_ring_write(ring, shadow_va ? 6279 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0); 6280 amdgpu_ring_write(ring, init_shadow ? 6281 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0); 6282 6283 if (ring->set_q_mode_offs) 6284 amdgpu_ring_patch_cond_exec(ring, end); 6285 6286 if (shadow_va) { 6287 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid; 6288 6289 /* 6290 * If the tokens match try to skip the last postfix SET_Q_MODE 6291 * packet to avoid saving/restoring the state all the time. 6292 */ 6293 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token) 6294 *ring->set_q_mode_ptr = 0; 6295 6296 ring->set_q_mode_token = token; 6297 } else { 6298 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs]; 6299 } 6300 6301 ring->set_q_mode_offs = offs; 6302 } 6303 6304 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume) 6305 { 6306 struct amdgpu_device *adev = ring->adev; 6307 struct v10_de_ib_state de_payload = {0}; 6308 uint64_t offset, gds_addr, de_payload_gpu_addr; 6309 void *de_payload_cpu_addr; 6310 int cnt; 6311 6312 offset = offsetof(struct v10_gfx_meta_data, de_payload); 6313 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 6314 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 6315 6316 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 6317 AMDGPU_CSA_SIZE - adev->gds.gds_size, 6318 PAGE_SIZE); 6319 6320 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 6321 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 6322 6323 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 6324 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 6325 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 6326 WRITE_DATA_DST_SEL(8) | 6327 WR_CONFIRM) | 6328 WRITE_DATA_CACHE_POLICY(0)); 6329 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 6330 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 6331 6332 if (resume) 6333 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 6334 sizeof(de_payload) >> 2); 6335 else 6336 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 6337 sizeof(de_payload) >> 2); 6338 } 6339 6340 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 6341 bool secure) 6342 { 6343 uint32_t v = secure ? FRAME_TMZ : 0; 6344 6345 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 6346 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 6347 } 6348 6349 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6350 uint32_t reg_val_offs) 6351 { 6352 struct amdgpu_device *adev = ring->adev; 6353 6354 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6355 amdgpu_ring_write(ring, 0 | /* src: register*/ 6356 (5 << 8) | /* dst: memory */ 6357 (1 << 20)); /* write confirm */ 6358 amdgpu_ring_write(ring, reg); 6359 amdgpu_ring_write(ring, 0); 6360 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6361 reg_val_offs * 4)); 6362 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6363 reg_val_offs * 4)); 6364 } 6365 6366 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6367 uint32_t val) 6368 { 6369 uint32_t cmd = 0; 6370 6371 switch (ring->funcs->type) { 6372 case AMDGPU_RING_TYPE_GFX: 6373 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6374 break; 6375 case AMDGPU_RING_TYPE_KIQ: 6376 cmd = (1 << 16); /* no inc addr */ 6377 break; 6378 default: 6379 cmd = WR_CONFIRM; 6380 break; 6381 } 6382 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6383 amdgpu_ring_write(ring, cmd); 6384 amdgpu_ring_write(ring, reg); 6385 amdgpu_ring_write(ring, 0); 6386 amdgpu_ring_write(ring, val); 6387 } 6388 6389 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 6390 uint32_t val, uint32_t mask) 6391 { 6392 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 6393 } 6394 6395 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 6396 uint32_t reg0, uint32_t reg1, 6397 uint32_t ref, uint32_t mask) 6398 { 6399 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6400 6401 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 6402 ref, mask, 0x20); 6403 } 6404 6405 static void 6406 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6407 uint32_t me, uint32_t pipe, 6408 enum amdgpu_interrupt_state state) 6409 { 6410 uint32_t cp_int_cntl, cp_int_cntl_reg; 6411 6412 if (!me) { 6413 switch (pipe) { 6414 case 0: 6415 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 6416 break; 6417 case 1: 6418 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 6419 break; 6420 default: 6421 DRM_DEBUG("invalid pipe %d\n", pipe); 6422 return; 6423 } 6424 } else { 6425 DRM_DEBUG("invalid me %d\n", me); 6426 return; 6427 } 6428 6429 switch (state) { 6430 case AMDGPU_IRQ_STATE_DISABLE: 6431 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6432 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6433 TIME_STAMP_INT_ENABLE, 0); 6434 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6435 GENERIC0_INT_ENABLE, 0); 6436 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6437 break; 6438 case AMDGPU_IRQ_STATE_ENABLE: 6439 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6440 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6441 TIME_STAMP_INT_ENABLE, 1); 6442 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6443 GENERIC0_INT_ENABLE, 1); 6444 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6445 break; 6446 default: 6447 break; 6448 } 6449 } 6450 6451 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6452 int me, int pipe, 6453 enum amdgpu_interrupt_state state) 6454 { 6455 u32 mec_int_cntl, mec_int_cntl_reg; 6456 6457 /* 6458 * amdgpu controls only the first MEC. That's why this function only 6459 * handles the setting of interrupts for this specific MEC. All other 6460 * pipes' interrupts are set by amdkfd. 6461 */ 6462 6463 if (me == 1) { 6464 switch (pipe) { 6465 case 0: 6466 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6467 break; 6468 case 1: 6469 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 6470 break; 6471 case 2: 6472 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 6473 break; 6474 case 3: 6475 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 6476 break; 6477 default: 6478 DRM_DEBUG("invalid pipe %d\n", pipe); 6479 return; 6480 } 6481 } else { 6482 DRM_DEBUG("invalid me %d\n", me); 6483 return; 6484 } 6485 6486 switch (state) { 6487 case AMDGPU_IRQ_STATE_DISABLE: 6488 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6489 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6490 TIME_STAMP_INT_ENABLE, 0); 6491 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6492 GENERIC0_INT_ENABLE, 0); 6493 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6494 break; 6495 case AMDGPU_IRQ_STATE_ENABLE: 6496 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6497 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6498 TIME_STAMP_INT_ENABLE, 1); 6499 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6500 GENERIC0_INT_ENABLE, 1); 6501 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6502 break; 6503 default: 6504 break; 6505 } 6506 } 6507 6508 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6509 struct amdgpu_irq_src *src, 6510 unsigned type, 6511 enum amdgpu_interrupt_state state) 6512 { 6513 switch (type) { 6514 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6515 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state); 6516 break; 6517 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP: 6518 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state); 6519 break; 6520 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6521 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6522 break; 6523 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6524 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6525 break; 6526 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6527 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6528 break; 6529 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6530 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6531 break; 6532 default: 6533 break; 6534 } 6535 return 0; 6536 } 6537 6538 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, 6539 struct amdgpu_irq_src *source, 6540 struct amdgpu_iv_entry *entry) 6541 { 6542 u32 doorbell_offset = entry->src_data[0]; 6543 6544 DRM_DEBUG("IH: CP EOP\n"); 6545 6546 if (!adev->gfx.disable_kq) { 6547 u8 me_id = (entry->ring_id & 0x0c) >> 2; 6548 u8 pipe_id = (entry->ring_id & 0x03) >> 0; 6549 u8 queue_id = (entry->ring_id & 0x70) >> 4; 6550 struct amdgpu_ring *ring; 6551 int i; 6552 6553 switch (me_id) { 6554 case 0: 6555 /* 6556 * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, 6557 * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). 6558 * Require a strict (me,pipe,queue) match so userq gfx 6559 * EOPs fall through to amdgpu_userq_process_fence_irq(). 6560 */ 6561 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6562 ring = &adev->gfx.gfx_ring[i]; 6563 if ((ring->me == me_id) && 6564 (ring->pipe == pipe_id) && 6565 (ring->queue == queue_id)) { 6566 amdgpu_fence_process(ring); 6567 return 0; 6568 } 6569 } 6570 break; 6571 case 1: 6572 case 2: 6573 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6574 ring = &adev->gfx.compute_ring[i]; 6575 /* Per-queue interrupt is supported for MEC starting from VI. 6576 * The interrupt can only be enabled/disabled per pipe instead 6577 * of per queue. 6578 */ 6579 if ((ring->me == me_id) && 6580 (ring->pipe == pipe_id) && 6581 (ring->queue == queue_id)) { 6582 amdgpu_fence_process(ring); 6583 return 0; 6584 } 6585 } 6586 break; 6587 default: 6588 break; 6589 } 6590 } 6591 6592 if (adev->enable_mes && doorbell_offset) 6593 amdgpu_userq_process_fence_irq(adev, doorbell_offset); 6594 6595 return 0; 6596 } 6597 6598 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6599 struct amdgpu_irq_src *source, 6600 unsigned int type, 6601 enum amdgpu_interrupt_state state) 6602 { 6603 u32 cp_int_cntl_reg, cp_int_cntl; 6604 int i, j; 6605 6606 switch (state) { 6607 case AMDGPU_IRQ_STATE_DISABLE: 6608 case AMDGPU_IRQ_STATE_ENABLE: 6609 for (i = 0; i < adev->gfx.me.num_me; i++) { 6610 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6611 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6612 6613 if (cp_int_cntl_reg) { 6614 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6615 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6616 PRIV_REG_INT_ENABLE, 6617 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6618 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6619 } 6620 } 6621 } 6622 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6623 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6624 /* MECs start at 1 */ 6625 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6626 6627 if (cp_int_cntl_reg) { 6628 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6629 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6630 PRIV_REG_INT_ENABLE, 6631 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6632 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6633 } 6634 } 6635 } 6636 break; 6637 default: 6638 break; 6639 } 6640 6641 return 0; 6642 } 6643 6644 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev, 6645 struct amdgpu_irq_src *source, 6646 unsigned type, 6647 enum amdgpu_interrupt_state state) 6648 { 6649 u32 cp_int_cntl_reg, cp_int_cntl; 6650 int i, j; 6651 6652 switch (state) { 6653 case AMDGPU_IRQ_STATE_DISABLE: 6654 case AMDGPU_IRQ_STATE_ENABLE: 6655 for (i = 0; i < adev->gfx.me.num_me; i++) { 6656 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6657 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6658 6659 if (cp_int_cntl_reg) { 6660 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6661 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6662 OPCODE_ERROR_INT_ENABLE, 6663 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6664 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6665 } 6666 } 6667 } 6668 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6669 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6670 /* MECs start at 1 */ 6671 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6672 6673 if (cp_int_cntl_reg) { 6674 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6675 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6676 OPCODE_ERROR_INT_ENABLE, 6677 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6678 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6679 } 6680 } 6681 } 6682 break; 6683 default: 6684 break; 6685 } 6686 return 0; 6687 } 6688 6689 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6690 struct amdgpu_irq_src *source, 6691 unsigned int type, 6692 enum amdgpu_interrupt_state state) 6693 { 6694 u32 cp_int_cntl_reg, cp_int_cntl; 6695 int i, j; 6696 6697 switch (state) { 6698 case AMDGPU_IRQ_STATE_DISABLE: 6699 case AMDGPU_IRQ_STATE_ENABLE: 6700 for (i = 0; i < adev->gfx.me.num_me; i++) { 6701 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6702 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6703 6704 if (cp_int_cntl_reg) { 6705 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6706 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6707 PRIV_INSTR_INT_ENABLE, 6708 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6709 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6710 } 6711 } 6712 } 6713 break; 6714 default: 6715 break; 6716 } 6717 6718 return 0; 6719 } 6720 6721 /* 6722 * A gfx exception IV carries no doorbell. For each faulted slot, read the 6723 * doorbell back from the HQD (left in place by the fatal fault), look up the 6724 * user queue and kick its per-queue reset. 6725 */ 6726 static void gfx_v11_0_userq_priv_fault_work(struct work_struct *work) 6727 { 6728 struct amdgpu_device *adev = 6729 container_of(work, struct amdgpu_device, gfx.userq_priv_fault_work); 6730 unsigned long slots = xchg(&adev->gfx.userq_priv_fault_slots, 0); 6731 unsigned int id; 6732 6733 for_each_set_bit(id, &slots, BITS_PER_LONG) { 6734 u8 pipe = id & 0x3; 6735 u8 queue = (id >> 2) & 0x7; 6736 struct amdgpu_usermode_queue *q; 6737 u32 db_ctrl, doorbell; 6738 6739 amdgpu_gfx_off_ctrl(adev, false); 6740 mutex_lock(&adev->srbm_mutex); 6741 soc21_grbm_select(adev, 0, pipe, queue, 0); 6742 db_ctrl = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); 6743 soc21_grbm_select(adev, 0, 0, 0, 0); 6744 mutex_unlock(&adev->srbm_mutex); 6745 amdgpu_gfx_off_ctrl(adev, true); 6746 6747 doorbell = (db_ctrl & CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK) >> 6748 CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT; 6749 q = xa_load(&adev->userq_doorbell_xa, doorbell); 6750 if (q) 6751 amdgpu_userq_start_hang_detect_work(q); 6752 } 6753 } 6754 6755 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, 6756 struct amdgpu_iv_entry *entry) 6757 { 6758 u8 me_id, pipe_id, queue_id; 6759 6760 me_id = (entry->ring_id & 0x0c) >> 2; 6761 pipe_id = (entry->ring_id & 0x03) >> 0; 6762 queue_id = (entry->ring_id & 0x70) >> 4; 6763 6764 amdgpu_gfx_handle_priv_fault(adev, entry, me_id, pipe_id, queue_id); 6765 } 6766 6767 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev, 6768 struct amdgpu_irq_src *source, 6769 struct amdgpu_iv_entry *entry) 6770 { 6771 DRM_ERROR("Illegal register access in command stream\n"); 6772 gfx_v11_0_handle_priv_fault(adev, entry); 6773 return 0; 6774 } 6775 6776 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev, 6777 struct amdgpu_irq_src *source, 6778 struct amdgpu_iv_entry *entry) 6779 { 6780 DRM_ERROR("Illegal opcode in command stream\n"); 6781 gfx_v11_0_handle_priv_fault(adev, entry); 6782 return 0; 6783 } 6784 6785 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev, 6786 struct amdgpu_irq_src *source, 6787 struct amdgpu_iv_entry *entry) 6788 { 6789 DRM_ERROR("Illegal instruction in command stream\n"); 6790 gfx_v11_0_handle_priv_fault(adev, entry); 6791 return 0; 6792 } 6793 6794 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev, 6795 struct amdgpu_irq_src *source, 6796 struct amdgpu_iv_entry *entry) 6797 { 6798 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq) 6799 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry); 6800 6801 return 0; 6802 } 6803 6804 #if 0 6805 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev, 6806 struct amdgpu_irq_src *src, 6807 unsigned int type, 6808 enum amdgpu_interrupt_state state) 6809 { 6810 uint32_t tmp, target; 6811 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring); 6812 6813 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6814 target += ring->pipe; 6815 6816 switch (type) { 6817 case AMDGPU_CP_KIQ_IRQ_DRIVER0: 6818 if (state == AMDGPU_IRQ_STATE_DISABLE) { 6819 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6820 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6821 GENERIC2_INT_ENABLE, 0); 6822 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6823 6824 tmp = RREG32_SOC15_IP(GC, target); 6825 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6826 GENERIC2_INT_ENABLE, 0); 6827 WREG32_SOC15_IP(GC, target, tmp); 6828 } else { 6829 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6830 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6831 GENERIC2_INT_ENABLE, 1); 6832 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6833 6834 tmp = RREG32_SOC15_IP(GC, target); 6835 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6836 GENERIC2_INT_ENABLE, 1); 6837 WREG32_SOC15_IP(GC, target, tmp); 6838 } 6839 break; 6840 default: 6841 /* kiq only support GENERIC2_INT now */ 6842 break; 6843 } 6844 return 0; 6845 } 6846 #endif 6847 6848 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring) 6849 { 6850 const unsigned int gcr_cntl = 6851 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) | 6852 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) | 6853 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) | 6854 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) | 6855 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) | 6856 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) | 6857 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) | 6858 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1); 6859 6860 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ 6861 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6)); 6862 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */ 6863 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6864 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6865 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6866 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6867 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6868 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ 6869 } 6870 6871 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, 6872 unsigned int vmid, 6873 struct amdgpu_fence *timedout_fence) 6874 { 6875 struct amdgpu_device *adev = ring->adev; 6876 bool use_mmio = adev->gfx.me.use_mmio_for_reset; 6877 6878 return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio); 6879 } 6880 6881 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, 6882 unsigned int vmid, 6883 struct amdgpu_fence *timedout_fence) 6884 { 6885 struct amdgpu_device *adev = ring->adev; 6886 6887 return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL); 6888 } 6889 6890 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 6891 { 6892 struct amdgpu_device *adev = ip_block->adev; 6893 uint32_t i, j, k, reg, index = 0; 6894 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6895 6896 if (!adev->gfx.ip_dump_core) 6897 return; 6898 6899 for (i = 0; i < reg_count; i++) 6900 drm_printf(p, "%-50s \t 0x%08x\n", 6901 gc_reg_list_11_0[i].reg_name, 6902 adev->gfx.ip_dump_core[i]); 6903 6904 /* print compute queue registers for all instances */ 6905 if (!adev->gfx.ip_dump_compute_queues) 6906 return; 6907 6908 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6909 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n", 6910 adev->gfx.mec.num_mec, 6911 adev->gfx.mec.num_pipe_per_mec, 6912 adev->gfx.mec.num_queue_per_pipe); 6913 6914 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6915 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6916 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6917 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k); 6918 for (reg = 0; reg < reg_count; reg++) { 6919 if (i && gc_cp_reg_list_11[reg].reg_offset == regCP_MEC_ME1_HEADER_DUMP) 6920 drm_printf(p, "%-50s \t 0x%08x\n", 6921 "regCP_MEC_ME2_HEADER_DUMP", 6922 adev->gfx.ip_dump_compute_queues[index + reg]); 6923 else 6924 drm_printf(p, "%-50s \t 0x%08x\n", 6925 gc_cp_reg_list_11[reg].reg_name, 6926 adev->gfx.ip_dump_compute_queues[index + reg]); 6927 } 6928 index += reg_count; 6929 } 6930 } 6931 } 6932 6933 /* print gfx queue registers for all instances */ 6934 if (!adev->gfx.ip_dump_gfx_queues) 6935 return; 6936 6937 index = 0; 6938 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 6939 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n", 6940 adev->gfx.me.num_me, 6941 adev->gfx.me.num_pipe_per_me, 6942 adev->gfx.me.num_queue_per_pipe); 6943 6944 for (i = 0; i < adev->gfx.me.num_me; i++) { 6945 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6946 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 6947 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k); 6948 for (reg = 0; reg < reg_count; reg++) { 6949 drm_printf(p, "%-50s \t 0x%08x\n", 6950 gc_gfx_queue_reg_list_11[reg].reg_name, 6951 adev->gfx.ip_dump_gfx_queues[index + reg]); 6952 } 6953 index += reg_count; 6954 } 6955 } 6956 } 6957 } 6958 6959 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block) 6960 { 6961 struct amdgpu_device *adev = ip_block->adev; 6962 uint32_t i, j, k, reg, index = 0; 6963 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6964 6965 if (!adev->gfx.ip_dump_core) 6966 return; 6967 6968 amdgpu_gfx_off_ctrl(adev, false); 6969 for (i = 0; i < reg_count; i++) 6970 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i])); 6971 amdgpu_gfx_off_ctrl(adev, true); 6972 6973 /* dump compute queue registers for all instances */ 6974 if (!adev->gfx.ip_dump_compute_queues) 6975 return; 6976 6977 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6978 amdgpu_gfx_off_ctrl(adev, false); 6979 mutex_lock(&adev->srbm_mutex); 6980 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6981 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6982 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6983 /* ME0 is for GFX so start from 1 for CP */ 6984 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0); 6985 for (reg = 0; reg < reg_count; reg++) { 6986 if (i && 6987 gc_cp_reg_list_11[reg].reg_offset == 6988 regCP_MEC_ME1_HEADER_DUMP) 6989 adev->gfx.ip_dump_compute_queues[index + reg] = 6990 RREG32(SOC15_REG_OFFSET(GC, 0, 6991 regCP_MEC_ME2_HEADER_DUMP)); 6992 else 6993 adev->gfx.ip_dump_compute_queues[index + reg] = 6994 RREG32(SOC15_REG_ENTRY_OFFSET( 6995 gc_cp_reg_list_11[reg])); 6996 } 6997 index += reg_count; 6998 } 6999 } 7000 } 7001 soc21_grbm_select(adev, 0, 0, 0, 0); 7002 mutex_unlock(&adev->srbm_mutex); 7003 amdgpu_gfx_off_ctrl(adev, true); 7004 7005 /* dump gfx queue registers for all instances */ 7006 if (!adev->gfx.ip_dump_gfx_queues) 7007 return; 7008 7009 index = 0; 7010 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 7011 amdgpu_gfx_off_ctrl(adev, false); 7012 mutex_lock(&adev->srbm_mutex); 7013 for (i = 0; i < adev->gfx.me.num_me; i++) { 7014 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 7015 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 7016 soc21_grbm_select(adev, i, j, k, 0); 7017 7018 for (reg = 0; reg < reg_count; reg++) { 7019 adev->gfx.ip_dump_gfx_queues[index + reg] = 7020 RREG32(SOC15_REG_ENTRY_OFFSET( 7021 gc_gfx_queue_reg_list_11[reg])); 7022 } 7023 index += reg_count; 7024 } 7025 } 7026 } 7027 soc21_grbm_select(adev, 0, 0, 0, 0); 7028 mutex_unlock(&adev->srbm_mutex); 7029 amdgpu_gfx_off_ctrl(adev, true); 7030 } 7031 7032 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring) 7033 { 7034 /* Emit the cleaner shader */ 7035 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0)); 7036 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */ 7037 } 7038 7039 static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring) 7040 { 7041 amdgpu_gfx_profile_ring_begin_use(ring); 7042 7043 amdgpu_gfx_enforce_isolation_ring_begin_use(ring); 7044 } 7045 7046 static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring) 7047 { 7048 amdgpu_gfx_profile_ring_end_use(ring); 7049 7050 amdgpu_gfx_enforce_isolation_ring_end_use(ring); 7051 } 7052 7053 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { 7054 .name = "gfx_v11_0", 7055 .early_init = gfx_v11_0_early_init, 7056 .sw_init = gfx_v11_0_sw_init, 7057 .sw_fini = gfx_v11_0_sw_fini, 7058 .hw_init = gfx_v11_0_hw_init, 7059 .hw_fini = gfx_v11_0_hw_fini, 7060 .suspend = gfx_v11_0_suspend, 7061 .resume = gfx_v11_0_resume, 7062 .is_idle = gfx_v11_0_is_idle, 7063 .wait_for_idle = gfx_v11_0_wait_for_idle, 7064 .soft_reset = gfx_v11_0_soft_reset, 7065 .set_clockgating_state = gfx_v11_0_set_clockgating_state, 7066 .set_powergating_state = gfx_v11_0_set_powergating_state, 7067 .get_clockgating_state = gfx_v11_0_get_clockgating_state, 7068 .dump_ip_state = gfx_v11_ip_dump, 7069 .print_ip_state = gfx_v11_ip_print, 7070 }; 7071 7072 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = { 7073 .type = AMDGPU_RING_TYPE_GFX, 7074 .align_mask = 0xff, 7075 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7076 .support_64bit_ptrs = true, 7077 .secure_submission_supported = true, 7078 .get_rptr = gfx_v11_0_ring_get_rptr_gfx, 7079 .get_wptr = gfx_v11_0_ring_get_wptr_gfx, 7080 .set_wptr = gfx_v11_0_ring_set_wptr_gfx, 7081 .emit_frame_size = /* totally 247 maximum if 16 IBs */ 7082 5 + /* update_spm_vmid */ 7083 5 + /* COND_EXEC */ 7084 22 + /* SET_Q_PREEMPTION_MODE */ 7085 7 + /* PIPELINE_SYNC */ 7086 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7087 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7088 4 + /* VM_FLUSH */ 7089 8 + /* FENCE for VM_FLUSH */ 7090 20 + /* GDS switch */ 7091 5 + /* COND_EXEC */ 7092 7 + /* HDP_flush */ 7093 4 + /* VGT_flush */ 7094 31 + /* DE_META */ 7095 3 + /* CNTX_CTRL */ 7096 5 + /* HDP_INVL */ 7097 22 + /* SET_Q_PREEMPTION_MODE */ 7098 8 + 8 + /* FENCE x2 */ 7099 8 + /* gfx_v11_0_emit_mem_sync */ 7100 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7101 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */ 7102 .emit_ib = gfx_v11_0_ring_emit_ib_gfx, 7103 .emit_fence = gfx_v11_0_ring_emit_fence, 7104 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7105 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7106 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7107 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7108 .test_ring = gfx_v11_0_ring_test_ring, 7109 .test_ib = gfx_v11_0_ring_test_ib, 7110 .insert_nop = gfx_v11_ring_insert_nop, 7111 .pad_ib = amdgpu_ring_generic_pad_ib, 7112 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl, 7113 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow, 7114 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec, 7115 .preempt_ib = amdgpu_gfx_ring_preempt_ib, 7116 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl, 7117 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7118 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7119 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7120 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7121 .reset = gfx_v11_0_reset_kgq, 7122 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7123 .begin_use = gfx_v11_0_ring_begin_use, 7124 .end_use = gfx_v11_0_ring_end_use, 7125 }; 7126 7127 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = { 7128 .type = AMDGPU_RING_TYPE_COMPUTE, 7129 .align_mask = 0xff, 7130 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7131 .support_64bit_ptrs = true, 7132 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7133 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7134 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7135 .emit_frame_size = 7136 5 + /* update_spm_vmid */ 7137 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7138 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7139 5 + /* hdp invalidate */ 7140 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7141 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7142 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7143 2 + /* gfx_v11_0_ring_emit_vm_flush */ 7144 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */ 7145 8 + /* gfx_v11_0_emit_mem_sync */ 7146 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7147 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7148 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7149 .emit_fence = gfx_v11_0_ring_emit_fence, 7150 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7151 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7152 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7153 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7154 .test_ring = gfx_v11_0_ring_test_ring, 7155 .test_ib = gfx_v11_0_ring_test_ib, 7156 .insert_nop = gfx_v11_ring_insert_nop, 7157 .pad_ib = amdgpu_ring_generic_pad_ib, 7158 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7159 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7160 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7161 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7162 .reset = gfx_v11_0_reset_kcq, 7163 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7164 .begin_use = gfx_v11_0_ring_begin_use, 7165 .end_use = gfx_v11_0_ring_end_use, 7166 }; 7167 7168 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = { 7169 .type = AMDGPU_RING_TYPE_KIQ, 7170 .align_mask = 0xff, 7171 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7172 .support_64bit_ptrs = true, 7173 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7174 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7175 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7176 .emit_frame_size = 7177 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7178 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7179 5 + /*hdp invalidate */ 7180 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7181 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7182 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7183 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7184 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7185 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7186 .emit_fence = gfx_v11_0_ring_emit_fence_kiq, 7187 .test_ring = gfx_v11_0_ring_test_ring, 7188 .test_ib = gfx_v11_0_ring_test_ib, 7189 .insert_nop = amdgpu_ring_insert_nop, 7190 .pad_ib = amdgpu_ring_generic_pad_ib, 7191 .emit_rreg = gfx_v11_0_ring_emit_rreg, 7192 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7193 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7194 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7195 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7196 }; 7197 7198 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev) 7199 { 7200 int i; 7201 7202 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq; 7203 7204 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7205 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx; 7206 7207 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7208 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute; 7209 } 7210 7211 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = { 7212 .set = gfx_v11_0_set_eop_interrupt_state, 7213 .process = gfx_v11_0_eop_irq, 7214 }; 7215 7216 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = { 7217 .set = gfx_v11_0_set_priv_reg_fault_state, 7218 .process = gfx_v11_0_priv_reg_irq, 7219 }; 7220 7221 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = { 7222 .set = gfx_v11_0_set_bad_op_fault_state, 7223 .process = gfx_v11_0_bad_op_irq, 7224 }; 7225 7226 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = { 7227 .set = gfx_v11_0_set_priv_inst_fault_state, 7228 .process = gfx_v11_0_priv_inst_irq, 7229 }; 7230 7231 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = { 7232 .process = gfx_v11_0_rlc_gc_fed_irq, 7233 }; 7234 7235 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev) 7236 { 7237 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7238 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs; 7239 7240 adev->gfx.priv_reg_irq.num_types = 1; 7241 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs; 7242 7243 adev->gfx.bad_op_irq.num_types = 1; 7244 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs; 7245 7246 adev->gfx.priv_inst_irq.num_types = 1; 7247 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs; 7248 7249 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */ 7250 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs; 7251 7252 } 7253 7254 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev) 7255 { 7256 if (adev->flags & AMD_IS_APU) 7257 adev->gfx.imu.mode = MISSION_MODE; 7258 else 7259 adev->gfx.imu.mode = DEBUG_MODE; 7260 7261 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs; 7262 } 7263 7264 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev) 7265 { 7266 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs; 7267 } 7268 7269 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev) 7270 { 7271 unsigned total_cu = adev->gfx.config.max_cu_per_sh * 7272 adev->gfx.config.max_sh_per_se * 7273 adev->gfx.config.max_shader_engines; 7274 7275 adev->gds.gds_size = 0x1000; 7276 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1; 7277 adev->gds.gws_size = 64; 7278 adev->gds.oa_size = 16; 7279 } 7280 7281 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev) 7282 { 7283 /* set gfx eng mqd */ 7284 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size = 7285 sizeof(struct v11_gfx_mqd); 7286 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd = 7287 gfx_v11_0_gfx_mqd_init; 7288 /* set compute eng mqd */ 7289 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size = 7290 sizeof(struct v11_compute_mqd); 7291 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd = 7292 gfx_v11_0_compute_mqd_init; 7293 } 7294 7295 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev, 7296 u32 bitmap) 7297 { 7298 u32 data; 7299 7300 if (!bitmap) 7301 return; 7302 7303 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7304 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7305 7306 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data); 7307 } 7308 7309 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev) 7310 { 7311 u32 data, wgp_bitmask; 7312 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG); 7313 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG); 7314 7315 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7316 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7317 7318 wgp_bitmask = 7319 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1); 7320 7321 return (~data) & wgp_bitmask; 7322 } 7323 7324 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev) 7325 { 7326 u32 wgp_idx, wgp_active_bitmap; 7327 u32 cu_bitmap_per_wgp, cu_active_bitmap; 7328 7329 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev); 7330 cu_active_bitmap = 0; 7331 7332 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) { 7333 /* if there is one WGP enabled, it means 2 CUs will be enabled */ 7334 cu_bitmap_per_wgp = 3 << (2 * wgp_idx); 7335 if (wgp_active_bitmap & (1 << wgp_idx)) 7336 cu_active_bitmap |= cu_bitmap_per_wgp; 7337 } 7338 7339 return cu_active_bitmap; 7340 } 7341 7342 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 7343 struct amdgpu_cu_info *cu_info) 7344 { 7345 int i, j, k, counter, active_cu_number = 0; 7346 u32 mask, bitmap; 7347 unsigned disable_masks[8 * 2]; 7348 7349 if (!adev || !cu_info) 7350 return -EINVAL; 7351 7352 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 8, 2); 7353 7354 mutex_lock(&adev->grbm_idx_mutex); 7355 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7356 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7357 bitmap = i * adev->gfx.config.max_sh_per_se + j; 7358 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1)) 7359 continue; 7360 mask = 1; 7361 counter = 0; 7362 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7363 if (i < 8 && j < 2) 7364 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh( 7365 adev, disable_masks[i * 2 + j]); 7366 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev); 7367 7368 /** 7369 * GFX11 could support more than 4 SEs, while the bitmap 7370 * in cu_info struct is 4x4 and ioctl interface struct 7371 * drm_amdgpu_info_device should keep stable. 7372 * So we use last two columns of bitmap to store cu mask for 7373 * SEs 4 to 7, the layout of the bitmap is as below: 7374 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]} 7375 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]} 7376 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]} 7377 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]} 7378 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]} 7379 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]} 7380 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]} 7381 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]} 7382 */ 7383 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap; 7384 7385 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { 7386 if (bitmap & mask) 7387 counter++; 7388 7389 mask <<= 1; 7390 } 7391 active_cu_number += counter; 7392 } 7393 } 7394 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7395 mutex_unlock(&adev->grbm_idx_mutex); 7396 7397 cu_info->number = active_cu_number; 7398 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7399 7400 return 0; 7401 } 7402 7403 const struct amdgpu_ip_block_version gfx_v11_0_ip_block = 7404 { 7405 .type = AMD_IP_BLOCK_TYPE_GFX, 7406 .major = 11, 7407 .minor = 0, 7408 .rev = 0, 7409 .funcs = &gfx_v11_0_ip_funcs, 7410 }; 7411