1 /* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/delay.h> 24 #include <linux/kernel.h> 25 #include <linux/firmware.h> 26 #include <linux/module.h> 27 #include <linux/pci.h> 28 #include "amdgpu.h" 29 #include "amdgpu_gfx.h" 30 #include "amdgpu_psp.h" 31 #include "amdgpu_smu.h" 32 #include "imu_v11_0.h" 33 #include "soc21.h" 34 #include "nvd.h" 35 36 #include "gc/gc_11_0_0_offset.h" 37 #include "gc/gc_11_0_0_sh_mask.h" 38 #include "smuio/smuio_13_0_6_offset.h" 39 #include "smuio/smuio_13_0_6_sh_mask.h" 40 #include "navi10_enum.h" 41 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h" 42 43 #include "soc15.h" 44 #include "clearstate_gfx11.h" 45 #include "v11_structs.h" 46 #include "gfx_v11_0.h" 47 #include "gfx_v11_0_cleaner_shader.h" 48 #include "gfx_v11_0_3.h" 49 #include "nbio_v4_3.h" 50 #include "mes_v11_0.h" 51 #include "mes_userqueue.h" 52 #include "amdgpu_userq_fence.h" 53 54 #define GFX11_NUM_GFX_RINGS 1 55 #define GFX11_MEC_HPD_SIZE 2048 56 57 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 58 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388 59 60 #define regCGTT_WD_CLK_CTRL 0x5086 61 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1 62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e 63 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1 64 #define regPC_CONFIG_CNTL_1 0x194d 65 #define regPC_CONFIG_CNTL_1_BASE_IDX 1 66 67 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0 0x0030 68 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0_BASE_IDX 1 69 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0 0x0031 70 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0_BASE_IDX 1 71 72 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100 73 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000 74 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000 75 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01 76 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000 77 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000 78 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000 79 80 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006 81 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 82 #define regCP_MQD_CONTROL_DEFAULT 0x00000100 83 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509 84 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 85 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000 86 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501 87 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000 88 89 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin"); 90 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin"); 91 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin"); 92 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin"); 93 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_kicker.bin"); 94 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin"); 95 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin"); 96 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin"); 97 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin"); 98 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin"); 99 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin"); 101 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin"); 102 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin"); 103 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin"); 104 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin"); 105 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin"); 106 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin"); 107 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin"); 109 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin"); 110 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin"); 112 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin"); 113 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin"); 114 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin"); 115 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin"); 116 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin"); 117 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin"); 118 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin"); 119 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin"); 120 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin"); 122 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin"); 124 MODULE_FIRMWARE("amdgpu/gc_11_5_3_pfp.bin"); 125 MODULE_FIRMWARE("amdgpu/gc_11_5_3_me.bin"); 126 MODULE_FIRMWARE("amdgpu/gc_11_5_3_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/gc_11_5_3_rlc.bin"); 128 MODULE_FIRMWARE("amdgpu/gc_11_5_4_pfp.bin"); 129 MODULE_FIRMWARE("amdgpu/gc_11_5_4_me.bin"); 130 MODULE_FIRMWARE("amdgpu/gc_11_5_4_mec.bin"); 131 MODULE_FIRMWARE("amdgpu/gc_11_5_4_rlc.bin"); 132 MODULE_FIRMWARE("amdgpu/gc_11_5_6_pfp.bin"); 133 MODULE_FIRMWARE("amdgpu/gc_11_5_6_me.bin"); 134 MODULE_FIRMWARE("amdgpu/gc_11_5_6_mec.bin"); 135 MODULE_FIRMWARE("amdgpu/gc_11_5_6_rlc.bin"); 136 MODULE_FIRMWARE("amdgpu/gc_11_7_0_pfp.bin"); 137 MODULE_FIRMWARE("amdgpu/gc_11_7_0_me.bin"); 138 MODULE_FIRMWARE("amdgpu/gc_11_7_0_mec.bin"); 139 MODULE_FIRMWARE("amdgpu/gc_11_7_0_rlc.bin"); 140 MODULE_FIRMWARE("amdgpu/gc_11_7_1_pfp.bin"); 141 MODULE_FIRMWARE("amdgpu/gc_11_7_1_me.bin"); 142 MODULE_FIRMWARE("amdgpu/gc_11_7_1_mec.bin"); 143 MODULE_FIRMWARE("amdgpu/gc_11_7_1_rlc.bin"); 144 145 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = { 146 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), 147 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2), 148 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3), 149 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1), 150 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2), 151 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3), 152 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1), 153 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1), 154 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT), 155 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT), 156 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT), 157 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2), 158 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2), 159 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS), 160 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR), 161 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0), 162 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE), 163 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 164 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR), 165 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE), 166 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR), 167 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR), 168 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE), 169 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR), 170 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR), 171 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 172 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ), 173 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 174 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 175 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 176 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO), 177 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI), 178 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ), 179 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS), 180 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS), 181 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS), 182 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT), 183 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT), 184 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS), 185 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2), 186 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS), 187 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS), 188 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS), 189 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES), 190 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS), 191 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS), 192 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL), 193 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS), 194 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG), 195 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL), 196 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL), 197 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR), 198 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR), 199 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR), 200 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR), 201 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR), 202 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS), 203 /* cp header registers */ 204 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 205 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 206 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 207 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 208 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 209 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 210 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 211 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 212 /* SE status registers */ 213 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0), 214 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1), 215 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2), 216 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3), 217 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4), 218 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5) 219 }; 220 221 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = { 222 /* compute registers */ 223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID), 224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE), 225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY), 226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY), 227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM), 228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE), 229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI), 230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR), 231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 232 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 233 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), 234 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL), 235 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR), 236 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI), 237 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR), 238 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL), 239 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST), 240 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR), 241 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI), 242 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL), 243 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR), 244 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR), 245 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS), 246 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO), 247 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI), 248 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL), 249 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET), 250 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE), 251 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET), 252 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE), 253 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE), 254 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR), 255 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM), 256 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO), 257 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI), 258 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET), 259 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT), 260 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET), 261 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS), 262 /* cp header registers */ 263 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 264 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 265 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 266 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 267 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 268 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 269 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 270 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 271 }; 272 273 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = { 274 /* gfx queue registers */ 275 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE), 276 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID), 277 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY), 278 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM), 279 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE), 280 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI), 281 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET), 282 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL), 283 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR), 284 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR), 285 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI), 286 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST), 287 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED), 288 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL), 289 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0), 290 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0), 291 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR), 292 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI), 293 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO), 294 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI), 295 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 296 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 297 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 298 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 299 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 300 /* cp header registers */ 301 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 302 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 303 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 304 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 305 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 306 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 307 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 308 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 309 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 310 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 311 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 312 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 313 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 314 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 315 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 316 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 317 }; 318 319 static const struct soc15_reg_golden golden_settings_gc_11_0[] = { 320 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000) 321 }; 322 323 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] = 324 { 325 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010), 326 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010), 327 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200), 328 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988), 329 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007), 330 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008), 331 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100), 332 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000), 333 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a) 334 }; 335 336 #define DEFAULT_SH_MEM_CONFIG \ 337 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \ 338 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \ 339 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT)) 340 341 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev); 342 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev); 343 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev); 344 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev); 345 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev); 346 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev); 347 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev); 348 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 349 struct amdgpu_cu_info *cu_info); 350 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev); 351 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 352 u32 sh_num, u32 instance, int xcc_id); 353 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev); 354 355 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume); 356 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); 357 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 358 uint32_t val); 359 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); 360 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 361 uint16_t pasid, uint32_t flush_type, 362 bool all_hub, uint8_t dst_sel); 363 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 364 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 365 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 366 bool enable); 367 368 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask) 369 { 370 struct amdgpu_device *adev = kiq_ring->adev; 371 u64 shader_mc_addr; 372 373 /* Cleaner shader MC address */ 374 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; 375 376 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 377 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | 378 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */ 379 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ 380 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 381 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 382 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */ 383 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */ 384 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 385 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 386 } 387 388 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring, 389 struct amdgpu_ring *ring) 390 { 391 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 392 uint64_t wptr_addr = ring->wptr_gpu_addr; 393 uint32_t me = 0, eng_sel = 0; 394 395 switch (ring->funcs->type) { 396 case AMDGPU_RING_TYPE_COMPUTE: 397 me = 1; 398 eng_sel = 0; 399 break; 400 case AMDGPU_RING_TYPE_GFX: 401 me = 0; 402 eng_sel = 4; 403 break; 404 case AMDGPU_RING_TYPE_MES: 405 me = 2; 406 eng_sel = 5; 407 break; 408 default: 409 WARN_ON(1); 410 } 411 412 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 413 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 414 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 415 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 416 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 417 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 418 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 419 PACKET3_MAP_QUEUES_ME((me)) | 420 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 421 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 422 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 423 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 424 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 425 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 426 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 427 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 428 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 429 } 430 431 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 432 struct amdgpu_ring *ring, 433 enum amdgpu_unmap_queues_action action, 434 u64 gpu_addr, u64 seq) 435 { 436 struct amdgpu_device *adev = kiq_ring->adev; 437 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 438 439 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) { 440 amdgpu_mes_unmap_legacy_queue(adev, ring, action, 441 gpu_addr, seq, 0); 442 return; 443 } 444 445 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 446 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 447 PACKET3_UNMAP_QUEUES_ACTION(action) | 448 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 449 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 450 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 451 amdgpu_ring_write(kiq_ring, 452 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 453 454 if (action == PREEMPT_QUEUES_NO_UNMAP) { 455 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 456 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 457 amdgpu_ring_write(kiq_ring, seq); 458 } else { 459 amdgpu_ring_write(kiq_ring, 0); 460 amdgpu_ring_write(kiq_ring, 0); 461 amdgpu_ring_write(kiq_ring, 0); 462 } 463 } 464 465 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring, 466 struct amdgpu_ring *ring, 467 u64 addr, 468 u64 seq) 469 { 470 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 471 472 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 473 amdgpu_ring_write(kiq_ring, 474 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 475 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 476 PACKET3_QUERY_STATUS_COMMAND(2)); 477 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 478 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 479 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 480 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 481 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 482 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 483 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 484 } 485 486 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 487 uint16_t pasid, uint32_t flush_type, 488 bool all_hub) 489 { 490 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1); 491 } 492 493 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = { 494 .kiq_set_resources = gfx11_kiq_set_resources, 495 .kiq_map_queues = gfx11_kiq_map_queues, 496 .kiq_unmap_queues = gfx11_kiq_unmap_queues, 497 .kiq_query_status = gfx11_kiq_query_status, 498 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs, 499 .set_resources_size = 8, 500 .map_queues_size = 7, 501 .unmap_queues_size = 6, 502 .query_status_size = 7, 503 .invalidate_tlbs_size = 2, 504 }; 505 506 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 507 { 508 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs; 509 } 510 511 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev) 512 { 513 if (amdgpu_sriov_vf(adev)) 514 return; 515 516 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 517 case IP_VERSION(11, 0, 1): 518 case IP_VERSION(11, 0, 4): 519 soc15_program_register_sequence(adev, 520 golden_settings_gc_11_0_1, 521 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1)); 522 break; 523 default: 524 break; 525 } 526 soc15_program_register_sequence(adev, 527 golden_settings_gc_11_0, 528 (const u32)ARRAY_SIZE(golden_settings_gc_11_0)); 529 530 } 531 532 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 533 bool wc, uint32_t reg, uint32_t val) 534 { 535 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 536 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 537 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0)); 538 amdgpu_ring_write(ring, reg); 539 amdgpu_ring_write(ring, 0); 540 amdgpu_ring_write(ring, val); 541 } 542 543 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 544 int mem_space, int opt, uint32_t addr0, 545 uint32_t addr1, uint32_t ref, uint32_t mask, 546 uint32_t inv) 547 { 548 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 549 amdgpu_ring_write(ring, 550 /* memory (1) or register (0) */ 551 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 552 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 553 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 554 WAIT_REG_MEM_ENGINE(eng_sel))); 555 556 if (mem_space) 557 WARN_ON(addr0 & 0x3); /* Dword align */ 558 amdgpu_ring_write(ring, addr0); 559 amdgpu_ring_write(ring, addr1); 560 amdgpu_ring_write(ring, ref); 561 amdgpu_ring_write(ring, mask); 562 amdgpu_ring_write(ring, inv); /* poll interval */ 563 } 564 565 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop) 566 { 567 /* Header itself is a NOP packet */ 568 if (num_nop == 1) { 569 amdgpu_ring_write(ring, ring->funcs->nop); 570 return; 571 } 572 573 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 574 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); 575 576 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ 577 amdgpu_ring_insert_nop(ring, num_nop - 1); 578 } 579 580 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring) 581 { 582 struct amdgpu_device *adev = ring->adev; 583 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 584 uint32_t tmp = 0; 585 unsigned i; 586 int r; 587 588 WREG32(scratch, 0xCAFEDEAD); 589 r = amdgpu_ring_alloc(ring, 5); 590 if (r) { 591 drm_err(adev_to_drm(adev), "cp failed to lock ring %d (%d).\n", 592 ring->idx, r); 593 return r; 594 } 595 596 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) { 597 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF); 598 } else { 599 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 600 amdgpu_ring_write(ring, scratch - 601 PACKET3_SET_UCONFIG_REG_START); 602 amdgpu_ring_write(ring, 0xDEADBEEF); 603 } 604 amdgpu_ring_commit(ring); 605 606 for (i = 0; i < adev->usec_timeout; i++) { 607 tmp = RREG32(scratch); 608 if (tmp == 0xDEADBEEF) 609 break; 610 if (amdgpu_emu_mode == 1) 611 msleep(1); 612 else 613 udelay(1); 614 } 615 616 if (i >= adev->usec_timeout) 617 r = -ETIMEDOUT; 618 return r; 619 } 620 621 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 622 { 623 struct amdgpu_device *adev = ring->adev; 624 struct amdgpu_ib ib; 625 struct dma_fence *f = NULL; 626 unsigned index; 627 uint64_t gpu_addr; 628 uint32_t *cpu_ptr; 629 long r; 630 631 /* MES KIQ fw hasn't indirect buffer support for now */ 632 if (adev->enable_mes_kiq && 633 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 634 return 0; 635 636 memset(&ib, 0, sizeof(ib)); 637 638 r = amdgpu_device_wb_get(adev, &index); 639 if (r) 640 return r; 641 642 gpu_addr = adev->wb.gpu_addr + (index * 4); 643 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 644 cpu_ptr = &adev->wb.wb[index]; 645 646 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 647 if (r) { 648 drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r); 649 goto err1; 650 } 651 652 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 653 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 654 ib.ptr[2] = lower_32_bits(gpu_addr); 655 ib.ptr[3] = upper_32_bits(gpu_addr); 656 ib.ptr[4] = 0xDEADBEEF; 657 ib.length_dw = 5; 658 659 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 660 if (r) 661 goto err2; 662 663 r = dma_fence_wait_timeout(f, false, timeout); 664 if (r == 0) { 665 r = -ETIMEDOUT; 666 goto err2; 667 } else if (r < 0) { 668 goto err2; 669 } 670 671 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF) 672 r = 0; 673 else 674 r = -EINVAL; 675 err2: 676 amdgpu_ib_free(&ib, NULL); 677 dma_fence_put(f); 678 err1: 679 amdgpu_device_wb_free(adev, index); 680 return r; 681 } 682 683 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev) 684 { 685 amdgpu_ucode_release(&adev->gfx.pfp_fw); 686 amdgpu_ucode_release(&adev->gfx.me_fw); 687 amdgpu_ucode_release(&adev->gfx.rlc_fw); 688 amdgpu_ucode_release(&adev->gfx.mec_fw); 689 690 kfree(adev->gfx.rlc.register_list_format); 691 } 692 693 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix) 694 { 695 const struct psp_firmware_header_v1_0 *toc_hdr; 696 int err = 0; 697 698 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, 699 AMDGPU_UCODE_REQUIRED, 700 "amdgpu/%s_toc.bin", ucode_prefix); 701 if (err) 702 goto out; 703 704 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data; 705 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version); 706 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version); 707 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes); 708 adev->psp.toc.start_addr = (uint8_t *)toc_hdr + 709 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes); 710 return 0; 711 out: 712 amdgpu_ucode_release(&adev->psp.toc_fw); 713 return err; 714 } 715 716 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev) 717 { 718 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 719 case IP_VERSION(11, 0, 0): 720 case IP_VERSION(11, 0, 2): 721 case IP_VERSION(11, 0, 3): 722 if ((adev->gfx.me_fw_version >= 1505) && 723 (adev->gfx.pfp_fw_version >= 1600) && 724 (adev->gfx.mec_fw_version >= 512)) { 725 if (amdgpu_sriov_vf(adev)) 726 adev->gfx.cp_gfx_shadow = true; 727 else 728 adev->gfx.cp_gfx_shadow = false; 729 } 730 break; 731 default: 732 adev->gfx.cp_gfx_shadow = false; 733 break; 734 } 735 } 736 737 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) 738 { 739 char ucode_prefix[25]; 740 int err; 741 const struct rlc_firmware_header_v2_0 *rlc_hdr; 742 uint16_t version_major; 743 uint16_t version_minor; 744 745 DRM_DEBUG("\n"); 746 747 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 748 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 749 AMDGPU_UCODE_REQUIRED, 750 "amdgpu/%s_pfp.bin", ucode_prefix); 751 if (err) 752 goto out; 753 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/ 754 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version( 755 (union amdgpu_firmware_header *) 756 adev->gfx.pfp_fw->data, 2, 0); 757 if (adev->gfx.rs64_enable) { 758 dev_info(adev->dev, "CP RS64 enable\n"); 759 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP); 760 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK); 761 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK); 762 } else { 763 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 764 } 765 766 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 767 AMDGPU_UCODE_REQUIRED, 768 "amdgpu/%s_me.bin", ucode_prefix); 769 if (err) 770 goto out; 771 if (adev->gfx.rs64_enable) { 772 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME); 773 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK); 774 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK); 775 } else { 776 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 777 } 778 779 if (!amdgpu_sriov_vf(adev)) { 780 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) && 781 adev->pdev->revision == 0xCE) 782 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 783 AMDGPU_UCODE_REQUIRED, 784 "amdgpu/gc_11_0_0_rlc_1.bin"); 785 else if (amdgpu_is_kicker_fw(adev)) 786 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 787 AMDGPU_UCODE_REQUIRED, 788 "amdgpu/%s_rlc_kicker.bin", ucode_prefix); 789 else 790 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 791 AMDGPU_UCODE_REQUIRED, 792 "amdgpu/%s_rlc.bin", ucode_prefix); 793 if (err) 794 goto out; 795 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 796 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 797 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 798 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 799 if (err) 800 goto out; 801 } 802 803 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 804 AMDGPU_UCODE_REQUIRED, 805 "amdgpu/%s_mec.bin", ucode_prefix); 806 if (err) 807 goto out; 808 if (adev->gfx.rs64_enable) { 809 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC); 810 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK); 811 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK); 812 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK); 813 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK); 814 } else { 815 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 816 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 817 } 818 819 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 820 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix); 821 822 /* only one MEC for gfx 11.0.0. */ 823 adev->gfx.mec2_fw = NULL; 824 825 gfx_v11_0_check_fw_cp_gfx_shadow(adev); 826 827 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) { 828 err = adev->gfx.imu.funcs->init_microcode(adev); 829 if (err) 830 DRM_ERROR("Failed to init imu firmware!\n"); 831 return err; 832 } 833 834 out: 835 if (err) { 836 amdgpu_ucode_release(&adev->gfx.pfp_fw); 837 amdgpu_ucode_release(&adev->gfx.me_fw); 838 amdgpu_ucode_release(&adev->gfx.rlc_fw); 839 amdgpu_ucode_release(&adev->gfx.mec_fw); 840 } 841 842 return err; 843 } 844 845 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev) 846 { 847 u32 count = 0; 848 const struct cs_section_def *sect = NULL; 849 const struct cs_extent_def *ext = NULL; 850 851 /* begin clear state */ 852 count += 2; 853 /* context control state */ 854 count += 3; 855 856 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 857 for (ext = sect->section; ext->extent != NULL; ++ext) { 858 if (sect->id == SECT_CONTEXT) 859 count += 2 + ext->reg_count; 860 else 861 return 0; 862 } 863 } 864 865 /* set PA_SC_TILE_STEERING_OVERRIDE */ 866 count += 3; 867 /* end clear state */ 868 count += 2; 869 /* clear state */ 870 count += 2; 871 872 return count; 873 } 874 875 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 876 { 877 u32 count = 0; 878 int ctx_reg_offset; 879 880 if (adev->gfx.rlc.cs_data == NULL) 881 return; 882 if (buffer == NULL) 883 return; 884 885 count = amdgpu_gfx_csb_preamble_start(buffer); 886 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 887 888 ctx_reg_offset = SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 889 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 890 buffer[count++] = cpu_to_le32(ctx_reg_offset); 891 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override); 892 893 amdgpu_gfx_csb_preamble_end(buffer, count); 894 } 895 896 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev) 897 { 898 /* clear state block */ 899 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 900 &adev->gfx.rlc.clear_state_gpu_addr, 901 (void **)&adev->gfx.rlc.cs_ptr); 902 903 /* jump table block */ 904 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 905 &adev->gfx.rlc.cp_table_gpu_addr, 906 (void **)&adev->gfx.rlc.cp_table_ptr); 907 } 908 909 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 910 { 911 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 912 913 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 914 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 915 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1); 916 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2); 917 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3); 918 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL); 919 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX); 920 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0); 921 adev->gfx.rlc.rlcg_reg_access_supported = true; 922 } 923 924 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev) 925 { 926 const struct cs_section_def *cs_data; 927 int r; 928 929 adev->gfx.rlc.cs_data = gfx11_cs_data; 930 931 cs_data = adev->gfx.rlc.cs_data; 932 933 if (cs_data) { 934 /* init clear state block */ 935 r = amdgpu_gfx_rlc_init_csb(adev); 936 if (r) 937 return r; 938 } 939 940 /* init spm vmid with 0xf */ 941 if (adev->gfx.rlc.funcs->update_spm_vmid) 942 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf); 943 944 return 0; 945 } 946 947 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev) 948 { 949 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 950 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 951 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL); 952 } 953 954 static void gfx_v11_0_me_init(struct amdgpu_device *adev) 955 { 956 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 957 958 amdgpu_gfx_graphics_queue_acquire(adev); 959 } 960 961 static int gfx_v11_0_mec_init(struct amdgpu_device *adev) 962 { 963 int r; 964 u32 *hpd; 965 size_t mec_hpd_size; 966 967 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 968 969 /* take ownership of the relevant compute queues */ 970 amdgpu_gfx_compute_queue_acquire(adev); 971 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE; 972 973 if (mec_hpd_size) { 974 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 975 AMDGPU_GEM_DOMAIN_GTT, 976 &adev->gfx.mec.hpd_eop_obj, 977 &adev->gfx.mec.hpd_eop_gpu_addr, 978 (void **)&hpd); 979 if (r) { 980 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 981 gfx_v11_0_mec_fini(adev); 982 return r; 983 } 984 985 memset(hpd, 0, mec_hpd_size); 986 987 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 988 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 989 } 990 991 return 0; 992 } 993 994 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address) 995 { 996 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 997 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 998 (address << SQ_IND_INDEX__INDEX__SHIFT)); 999 return RREG32_SOC15(GC, 0, regSQ_IND_DATA); 1000 } 1001 1002 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave, 1003 uint32_t thread, uint32_t regno, 1004 uint32_t num, uint32_t *out) 1005 { 1006 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 1007 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1008 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1009 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) | 1010 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1011 while (num--) 1012 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA); 1013 } 1014 1015 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1016 { 1017 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE 1018 * field when performing a select_se_sh so it should be 1019 * zero here */ 1020 WARN_ON(simd != 0); 1021 1022 /* type 3 wave data */ 1023 dst[(*no_fields)++] = 3; 1024 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS); 1025 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO); 1026 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI); 1027 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO); 1028 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI); 1029 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1); 1030 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2); 1031 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC); 1032 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC); 1033 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS); 1034 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS); 1035 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2); 1036 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1); 1037 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0); 1038 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE); 1039 } 1040 1041 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1042 uint32_t wave, uint32_t start, 1043 uint32_t size, uint32_t *dst) 1044 { 1045 WARN_ON(simd != 0); 1046 1047 wave_read_regs( 1048 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size, 1049 dst); 1050 } 1051 1052 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1053 uint32_t wave, uint32_t thread, 1054 uint32_t start, uint32_t size, 1055 uint32_t *dst) 1056 { 1057 wave_read_regs( 1058 adev, wave, thread, 1059 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1060 } 1061 1062 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev, 1063 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1064 { 1065 soc21_grbm_select(adev, me, pipe, q, vm); 1066 } 1067 1068 /* all sizes are in bytes */ 1069 #define MQD_SHADOW_BASE_SIZE 73728 1070 #define MQD_SHADOW_BASE_ALIGNMENT 256 1071 #define MQD_FWWORKAREA_SIZE 484 1072 #define MQD_FWWORKAREA_ALIGNMENT 256 1073 1074 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev, 1075 struct amdgpu_gfx_shadow_info *shadow_info) 1076 { 1077 /* for gfx */ 1078 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE; 1079 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT; 1080 shadow_info->csa_size = MQD_FWWORKAREA_SIZE; 1081 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT; 1082 /* for compute */ 1083 shadow_info->eop_size = GFX11_MEC_HPD_SIZE; 1084 shadow_info->eop_alignment = 256; 1085 } 1086 1087 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev, 1088 struct amdgpu_gfx_shadow_info *shadow_info, 1089 bool skip_check) 1090 { 1091 if (adev->gfx.cp_gfx_shadow || skip_check) { 1092 gfx_v11_0_get_gfx_shadow_info_nocheck(adev, shadow_info); 1093 return 0; 1094 } else { 1095 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info)); 1096 return -ENOTSUPP; 1097 } 1098 } 1099 1100 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = { 1101 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter, 1102 .select_se_sh = &gfx_v11_0_select_se_sh, 1103 .read_wave_data = &gfx_v11_0_read_wave_data, 1104 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs, 1105 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs, 1106 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q, 1107 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk, 1108 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info, 1109 .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask, 1110 }; 1111 1112 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) 1113 { 1114 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1115 case IP_VERSION(11, 0, 0): 1116 case IP_VERSION(11, 0, 2): 1117 adev->gfx.config.max_hw_contexts = 8; 1118 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1119 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1120 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1121 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1122 break; 1123 case IP_VERSION(11, 0, 3): 1124 adev->gfx.ras = &gfx_v11_0_3_ras; 1125 adev->gfx.config.max_hw_contexts = 8; 1126 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1127 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1128 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1129 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1130 break; 1131 case IP_VERSION(11, 0, 1): 1132 case IP_VERSION(11, 0, 4): 1133 case IP_VERSION(11, 5, 0): 1134 case IP_VERSION(11, 5, 1): 1135 case IP_VERSION(11, 5, 2): 1136 case IP_VERSION(11, 5, 3): 1137 case IP_VERSION(11, 5, 4): 1138 case IP_VERSION(11, 5, 6): 1139 case IP_VERSION(11, 7, 0): 1140 case IP_VERSION(11, 7, 1): 1141 adev->gfx.config.max_hw_contexts = 8; 1142 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1143 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1144 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 1145 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300; 1146 break; 1147 default: 1148 BUG(); 1149 break; 1150 } 1151 1152 return 0; 1153 } 1154 1155 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id, 1156 int me, int pipe, int queue) 1157 { 1158 struct amdgpu_ring *ring; 1159 unsigned int irq_type; 1160 unsigned int hw_prio; 1161 1162 ring = &adev->gfx.gfx_ring[ring_id]; 1163 1164 ring->me = me; 1165 ring->pipe = pipe; 1166 ring->queue = queue; 1167 1168 ring->ring_obj = NULL; 1169 ring->use_doorbell = true; 1170 if (adev->gfx.disable_kq) { 1171 ring->no_scheduler = true; 1172 ring->no_user_submission = true; 1173 } 1174 1175 if (!ring_id) 1176 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 1177 else 1178 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1; 1179 ring->vm_hub = AMDGPU_GFXHUB(0); 1180 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1181 1182 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe; 1183 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ? 1184 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1185 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1186 hw_prio, NULL); 1187 } 1188 1189 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1190 int mec, int pipe, int queue) 1191 { 1192 int r; 1193 unsigned irq_type; 1194 struct amdgpu_ring *ring; 1195 unsigned int hw_prio; 1196 1197 ring = &adev->gfx.compute_ring[ring_id]; 1198 1199 /* mec0 is me1 */ 1200 ring->me = mec + 1; 1201 ring->pipe = pipe; 1202 ring->queue = queue; 1203 1204 ring->ring_obj = NULL; 1205 ring->use_doorbell = true; 1206 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 1207 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1208 + (ring_id * GFX11_MEC_HPD_SIZE); 1209 ring->vm_hub = AMDGPU_GFXHUB(0); 1210 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1211 1212 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1213 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1214 + ring->pipe; 1215 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1216 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1217 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1218 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1219 hw_prio, NULL); 1220 if (r) 1221 return r; 1222 1223 return 0; 1224 } 1225 1226 static struct { 1227 SOC21_FIRMWARE_ID id; 1228 unsigned int offset; 1229 unsigned int size; 1230 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX]; 1231 1232 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc) 1233 { 1234 RLC_TABLE_OF_CONTENT *ucode = rlc_toc; 1235 1236 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) && 1237 (ucode->id < SOC21_FIRMWARE_ID_MAX)) { 1238 rlc_autoload_info[ucode->id].id = ucode->id; 1239 rlc_autoload_info[ucode->id].offset = ucode->offset * 4; 1240 rlc_autoload_info[ucode->id].size = ucode->size * 4; 1241 1242 ucode++; 1243 } 1244 } 1245 1246 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev) 1247 { 1248 uint32_t total_size = 0; 1249 SOC21_FIRMWARE_ID id; 1250 1251 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr); 1252 1253 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++) 1254 total_size += rlc_autoload_info[id].size; 1255 1256 /* In case the offset in rlc toc ucode is aligned */ 1257 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset) 1258 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset + 1259 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size; 1260 1261 return total_size; 1262 } 1263 1264 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev) 1265 { 1266 int r; 1267 uint32_t total_size; 1268 1269 total_size = gfx_v11_0_calc_toc_total_size(adev); 1270 1271 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024, 1272 AMDGPU_GEM_DOMAIN_VRAM | 1273 AMDGPU_GEM_DOMAIN_GTT, 1274 &adev->gfx.rlc.rlc_autoload_bo, 1275 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1276 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1277 1278 if (r) { 1279 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r); 1280 return r; 1281 } 1282 1283 return 0; 1284 } 1285 1286 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev, 1287 SOC21_FIRMWARE_ID id, 1288 const void *fw_data, 1289 uint32_t fw_size, 1290 uint32_t *fw_autoload_mask) 1291 { 1292 uint32_t toc_offset; 1293 uint32_t toc_fw_size; 1294 char *ptr = adev->gfx.rlc.rlc_autoload_ptr; 1295 1296 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX) 1297 return; 1298 1299 toc_offset = rlc_autoload_info[id].offset; 1300 toc_fw_size = rlc_autoload_info[id].size; 1301 1302 if (fw_size == 0) 1303 fw_size = toc_fw_size; 1304 1305 if (fw_size > toc_fw_size) 1306 fw_size = toc_fw_size; 1307 1308 memcpy(ptr + toc_offset, fw_data, fw_size); 1309 1310 if (fw_size < toc_fw_size) 1311 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); 1312 1313 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME)) 1314 *(uint64_t *)fw_autoload_mask |= 1ULL << id; 1315 } 1316 1317 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev, 1318 uint32_t *fw_autoload_mask) 1319 { 1320 void *data; 1321 uint32_t size; 1322 uint64_t *toc_ptr; 1323 1324 *(uint64_t *)fw_autoload_mask |= 0x1; 1325 1326 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask); 1327 1328 data = adev->psp.toc.start_addr; 1329 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size; 1330 1331 toc_ptr = (uint64_t *)data + size / 8 - 1; 1332 *toc_ptr = *(uint64_t *)fw_autoload_mask; 1333 1334 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC, 1335 data, size, fw_autoload_mask); 1336 } 1337 1338 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev, 1339 uint32_t *fw_autoload_mask) 1340 { 1341 const __le32 *fw_data; 1342 uint32_t fw_size; 1343 const struct gfx_firmware_header_v1_0 *cp_hdr; 1344 const struct gfx_firmware_header_v2_0 *cpv2_hdr; 1345 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1346 const struct rlc_firmware_header_v2_2 *rlcv22_hdr; 1347 uint16_t version_major, version_minor; 1348 1349 if (adev->gfx.rs64_enable) { 1350 /* pfp ucode */ 1351 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1352 adev->gfx.pfp_fw->data; 1353 /* instruction */ 1354 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1355 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1356 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1357 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP, 1358 fw_data, fw_size, fw_autoload_mask); 1359 /* data */ 1360 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1361 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1362 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1363 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK, 1364 fw_data, fw_size, fw_autoload_mask); 1365 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK, 1366 fw_data, fw_size, fw_autoload_mask); 1367 /* me ucode */ 1368 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1369 adev->gfx.me_fw->data; 1370 /* instruction */ 1371 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1372 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1373 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1374 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME, 1375 fw_data, fw_size, fw_autoload_mask); 1376 /* data */ 1377 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1378 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1379 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1380 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK, 1381 fw_data, fw_size, fw_autoload_mask); 1382 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK, 1383 fw_data, fw_size, fw_autoload_mask); 1384 /* mec ucode */ 1385 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1386 adev->gfx.mec_fw->data; 1387 /* instruction */ 1388 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1389 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1390 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1391 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC, 1392 fw_data, fw_size, fw_autoload_mask); 1393 /* data */ 1394 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1395 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1396 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1397 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK, 1398 fw_data, fw_size, fw_autoload_mask); 1399 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK, 1400 fw_data, fw_size, fw_autoload_mask); 1401 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK, 1402 fw_data, fw_size, fw_autoload_mask); 1403 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK, 1404 fw_data, fw_size, fw_autoload_mask); 1405 } else { 1406 /* pfp ucode */ 1407 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1408 adev->gfx.pfp_fw->data; 1409 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1410 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1411 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1412 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP, 1413 fw_data, fw_size, fw_autoload_mask); 1414 1415 /* me ucode */ 1416 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1417 adev->gfx.me_fw->data; 1418 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1419 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1420 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1421 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME, 1422 fw_data, fw_size, fw_autoload_mask); 1423 1424 /* mec ucode */ 1425 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1426 adev->gfx.mec_fw->data; 1427 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1428 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1429 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1430 cp_hdr->jt_size * 4; 1431 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC, 1432 fw_data, fw_size, fw_autoload_mask); 1433 } 1434 1435 /* rlc ucode */ 1436 rlc_hdr = (const struct rlc_firmware_header_v2_0 *) 1437 adev->gfx.rlc_fw->data; 1438 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1439 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes)); 1440 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes); 1441 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE, 1442 fw_data, fw_size, fw_autoload_mask); 1443 1444 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1445 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1446 if (version_major == 2) { 1447 if (version_minor >= 2) { 1448 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 1449 1450 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1451 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes)); 1452 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes); 1453 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE, 1454 fw_data, fw_size, fw_autoload_mask); 1455 1456 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1457 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes)); 1458 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes); 1459 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT, 1460 fw_data, fw_size, fw_autoload_mask); 1461 } 1462 } 1463 } 1464 1465 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev, 1466 uint32_t *fw_autoload_mask) 1467 { 1468 const __le32 *fw_data; 1469 uint32_t fw_size; 1470 const struct sdma_firmware_header_v2_0 *sdma_hdr; 1471 1472 sdma_hdr = (const struct sdma_firmware_header_v2_0 *) 1473 adev->sdma.instance[0].fw->data; 1474 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1475 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes)); 1476 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes); 1477 1478 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1479 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask); 1480 1481 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1482 le32_to_cpu(sdma_hdr->ctl_ucode_offset)); 1483 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes); 1484 1485 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1486 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask); 1487 } 1488 1489 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev, 1490 uint32_t *fw_autoload_mask) 1491 { 1492 const __le32 *fw_data; 1493 unsigned fw_size; 1494 const struct mes_firmware_header_v1_0 *mes_hdr; 1495 int pipe, ucode_id, data_id; 1496 1497 for (pipe = 0; pipe < 2; pipe++) { 1498 if (pipe==0) { 1499 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0; 1500 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK; 1501 } else { 1502 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1; 1503 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK; 1504 } 1505 1506 mes_hdr = (const struct mes_firmware_header_v1_0 *) 1507 adev->mes.fw[pipe]->data; 1508 1509 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1510 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); 1511 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); 1512 1513 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1514 ucode_id, fw_data, fw_size, fw_autoload_mask); 1515 1516 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1517 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); 1518 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); 1519 1520 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1521 data_id, fw_data, fw_size, fw_autoload_mask); 1522 } 1523 } 1524 1525 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) 1526 { 1527 uint32_t rlc_g_offset, rlc_g_size; 1528 uint64_t gpu_addr; 1529 uint32_t autoload_fw_id[2]; 1530 1531 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2); 1532 1533 /* RLC autoload sequence 2: copy ucode */ 1534 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id); 1535 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id); 1536 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id); 1537 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id); 1538 1539 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset; 1540 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size; 1541 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset; 1542 1543 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr)); 1544 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr)); 1545 1546 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size); 1547 1548 /* RLC autoload sequence 3: load IMU fw */ 1549 if (adev->gfx.imu.funcs->load_microcode) 1550 adev->gfx.imu.funcs->load_microcode(adev); 1551 /* RLC autoload sequence 4 init IMU fw */ 1552 if (adev->gfx.imu.funcs->setup_imu) 1553 adev->gfx.imu.funcs->setup_imu(adev); 1554 if (adev->gfx.imu.funcs->start_imu) 1555 adev->gfx.imu.funcs->start_imu(adev); 1556 1557 /* RLC autoload sequence 5 disable gpa mode */ 1558 gfx_v11_0_disable_gpa_mode(adev); 1559 1560 return 0; 1561 } 1562 1563 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev) 1564 { 1565 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 1566 uint32_t *ptr; 1567 uint32_t inst; 1568 1569 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL); 1570 if (!ptr) { 1571 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 1572 adev->gfx.ip_dump_core = NULL; 1573 } else { 1574 adev->gfx.ip_dump_core = ptr; 1575 } 1576 1577 /* Allocate memory for compute queue registers for all the instances */ 1578 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 1579 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1580 adev->gfx.mec.num_queue_per_pipe; 1581 1582 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1583 if (!ptr) { 1584 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 1585 adev->gfx.ip_dump_compute_queues = NULL; 1586 } else { 1587 adev->gfx.ip_dump_compute_queues = ptr; 1588 } 1589 1590 /* Allocate memory for gfx queue registers for all the instances */ 1591 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 1592 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me * 1593 adev->gfx.me.num_queue_per_pipe; 1594 1595 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1596 if (!ptr) { 1597 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n"); 1598 adev->gfx.ip_dump_gfx_queues = NULL; 1599 } else { 1600 adev->gfx.ip_dump_gfx_queues = ptr; 1601 } 1602 } 1603 1604 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) 1605 { 1606 int i, j, k, r, ring_id; 1607 int xcc_id = 0; 1608 struct amdgpu_device *adev = ip_block->adev; 1609 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */ 1610 1611 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler); 1612 1613 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1614 case IP_VERSION(11, 0, 0): 1615 case IP_VERSION(11, 0, 1): 1616 case IP_VERSION(11, 0, 2): 1617 case IP_VERSION(11, 0, 3): 1618 case IP_VERSION(11, 0, 4): 1619 case IP_VERSION(11, 5, 0): 1620 case IP_VERSION(11, 5, 1): 1621 case IP_VERSION(11, 5, 2): 1622 case IP_VERSION(11, 5, 3): 1623 case IP_VERSION(11, 5, 4): 1624 case IP_VERSION(11, 5, 6): 1625 case IP_VERSION(11, 7, 0): 1626 case IP_VERSION(11, 7, 1): 1627 adev->gfx.me.num_me = 1; 1628 adev->gfx.me.num_pipe_per_me = 1; 1629 adev->gfx.me.num_queue_per_pipe = 2; 1630 adev->gfx.mec.num_mec = 1; 1631 adev->gfx.mec.num_pipe_per_mec = 4; 1632 adev->gfx.mec.num_queue_per_pipe = 4; 1633 break; 1634 default: 1635 adev->gfx.me.num_me = 1; 1636 adev->gfx.me.num_pipe_per_me = 1; 1637 adev->gfx.me.num_queue_per_pipe = 1; 1638 adev->gfx.mec.num_mec = 1; 1639 adev->gfx.mec.num_pipe_per_mec = 4; 1640 adev->gfx.mec.num_queue_per_pipe = 8; 1641 break; 1642 } 1643 1644 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1645 case IP_VERSION(11, 0, 0): 1646 case IP_VERSION(11, 0, 2): 1647 case IP_VERSION(11, 0, 3): 1648 if (!adev->gfx.disable_uq && 1649 adev->gfx.me_fw_version >= 2420 && 1650 adev->gfx.pfp_fw_version >= 2580 && 1651 adev->gfx.mec_fw_version >= 2650 && 1652 adev->mes.fw_version[0] >= 120) { 1653 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1654 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1655 } 1656 break; 1657 case IP_VERSION(11, 0, 1): 1658 case IP_VERSION(11, 0, 4): 1659 case IP_VERSION(11, 5, 0): 1660 case IP_VERSION(11, 5, 1): 1661 case IP_VERSION(11, 5, 2): 1662 case IP_VERSION(11, 5, 3): 1663 /* add firmware version checks here */ 1664 if (0 && !adev->gfx.disable_uq) { 1665 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1666 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1667 } 1668 break; 1669 default: 1670 break; 1671 } 1672 1673 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1674 case IP_VERSION(11, 0, 0): 1675 case IP_VERSION(11, 0, 2): 1676 case IP_VERSION(11, 0, 3): 1677 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1678 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1679 if (adev->gfx.me_fw_version >= 2280 && 1680 adev->gfx.pfp_fw_version >= 2370 && 1681 adev->gfx.mec_fw_version >= 2450 && 1682 adev->mes.fw_version[0] >= 99) { 1683 adev->gfx.enable_cleaner_shader = true; 1684 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1685 if (r) { 1686 adev->gfx.enable_cleaner_shader = false; 1687 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1688 } 1689 } 1690 break; 1691 case IP_VERSION(11, 0, 1): 1692 case IP_VERSION(11, 0, 4): 1693 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1694 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1695 if (adev->gfx.pfp_fw_version >= 102 && 1696 adev->gfx.mec_fw_version >= 66 && 1697 adev->mes.fw_version[0] >= 128) { 1698 adev->gfx.enable_cleaner_shader = true; 1699 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1700 if (r) { 1701 adev->gfx.enable_cleaner_shader = false; 1702 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1703 } 1704 } 1705 break; 1706 case IP_VERSION(11, 5, 0): 1707 case IP_VERSION(11, 5, 1): 1708 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1709 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1710 if (adev->gfx.mec_fw_version >= 26 && 1711 adev->mes.fw_version[0] >= 114) { 1712 adev->gfx.enable_cleaner_shader = true; 1713 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1714 if (r) { 1715 adev->gfx.enable_cleaner_shader = false; 1716 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1717 } 1718 } 1719 break; 1720 case IP_VERSION(11, 5, 2): 1721 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1722 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1723 if (adev->gfx.me_fw_version >= 12 && 1724 adev->gfx.pfp_fw_version >= 15 && 1725 adev->gfx.mec_fw_version >= 15) { 1726 adev->gfx.enable_cleaner_shader = true; 1727 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1728 if (r) { 1729 adev->gfx.enable_cleaner_shader = false; 1730 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1731 } 1732 } 1733 break; 1734 case IP_VERSION(11, 5, 3): 1735 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1736 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1737 if (adev->gfx.me_fw_version >= 7 && 1738 adev->gfx.pfp_fw_version >= 8 && 1739 adev->gfx.mec_fw_version >= 8) { 1740 adev->gfx.enable_cleaner_shader = true; 1741 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1742 if (r) { 1743 adev->gfx.enable_cleaner_shader = false; 1744 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1745 } 1746 } 1747 break; 1748 case IP_VERSION(11, 5, 4): 1749 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1750 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1751 if (adev->gfx.me_fw_version >= 4 && 1752 adev->gfx.pfp_fw_version >= 7 && 1753 adev->gfx.mec_fw_version >= 5) { 1754 adev->gfx.enable_cleaner_shader = true; 1755 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1756 if (r) { 1757 adev->gfx.enable_cleaner_shader = false; 1758 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1759 } 1760 } 1761 break; 1762 default: 1763 adev->gfx.enable_cleaner_shader = false; 1764 break; 1765 } 1766 1767 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */ 1768 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) && 1769 amdgpu_sriov_is_pp_one_vf(adev)) 1770 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG; 1771 1772 /* EOP Event */ 1773 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1774 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT, 1775 &adev->gfx.eop_irq); 1776 if (r) 1777 return r; 1778 1779 /* Bad opcode Event */ 1780 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1781 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR, 1782 &adev->gfx.bad_op_irq); 1783 if (r) 1784 return r; 1785 1786 /* Privileged reg */ 1787 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1788 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT, 1789 &adev->gfx.priv_reg_irq); 1790 if (r) 1791 return r; 1792 1793 /* Privileged inst */ 1794 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1795 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT, 1796 &adev->gfx.priv_inst_irq); 1797 if (r) 1798 return r; 1799 1800 /* FED error */ 1801 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX, 1802 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT, 1803 &adev->gfx.rlc_gc_fed_irq); 1804 if (r) 1805 return r; 1806 1807 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1808 1809 gfx_v11_0_me_init(adev); 1810 1811 r = gfx_v11_0_rlc_init(adev); 1812 if (r) { 1813 DRM_ERROR("Failed to init rlc BOs!\n"); 1814 return r; 1815 } 1816 1817 r = gfx_v11_0_mec_init(adev); 1818 if (r) { 1819 DRM_ERROR("Failed to init MEC BOs!\n"); 1820 return r; 1821 } 1822 1823 if (adev->gfx.num_gfx_rings) { 1824 ring_id = 0; 1825 /* set up the gfx ring */ 1826 for (i = 0; i < adev->gfx.me.num_me; i++) { 1827 for (j = 0; j < num_queue_per_pipe; j++) { 1828 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 1829 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j)) 1830 continue; 1831 1832 r = gfx_v11_0_gfx_ring_init(adev, ring_id, 1833 i, k, j); 1834 if (r) 1835 return r; 1836 ring_id++; 1837 } 1838 } 1839 } 1840 } 1841 1842 if (adev->gfx.num_compute_rings) { 1843 ring_id = 0; 1844 /* set up the compute queues - allocate horizontally across pipes */ 1845 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1846 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1847 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1848 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 1849 k, j)) 1850 continue; 1851 1852 r = gfx_v11_0_compute_ring_init(adev, ring_id, 1853 i, k, j); 1854 if (r) 1855 return r; 1856 1857 ring_id++; 1858 } 1859 } 1860 } 1861 } 1862 1863 adev->gfx.gfx_supported_reset = 1864 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 1865 adev->gfx.compute_supported_reset = 1866 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 1867 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1868 case IP_VERSION(11, 0, 0): 1869 case IP_VERSION(11, 0, 2): 1870 case IP_VERSION(11, 0, 3): 1871 if ((adev->gfx.me_fw_version >= 2280) && 1872 (adev->gfx.mec_fw_version >= 2410) && 1873 !amdgpu_sriov_vf(adev) && 1874 !adev->debug_disable_gpu_ring_reset) { 1875 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1876 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1877 } 1878 break; 1879 default: 1880 if (!amdgpu_sriov_vf(adev) && 1881 !adev->debug_disable_gpu_ring_reset) { 1882 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1883 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1884 } 1885 break; 1886 } 1887 1888 if (!adev->enable_mes_kiq) { 1889 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0); 1890 if (r) { 1891 DRM_ERROR("Failed to init KIQ BOs!\n"); 1892 return r; 1893 } 1894 1895 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 1896 if (r) 1897 return r; 1898 } 1899 1900 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0); 1901 if (r) 1902 return r; 1903 1904 /* allocate visible FB for rlc auto-loading fw */ 1905 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 1906 r = gfx_v11_0_rlc_autoload_buffer_init(adev); 1907 if (r) 1908 return r; 1909 } 1910 1911 r = gfx_v11_0_gpu_early_init(adev); 1912 if (r) 1913 return r; 1914 1915 if (amdgpu_gfx_ras_sw_init(adev)) { 1916 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 1917 return -EINVAL; 1918 } 1919 1920 gfx_v11_0_alloc_ip_dump(adev); 1921 1922 r = amdgpu_gfx_sysfs_init(adev); 1923 if (r) 1924 return r; 1925 1926 return 0; 1927 } 1928 1929 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev) 1930 { 1931 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj, 1932 &adev->gfx.pfp.pfp_fw_gpu_addr, 1933 (void **)&adev->gfx.pfp.pfp_fw_ptr); 1934 1935 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj, 1936 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 1937 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 1938 } 1939 1940 static void gfx_v11_0_me_fini(struct amdgpu_device *adev) 1941 { 1942 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj, 1943 &adev->gfx.me.me_fw_gpu_addr, 1944 (void **)&adev->gfx.me.me_fw_ptr); 1945 1946 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj, 1947 &adev->gfx.me.me_fw_data_gpu_addr, 1948 (void **)&adev->gfx.me.me_fw_data_ptr); 1949 } 1950 1951 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev) 1952 { 1953 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo, 1954 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1955 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1956 } 1957 1958 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block) 1959 { 1960 int i; 1961 struct amdgpu_device *adev = ip_block->adev; 1962 1963 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 1964 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 1965 for (i = 0; i < adev->gfx.num_compute_rings; i++) 1966 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 1967 1968 amdgpu_gfx_mqd_sw_fini(adev, 0); 1969 1970 if (!adev->enable_mes_kiq) { 1971 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 1972 amdgpu_gfx_kiq_fini(adev, 0); 1973 } 1974 1975 amdgpu_gfx_cleaner_shader_sw_fini(adev); 1976 1977 gfx_v11_0_pfp_fini(adev); 1978 gfx_v11_0_me_fini(adev); 1979 gfx_v11_0_rlc_fini(adev); 1980 gfx_v11_0_mec_fini(adev); 1981 1982 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 1983 gfx_v11_0_rlc_autoload_buffer_fini(adev); 1984 1985 gfx_v11_0_free_microcode(adev); 1986 1987 amdgpu_gfx_sysfs_fini(adev); 1988 1989 kfree(adev->gfx.ip_dump_core); 1990 kfree(adev->gfx.ip_dump_compute_queues); 1991 kfree(adev->gfx.ip_dump_gfx_queues); 1992 1993 return 0; 1994 } 1995 1996 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 1997 u32 sh_num, u32 instance, int xcc_id) 1998 { 1999 u32 data; 2000 2001 if (instance == 0xffffffff) 2002 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, 2003 INSTANCE_BROADCAST_WRITES, 1); 2004 else 2005 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, 2006 instance); 2007 2008 if (se_num == 0xffffffff) 2009 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 2010 1); 2011 else 2012 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2013 2014 if (sh_num == 0xffffffff) 2015 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES, 2016 1); 2017 else 2018 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num); 2019 2020 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data); 2021 } 2022 2023 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev) 2024 { 2025 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask; 2026 2027 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE); 2028 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask, 2029 CC_GC_SA_UNIT_DISABLE, 2030 SA_DISABLE); 2031 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE); 2032 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask, 2033 GC_USER_SA_UNIT_DISABLE, 2034 SA_DISABLE); 2035 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * 2036 adev->gfx.config.max_shader_engines); 2037 2038 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask)); 2039 } 2040 2041 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2042 { 2043 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask; 2044 u32 rb_mask; 2045 2046 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE); 2047 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask, 2048 CC_RB_BACKEND_DISABLE, 2049 BACKEND_DISABLE); 2050 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE); 2051 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask, 2052 GC_USER_RB_BACKEND_DISABLE, 2053 BACKEND_DISABLE); 2054 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se * 2055 adev->gfx.config.max_shader_engines); 2056 2057 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask)); 2058 } 2059 2060 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev) 2061 { 2062 u32 rb_bitmap_per_sa; 2063 u32 rb_bitmap_width_per_sa; 2064 u32 max_sa; 2065 u32 active_sa_bitmap; 2066 u32 global_active_rb_bitmap; 2067 u32 active_rb_bitmap = 0; 2068 u32 i; 2069 2070 /* query sa bitmap from SA_UNIT_DISABLE registers */ 2071 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev); 2072 /* query rb bitmap from RB_BACKEND_DISABLE registers */ 2073 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev); 2074 2075 /* generate active rb bitmap according to active sa bitmap */ 2076 max_sa = adev->gfx.config.max_shader_engines * 2077 adev->gfx.config.max_sh_per_se; 2078 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se / 2079 adev->gfx.config.max_sh_per_se; 2080 rb_bitmap_per_sa = amdgpu_gfx_create_bitmask(rb_bitmap_width_per_sa); 2081 2082 for (i = 0; i < max_sa; i++) { 2083 if (active_sa_bitmap & (1 << i)) 2084 active_rb_bitmap |= (rb_bitmap_per_sa << (i * rb_bitmap_width_per_sa)); 2085 } 2086 2087 active_rb_bitmap &= global_active_rb_bitmap; 2088 adev->gfx.config.backend_enable_mask = active_rb_bitmap; 2089 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap); 2090 } 2091 2092 #define DEFAULT_SH_MEM_BASES (0x6000) 2093 #define LDS_APP_BASE 0x1 2094 #define SCRATCH_APP_BASE 0x2 2095 2096 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev) 2097 { 2098 int i; 2099 uint32_t sh_mem_bases; 2100 uint32_t data; 2101 2102 /* 2103 * Configure apertures: 2104 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2105 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2106 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2107 */ 2108 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) | 2109 SCRATCH_APP_BASE; 2110 2111 mutex_lock(&adev->srbm_mutex); 2112 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2113 soc21_grbm_select(adev, 0, 0, 0, i); 2114 /* CP and shaders */ 2115 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2116 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases); 2117 2118 /* Enable trap for each kfd vmid. */ 2119 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL); 2120 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 2121 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data); 2122 } 2123 soc21_grbm_select(adev, 0, 0, 0, 0); 2124 mutex_unlock(&adev->srbm_mutex); 2125 2126 /* 2127 * Initialize all compute VMIDs to have no GDS, GWS, or OA 2128 * access. These should be enabled by FW for target VMIDs. 2129 */ 2130 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2131 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0); 2132 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0); 2133 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0); 2134 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0); 2135 } 2136 } 2137 2138 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev) 2139 { 2140 int vmid; 2141 2142 /* 2143 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2144 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2145 * the driver can enable them for graphics. VMID0 should maintain 2146 * access so that HWS firmware can save/restore entries. 2147 */ 2148 for (vmid = 1; vmid < 16; vmid++) { 2149 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0); 2150 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0); 2151 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0); 2152 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0); 2153 } 2154 } 2155 2156 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev) 2157 { 2158 /* TODO: harvest feature to be added later. */ 2159 } 2160 2161 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev) 2162 { 2163 /* TCCs are global (not instanced). */ 2164 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) | 2165 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE); 2166 2167 adev->gfx.config.tcc_disabled_mask = 2168 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) | 2169 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16); 2170 } 2171 2172 static void gfx_v11_0_constants_init(struct amdgpu_device *adev) 2173 { 2174 u32 tmp; 2175 int i; 2176 2177 if (!amdgpu_sriov_vf(adev)) 2178 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2179 2180 gfx_v11_0_setup_rb(adev); 2181 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info); 2182 gfx_v11_0_get_tcc_info(adev); 2183 adev->gfx.config.pa_sc_tile_steering_override = 0; 2184 2185 /* Set whether texture coordinate truncation is conformant. */ 2186 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); 2187 adev->gfx.config.ta_cntl2_truncate_coord_mode = 2188 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); 2189 2190 /* XXX SH_MEM regs */ 2191 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2192 mutex_lock(&adev->srbm_mutex); 2193 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2194 soc21_grbm_select(adev, 0, 0, 0, i); 2195 /* CP and shaders */ 2196 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2197 if (i != 0) { 2198 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2199 (adev->gmc.private_aperture_start >> 48)); 2200 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2201 (adev->gmc.shared_aperture_start >> 48)); 2202 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp); 2203 } 2204 } 2205 soc21_grbm_select(adev, 0, 0, 0, 0); 2206 2207 mutex_unlock(&adev->srbm_mutex); 2208 2209 gfx_v11_0_init_compute_vmid(adev); 2210 gfx_v11_0_init_gds_vmid(adev); 2211 } 2212 2213 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev, 2214 int me, int pipe) 2215 { 2216 if (me != 0) 2217 return 0; 2218 2219 switch (pipe) { 2220 case 0: 2221 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 2222 case 1: 2223 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 2224 default: 2225 return 0; 2226 } 2227 } 2228 2229 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev, 2230 int me, int pipe) 2231 { 2232 /* 2233 * amdgpu controls only the first MEC. That's why this function only 2234 * handles the setting of interrupts for this specific MEC. All other 2235 * pipes' interrupts are set by amdkfd. 2236 */ 2237 if (me != 1) 2238 return 0; 2239 2240 switch (pipe) { 2241 case 0: 2242 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 2243 case 1: 2244 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 2245 case 2: 2246 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 2247 case 3: 2248 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 2249 default: 2250 return 0; 2251 } 2252 } 2253 2254 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2255 bool enable) 2256 { 2257 u32 tmp, cp_int_cntl_reg; 2258 int i, j; 2259 2260 if (amdgpu_sriov_vf(adev)) 2261 return; 2262 2263 for (i = 0; i < adev->gfx.me.num_me; i++) { 2264 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 2265 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 2266 2267 if (cp_int_cntl_reg) { 2268 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 2269 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 2270 enable ? 1 : 0); 2271 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 2272 enable ? 1 : 0); 2273 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 2274 enable ? 1 : 0); 2275 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 2276 enable ? 1 : 0); 2277 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp); 2278 } 2279 } 2280 } 2281 } 2282 2283 static int gfx_v11_0_init_csb(struct amdgpu_device *adev) 2284 { 2285 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2286 2287 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI, 2288 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2289 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO, 2290 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2291 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); 2292 2293 return 0; 2294 } 2295 2296 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev) 2297 { 2298 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL); 2299 2300 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); 2301 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp); 2302 } 2303 2304 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev) 2305 { 2306 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2307 udelay(50); 2308 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2309 udelay(50); 2310 } 2311 2312 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev, 2313 bool enable) 2314 { 2315 uint32_t rlc_pg_cntl; 2316 2317 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 2318 2319 if (!enable) { 2320 /* RLC_PG_CNTL[23] = 0 (default) 2321 * RLC will wait for handshake acks with SMU 2322 * GFXOFF will be enabled 2323 * RLC_PG_CNTL[23] = 1 2324 * RLC will not issue any message to SMU 2325 * hence no handshake between SMU & RLC 2326 * GFXOFF will be disabled 2327 */ 2328 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2329 } else 2330 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2331 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl); 2332 } 2333 2334 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev) 2335 { 2336 /* TODO: enable rlc & smu handshake until smu 2337 * and gfxoff feature works as expected */ 2338 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK)) 2339 gfx_v11_0_rlc_smu_handshake_cntl(adev, false); 2340 2341 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 2342 udelay(50); 2343 } 2344 2345 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev) 2346 { 2347 uint32_t tmp; 2348 2349 /* enable Save Restore Machine */ 2350 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL)); 2351 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2352 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK; 2353 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp); 2354 } 2355 2356 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev) 2357 { 2358 const struct rlc_firmware_header_v2_0 *hdr; 2359 const __le32 *fw_data; 2360 unsigned i, fw_size; 2361 2362 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2363 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2364 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2365 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 2366 2367 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, 2368 RLCG_UCODE_LOADING_START_ADDRESS); 2369 2370 for (i = 0; i < fw_size; i++) 2371 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA, 2372 le32_to_cpup(fw_data++)); 2373 2374 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 2375 } 2376 2377 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev) 2378 { 2379 const struct rlc_firmware_header_v2_2 *hdr; 2380 const __le32 *fw_data; 2381 unsigned i, fw_size; 2382 u32 tmp; 2383 2384 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 2385 2386 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2387 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes)); 2388 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4; 2389 2390 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0); 2391 2392 for (i = 0; i < fw_size; i++) { 2393 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2394 msleep(1); 2395 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA, 2396 le32_to_cpup(fw_data++)); 2397 } 2398 2399 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2400 2401 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2402 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes)); 2403 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4; 2404 2405 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0); 2406 for (i = 0; i < fw_size; i++) { 2407 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2408 msleep(1); 2409 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA, 2410 le32_to_cpup(fw_data++)); 2411 } 2412 2413 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2414 2415 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL); 2416 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1); 2417 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0); 2418 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp); 2419 } 2420 2421 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev) 2422 { 2423 const struct rlc_firmware_header_v2_3 *hdr; 2424 const __le32 *fw_data; 2425 unsigned i, fw_size; 2426 u32 tmp; 2427 2428 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data; 2429 2430 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2431 le32_to_cpu(hdr->rlcp_ucode_offset_bytes)); 2432 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4; 2433 2434 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0); 2435 2436 for (i = 0; i < fw_size; i++) { 2437 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2438 msleep(1); 2439 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA, 2440 le32_to_cpup(fw_data++)); 2441 } 2442 2443 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version); 2444 2445 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE); 2446 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1); 2447 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp); 2448 2449 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2450 le32_to_cpu(hdr->rlcv_ucode_offset_bytes)); 2451 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4; 2452 2453 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0); 2454 2455 for (i = 0; i < fw_size; i++) { 2456 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2457 msleep(1); 2458 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA, 2459 le32_to_cpup(fw_data++)); 2460 } 2461 2462 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version); 2463 2464 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL); 2465 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1); 2466 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp); 2467 } 2468 2469 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev) 2470 { 2471 const struct rlc_firmware_header_v2_0 *hdr; 2472 uint16_t version_major; 2473 uint16_t version_minor; 2474 2475 if (!adev->gfx.rlc_fw) 2476 return -EINVAL; 2477 2478 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2479 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2480 2481 version_major = le16_to_cpu(hdr->header.header_version_major); 2482 version_minor = le16_to_cpu(hdr->header.header_version_minor); 2483 2484 if (version_major == 2) { 2485 gfx_v11_0_load_rlcg_microcode(adev); 2486 if (amdgpu_dpm == 1) { 2487 if (version_minor >= 2) 2488 gfx_v11_0_load_rlc_iram_dram_microcode(adev); 2489 if (version_minor == 3) 2490 gfx_v11_0_load_rlcp_rlcv_microcode(adev); 2491 } 2492 2493 return 0; 2494 } 2495 2496 return -EINVAL; 2497 } 2498 2499 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev) 2500 { 2501 int r; 2502 2503 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 2504 gfx_v11_0_init_csb(adev); 2505 2506 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */ 2507 gfx_v11_0_rlc_enable_srm(adev); 2508 } else { 2509 if (amdgpu_sriov_vf(adev)) { 2510 gfx_v11_0_init_csb(adev); 2511 return 0; 2512 } 2513 2514 adev->gfx.rlc.funcs->stop(adev); 2515 2516 /* disable CG */ 2517 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0); 2518 2519 /* disable PG */ 2520 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0); 2521 2522 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 2523 /* legacy rlc firmware loading */ 2524 r = gfx_v11_0_rlc_load_microcode(adev); 2525 if (r) 2526 return r; 2527 } 2528 2529 gfx_v11_0_init_csb(adev); 2530 2531 adev->gfx.rlc.funcs->start(adev); 2532 } 2533 return 0; 2534 } 2535 2536 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr) 2537 { 2538 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2539 uint32_t tmp; 2540 int i; 2541 2542 /* Trigger an invalidation of the L1 instruction caches */ 2543 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2544 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2545 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2546 2547 /* Wait for invalidation complete */ 2548 for (i = 0; i < usec_timeout; i++) { 2549 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2550 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2551 INVALIDATE_CACHE_COMPLETE)) 2552 break; 2553 udelay(1); 2554 } 2555 2556 if (i >= usec_timeout) { 2557 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2558 return -EINVAL; 2559 } 2560 2561 if (amdgpu_emu_mode == 1) 2562 amdgpu_device_flush_hdp(adev, NULL); 2563 2564 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2565 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2566 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2567 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2568 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2569 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2570 2571 /* Program me ucode address into intruction cache address register */ 2572 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2573 lower_32_bits(addr) & 0xFFFFF000); 2574 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2575 upper_32_bits(addr)); 2576 2577 return 0; 2578 } 2579 2580 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr) 2581 { 2582 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2583 uint32_t tmp; 2584 int i; 2585 2586 /* Trigger an invalidation of the L1 instruction caches */ 2587 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2588 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2589 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2590 2591 /* Wait for invalidation complete */ 2592 for (i = 0; i < usec_timeout; i++) { 2593 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2594 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2595 INVALIDATE_CACHE_COMPLETE)) 2596 break; 2597 udelay(1); 2598 } 2599 2600 if (i >= usec_timeout) { 2601 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2602 return -EINVAL; 2603 } 2604 2605 if (amdgpu_emu_mode == 1) 2606 amdgpu_device_flush_hdp(adev, NULL); 2607 2608 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2609 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2610 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2611 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2612 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2613 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2614 2615 /* Program pfp ucode address into intruction cache address register */ 2616 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2617 lower_32_bits(addr) & 0xFFFFF000); 2618 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2619 upper_32_bits(addr)); 2620 2621 return 0; 2622 } 2623 2624 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr) 2625 { 2626 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2627 uint32_t tmp; 2628 int i; 2629 2630 /* Trigger an invalidation of the L1 instruction caches */ 2631 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2632 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2633 2634 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2635 2636 /* Wait for invalidation complete */ 2637 for (i = 0; i < usec_timeout; i++) { 2638 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2639 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2640 INVALIDATE_CACHE_COMPLETE)) 2641 break; 2642 udelay(1); 2643 } 2644 2645 if (i >= usec_timeout) { 2646 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2647 return -EINVAL; 2648 } 2649 2650 if (amdgpu_emu_mode == 1) 2651 amdgpu_device_flush_hdp(adev, NULL); 2652 2653 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2654 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2655 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2656 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2657 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2658 2659 /* Program mec1 ucode address into intruction cache address register */ 2660 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, 2661 lower_32_bits(addr) & 0xFFFFF000); 2662 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2663 upper_32_bits(addr)); 2664 2665 return 0; 2666 } 2667 2668 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2669 { 2670 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2671 uint32_t tmp; 2672 unsigned i, pipe_id; 2673 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2674 2675 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 2676 adev->gfx.pfp_fw->data; 2677 2678 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2679 lower_32_bits(addr)); 2680 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2681 upper_32_bits(addr)); 2682 2683 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2684 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2685 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2686 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2687 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2688 2689 /* 2690 * Programming any of the CP_PFP_IC_BASE registers 2691 * forces invalidation of the ME L1 I$. Wait for the 2692 * invalidation complete 2693 */ 2694 for (i = 0; i < usec_timeout; i++) { 2695 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2696 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2697 INVALIDATE_CACHE_COMPLETE)) 2698 break; 2699 udelay(1); 2700 } 2701 2702 if (i >= usec_timeout) { 2703 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2704 return -EINVAL; 2705 } 2706 2707 /* Prime the L1 instruction caches */ 2708 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2709 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 2710 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2711 /* Waiting for cache primed*/ 2712 for (i = 0; i < usec_timeout; i++) { 2713 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2714 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2715 ICACHE_PRIMED)) 2716 break; 2717 udelay(1); 2718 } 2719 2720 if (i >= usec_timeout) { 2721 dev_err(adev->dev, "failed to prime instruction cache\n"); 2722 return -EINVAL; 2723 } 2724 2725 mutex_lock(&adev->srbm_mutex); 2726 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2727 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2728 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 2729 (pfp_hdr->ucode_start_addr_hi << 30) | 2730 (pfp_hdr->ucode_start_addr_lo >> 2)); 2731 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 2732 pfp_hdr->ucode_start_addr_hi >> 2); 2733 2734 /* 2735 * Program CP_ME_CNTL to reset given PIPE to take 2736 * effect of CP_PFP_PRGRM_CNTR_START. 2737 */ 2738 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2739 if (pipe_id == 0) 2740 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2741 PFP_PIPE0_RESET, 1); 2742 else 2743 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2744 PFP_PIPE1_RESET, 1); 2745 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2746 2747 /* Clear pfp pipe0 reset bit. */ 2748 if (pipe_id == 0) 2749 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2750 PFP_PIPE0_RESET, 0); 2751 else 2752 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2753 PFP_PIPE1_RESET, 0); 2754 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2755 2756 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 2757 lower_32_bits(addr2)); 2758 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 2759 upper_32_bits(addr2)); 2760 } 2761 soc21_grbm_select(adev, 0, 0, 0, 0); 2762 mutex_unlock(&adev->srbm_mutex); 2763 2764 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2765 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2766 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2767 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2768 2769 /* Invalidate the data caches */ 2770 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2771 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2772 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2773 2774 for (i = 0; i < usec_timeout; i++) { 2775 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2776 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2777 INVALIDATE_DCACHE_COMPLETE)) 2778 break; 2779 udelay(1); 2780 } 2781 2782 if (i >= usec_timeout) { 2783 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2784 return -EINVAL; 2785 } 2786 2787 return 0; 2788 } 2789 2790 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2791 { 2792 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2793 uint32_t tmp; 2794 unsigned i, pipe_id; 2795 const struct gfx_firmware_header_v2_0 *me_hdr; 2796 2797 me_hdr = (const struct gfx_firmware_header_v2_0 *) 2798 adev->gfx.me_fw->data; 2799 2800 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2801 lower_32_bits(addr)); 2802 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2803 upper_32_bits(addr)); 2804 2805 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2806 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2807 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2808 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2809 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2810 2811 /* 2812 * Programming any of the CP_ME_IC_BASE registers 2813 * forces invalidation of the ME L1 I$. Wait for the 2814 * invalidation complete 2815 */ 2816 for (i = 0; i < usec_timeout; i++) { 2817 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2818 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2819 INVALIDATE_CACHE_COMPLETE)) 2820 break; 2821 udelay(1); 2822 } 2823 2824 if (i >= usec_timeout) { 2825 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2826 return -EINVAL; 2827 } 2828 2829 /* Prime the instruction caches */ 2830 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2831 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 2832 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2833 2834 /* Waiting for instruction cache primed*/ 2835 for (i = 0; i < usec_timeout; i++) { 2836 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2837 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2838 ICACHE_PRIMED)) 2839 break; 2840 udelay(1); 2841 } 2842 2843 if (i >= usec_timeout) { 2844 dev_err(adev->dev, "failed to prime instruction cache\n"); 2845 return -EINVAL; 2846 } 2847 2848 mutex_lock(&adev->srbm_mutex); 2849 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2850 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2851 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 2852 (me_hdr->ucode_start_addr_hi << 30) | 2853 (me_hdr->ucode_start_addr_lo >> 2) ); 2854 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 2855 me_hdr->ucode_start_addr_hi>>2); 2856 2857 /* 2858 * Program CP_ME_CNTL to reset given PIPE to take 2859 * effect of CP_PFP_PRGRM_CNTR_START. 2860 */ 2861 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2862 if (pipe_id == 0) 2863 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2864 ME_PIPE0_RESET, 1); 2865 else 2866 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2867 ME_PIPE1_RESET, 1); 2868 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2869 2870 /* Clear pfp pipe0 reset bit. */ 2871 if (pipe_id == 0) 2872 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2873 ME_PIPE0_RESET, 0); 2874 else 2875 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2876 ME_PIPE1_RESET, 0); 2877 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2878 2879 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 2880 lower_32_bits(addr2)); 2881 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 2882 upper_32_bits(addr2)); 2883 } 2884 soc21_grbm_select(adev, 0, 0, 0, 0); 2885 mutex_unlock(&adev->srbm_mutex); 2886 2887 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2888 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2889 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2890 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2891 2892 /* Invalidate the data caches */ 2893 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2894 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2895 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2896 2897 for (i = 0; i < usec_timeout; i++) { 2898 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2899 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2900 INVALIDATE_DCACHE_COMPLETE)) 2901 break; 2902 udelay(1); 2903 } 2904 2905 if (i >= usec_timeout) { 2906 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2907 return -EINVAL; 2908 } 2909 2910 return 0; 2911 } 2912 2913 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2914 { 2915 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2916 uint32_t tmp; 2917 unsigned i; 2918 const struct gfx_firmware_header_v2_0 *mec_hdr; 2919 2920 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2921 adev->gfx.mec_fw->data; 2922 2923 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2924 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 2925 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2926 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2927 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2928 2929 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 2930 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 2931 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 2932 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 2933 2934 mutex_lock(&adev->srbm_mutex); 2935 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 2936 soc21_grbm_select(adev, 1, i, 0, 0); 2937 2938 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2); 2939 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 2940 upper_32_bits(addr2)); 2941 2942 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 2943 mec_hdr->ucode_start_addr_lo >> 2 | 2944 mec_hdr->ucode_start_addr_hi << 30); 2945 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2946 mec_hdr->ucode_start_addr_hi >> 2); 2947 2948 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr); 2949 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2950 upper_32_bits(addr)); 2951 } 2952 mutex_unlock(&adev->srbm_mutex); 2953 soc21_grbm_select(adev, 0, 0, 0, 0); 2954 2955 /* Trigger an invalidation of the L1 instruction caches */ 2956 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2957 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2958 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 2959 2960 /* Wait for invalidation complete */ 2961 for (i = 0; i < usec_timeout; i++) { 2962 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2963 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 2964 INVALIDATE_DCACHE_COMPLETE)) 2965 break; 2966 udelay(1); 2967 } 2968 2969 if (i >= usec_timeout) { 2970 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2971 return -EINVAL; 2972 } 2973 2974 /* Trigger an invalidation of the L1 instruction caches */ 2975 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2976 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2977 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2978 2979 /* Wait for invalidation complete */ 2980 for (i = 0; i < usec_timeout; i++) { 2981 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2982 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2983 INVALIDATE_CACHE_COMPLETE)) 2984 break; 2985 udelay(1); 2986 } 2987 2988 if (i >= usec_timeout) { 2989 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2990 return -EINVAL; 2991 } 2992 2993 return 0; 2994 } 2995 2996 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev) 2997 { 2998 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2999 const struct gfx_firmware_header_v2_0 *me_hdr; 3000 const struct gfx_firmware_header_v2_0 *mec_hdr; 3001 uint32_t pipe_id, tmp; 3002 3003 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 3004 adev->gfx.mec_fw->data; 3005 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3006 adev->gfx.me_fw->data; 3007 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3008 adev->gfx.pfp_fw->data; 3009 3010 /* config pfp program start addr */ 3011 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3012 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3013 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3014 (pfp_hdr->ucode_start_addr_hi << 30) | 3015 (pfp_hdr->ucode_start_addr_lo >> 2)); 3016 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3017 pfp_hdr->ucode_start_addr_hi >> 2); 3018 } 3019 soc21_grbm_select(adev, 0, 0, 0, 0); 3020 3021 /* reset pfp pipe */ 3022 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3023 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1); 3024 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1); 3025 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3026 3027 /* clear pfp pipe reset */ 3028 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0); 3029 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0); 3030 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3031 3032 /* config me program start addr */ 3033 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3034 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3035 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3036 (me_hdr->ucode_start_addr_hi << 30) | 3037 (me_hdr->ucode_start_addr_lo >> 2) ); 3038 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3039 me_hdr->ucode_start_addr_hi>>2); 3040 } 3041 soc21_grbm_select(adev, 0, 0, 0, 0); 3042 3043 /* reset me pipe */ 3044 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3045 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1); 3046 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1); 3047 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3048 3049 /* clear me pipe reset */ 3050 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0); 3051 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0); 3052 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3053 3054 /* config mec program start addr */ 3055 for (pipe_id = 0; pipe_id < 4; pipe_id++) { 3056 soc21_grbm_select(adev, 1, pipe_id, 0, 0); 3057 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 3058 mec_hdr->ucode_start_addr_lo >> 2 | 3059 mec_hdr->ucode_start_addr_hi << 30); 3060 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 3061 mec_hdr->ucode_start_addr_hi >> 2); 3062 } 3063 soc21_grbm_select(adev, 0, 0, 0, 0); 3064 3065 /* reset mec pipe */ 3066 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3067 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1); 3068 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1); 3069 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1); 3070 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1); 3071 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3072 3073 /* clear mec pipe reset */ 3074 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0); 3075 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0); 3076 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0); 3077 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0); 3078 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3079 } 3080 3081 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) 3082 { 3083 uint32_t cp_status; 3084 uint32_t bootload_status; 3085 int i, r; 3086 uint64_t addr, addr2; 3087 3088 for (i = 0; i < adev->usec_timeout; i++) { 3089 cp_status = RREG32_SOC15(GC, 0, regCP_STAT); 3090 3091 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 3092 IP_VERSION(11, 0, 1) || 3093 amdgpu_ip_version(adev, GC_HWIP, 0) == 3094 IP_VERSION(11, 0, 4) || 3095 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) || 3096 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) || 3097 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) || 3098 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3) || 3099 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4) || 3100 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6) || 3101 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 0) || 3102 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 1)) 3103 bootload_status = RREG32_SOC15(GC, 0, 3104 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1); 3105 else 3106 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS); 3107 3108 if ((cp_status == 0) && 3109 (REG_GET_FIELD(bootload_status, 3110 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) { 3111 break; 3112 } 3113 udelay(1); 3114 } 3115 3116 if (i >= adev->usec_timeout) { 3117 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n"); 3118 return -ETIMEDOUT; 3119 } 3120 3121 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 3122 if (adev->gfx.rs64_enable) { 3123 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3124 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset; 3125 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3126 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset; 3127 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2); 3128 if (r) 3129 return r; 3130 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3131 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset; 3132 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3133 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset; 3134 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2); 3135 if (r) 3136 return r; 3137 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3138 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset; 3139 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3140 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset; 3141 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2); 3142 if (r) 3143 return r; 3144 } else { 3145 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3146 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset; 3147 r = gfx_v11_0_config_me_cache(adev, addr); 3148 if (r) 3149 return r; 3150 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3151 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset; 3152 r = gfx_v11_0_config_pfp_cache(adev, addr); 3153 if (r) 3154 return r; 3155 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3156 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset; 3157 r = gfx_v11_0_config_mec_cache(adev, addr); 3158 if (r) 3159 return r; 3160 } 3161 } 3162 3163 return 0; 3164 } 3165 3166 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3167 { 3168 int i; 3169 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3170 3171 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3172 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3173 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3174 3175 for (i = 0; i < adev->usec_timeout; i++) { 3176 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0) 3177 break; 3178 udelay(1); 3179 } 3180 3181 if (i >= adev->usec_timeout) 3182 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt"); 3183 3184 return 0; 3185 } 3186 3187 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev) 3188 { 3189 int r; 3190 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3191 const __le32 *fw_data; 3192 unsigned i, fw_size; 3193 3194 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3195 adev->gfx.pfp_fw->data; 3196 3197 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3198 3199 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3200 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3201 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes); 3202 3203 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes, 3204 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3205 &adev->gfx.pfp.pfp_fw_obj, 3206 &adev->gfx.pfp.pfp_fw_gpu_addr, 3207 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3208 if (r) { 3209 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r); 3210 gfx_v11_0_pfp_fini(adev); 3211 return r; 3212 } 3213 3214 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size); 3215 3216 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3217 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3218 3219 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr); 3220 3221 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0); 3222 3223 for (i = 0; i < pfp_hdr->jt_size; i++) 3224 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA, 3225 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i)); 3226 3227 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3228 3229 return 0; 3230 } 3231 3232 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) 3233 { 3234 int r; 3235 const struct gfx_firmware_header_v2_0 *pfp_hdr; 3236 const __le32 *fw_ucode, *fw_data; 3237 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3238 uint32_t tmp; 3239 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3240 3241 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3242 adev->gfx.pfp_fw->data; 3243 3244 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3245 3246 /* instruction */ 3247 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data + 3248 le32_to_cpu(pfp_hdr->ucode_offset_bytes)); 3249 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes); 3250 /* data */ 3251 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3252 le32_to_cpu(pfp_hdr->data_offset_bytes)); 3253 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes); 3254 3255 /* 64kb align */ 3256 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3257 64 * 1024, 3258 AMDGPU_GEM_DOMAIN_VRAM | 3259 AMDGPU_GEM_DOMAIN_GTT, 3260 &adev->gfx.pfp.pfp_fw_obj, 3261 &adev->gfx.pfp.pfp_fw_gpu_addr, 3262 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3263 if (r) { 3264 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r); 3265 gfx_v11_0_pfp_fini(adev); 3266 return r; 3267 } 3268 3269 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3270 64 * 1024, 3271 AMDGPU_GEM_DOMAIN_VRAM | 3272 AMDGPU_GEM_DOMAIN_GTT, 3273 &adev->gfx.pfp.pfp_fw_data_obj, 3274 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 3275 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 3276 if (r) { 3277 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r); 3278 gfx_v11_0_pfp_fini(adev); 3279 return r; 3280 } 3281 3282 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size); 3283 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); 3284 3285 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3286 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj); 3287 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3288 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj); 3289 3290 if (amdgpu_emu_mode == 1) 3291 amdgpu_device_flush_hdp(adev, NULL); 3292 3293 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 3294 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3295 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 3296 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3297 3298 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 3299 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 3300 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 3301 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 3302 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 3303 3304 /* 3305 * Programming any of the CP_PFP_IC_BASE registers 3306 * forces invalidation of the ME L1 I$. Wait for the 3307 * invalidation complete 3308 */ 3309 for (i = 0; i < usec_timeout; i++) { 3310 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3311 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3312 INVALIDATE_CACHE_COMPLETE)) 3313 break; 3314 udelay(1); 3315 } 3316 3317 if (i >= usec_timeout) { 3318 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3319 return -EINVAL; 3320 } 3321 3322 /* Prime the L1 instruction caches */ 3323 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3324 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 3325 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 3326 /* Waiting for cache primed*/ 3327 for (i = 0; i < usec_timeout; i++) { 3328 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3329 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3330 ICACHE_PRIMED)) 3331 break; 3332 udelay(1); 3333 } 3334 3335 if (i >= usec_timeout) { 3336 dev_err(adev->dev, "failed to prime instruction cache\n"); 3337 return -EINVAL; 3338 } 3339 3340 mutex_lock(&adev->srbm_mutex); 3341 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3342 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3343 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3344 (pfp_hdr->ucode_start_addr_hi << 30) | 3345 (pfp_hdr->ucode_start_addr_lo >> 2) ); 3346 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3347 pfp_hdr->ucode_start_addr_hi>>2); 3348 3349 /* 3350 * Program CP_ME_CNTL to reset given PIPE to take 3351 * effect of CP_PFP_PRGRM_CNTR_START. 3352 */ 3353 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3354 if (pipe_id == 0) 3355 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3356 PFP_PIPE0_RESET, 1); 3357 else 3358 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3359 PFP_PIPE1_RESET, 1); 3360 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3361 3362 /* Clear pfp pipe0 reset bit. */ 3363 if (pipe_id == 0) 3364 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3365 PFP_PIPE0_RESET, 0); 3366 else 3367 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3368 PFP_PIPE1_RESET, 0); 3369 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3370 3371 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 3372 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3373 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 3374 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3375 } 3376 soc21_grbm_select(adev, 0, 0, 0, 0); 3377 mutex_unlock(&adev->srbm_mutex); 3378 3379 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3380 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3381 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3382 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3383 3384 /* Invalidate the data caches */ 3385 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3386 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3387 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3388 3389 for (i = 0; i < usec_timeout; i++) { 3390 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3391 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3392 INVALIDATE_DCACHE_COMPLETE)) 3393 break; 3394 udelay(1); 3395 } 3396 3397 if (i >= usec_timeout) { 3398 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3399 return -EINVAL; 3400 } 3401 3402 return 0; 3403 } 3404 3405 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev) 3406 { 3407 int r; 3408 const struct gfx_firmware_header_v1_0 *me_hdr; 3409 const __le32 *fw_data; 3410 unsigned i, fw_size; 3411 3412 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3413 adev->gfx.me_fw->data; 3414 3415 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3416 3417 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3418 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3419 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes); 3420 3421 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes, 3422 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3423 &adev->gfx.me.me_fw_obj, 3424 &adev->gfx.me.me_fw_gpu_addr, 3425 (void **)&adev->gfx.me.me_fw_ptr); 3426 if (r) { 3427 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r); 3428 gfx_v11_0_me_fini(adev); 3429 return r; 3430 } 3431 3432 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size); 3433 3434 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3435 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3436 3437 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr); 3438 3439 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0); 3440 3441 for (i = 0; i < me_hdr->jt_size; i++) 3442 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA, 3443 le32_to_cpup(fw_data + me_hdr->jt_offset + i)); 3444 3445 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version); 3446 3447 return 0; 3448 } 3449 3450 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) 3451 { 3452 int r; 3453 const struct gfx_firmware_header_v2_0 *me_hdr; 3454 const __le32 *fw_ucode, *fw_data; 3455 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3456 uint32_t tmp; 3457 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3458 3459 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3460 adev->gfx.me_fw->data; 3461 3462 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3463 3464 /* instruction */ 3465 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data + 3466 le32_to_cpu(me_hdr->ucode_offset_bytes)); 3467 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes); 3468 /* data */ 3469 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3470 le32_to_cpu(me_hdr->data_offset_bytes)); 3471 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes); 3472 3473 /* 64kb align*/ 3474 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3475 64 * 1024, 3476 AMDGPU_GEM_DOMAIN_VRAM | 3477 AMDGPU_GEM_DOMAIN_GTT, 3478 &adev->gfx.me.me_fw_obj, 3479 &adev->gfx.me.me_fw_gpu_addr, 3480 (void **)&adev->gfx.me.me_fw_ptr); 3481 if (r) { 3482 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r); 3483 gfx_v11_0_me_fini(adev); 3484 return r; 3485 } 3486 3487 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3488 64 * 1024, 3489 AMDGPU_GEM_DOMAIN_VRAM | 3490 AMDGPU_GEM_DOMAIN_GTT, 3491 &adev->gfx.me.me_fw_data_obj, 3492 &adev->gfx.me.me_fw_data_gpu_addr, 3493 (void **)&adev->gfx.me.me_fw_data_ptr); 3494 if (r) { 3495 dev_err(adev->dev, "(%d) failed to create me data bo\n", r); 3496 gfx_v11_0_pfp_fini(adev); 3497 return r; 3498 } 3499 3500 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size); 3501 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); 3502 3503 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3504 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj); 3505 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3506 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj); 3507 3508 if (amdgpu_emu_mode == 1) 3509 amdgpu_device_flush_hdp(adev, NULL); 3510 3511 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 3512 lower_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3513 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 3514 upper_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3515 3516 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 3517 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 3518 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 3519 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 3520 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 3521 3522 /* 3523 * Programming any of the CP_ME_IC_BASE registers 3524 * forces invalidation of the ME L1 I$. Wait for the 3525 * invalidation complete 3526 */ 3527 for (i = 0; i < usec_timeout; i++) { 3528 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3529 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3530 INVALIDATE_CACHE_COMPLETE)) 3531 break; 3532 udelay(1); 3533 } 3534 3535 if (i >= usec_timeout) { 3536 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3537 return -EINVAL; 3538 } 3539 3540 /* Prime the instruction caches */ 3541 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3542 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 3543 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 3544 3545 /* Waiting for instruction cache primed*/ 3546 for (i = 0; i < usec_timeout; i++) { 3547 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3548 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3549 ICACHE_PRIMED)) 3550 break; 3551 udelay(1); 3552 } 3553 3554 if (i >= usec_timeout) { 3555 dev_err(adev->dev, "failed to prime instruction cache\n"); 3556 return -EINVAL; 3557 } 3558 3559 mutex_lock(&adev->srbm_mutex); 3560 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3561 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3562 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3563 (me_hdr->ucode_start_addr_hi << 30) | 3564 (me_hdr->ucode_start_addr_lo >> 2) ); 3565 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3566 me_hdr->ucode_start_addr_hi>>2); 3567 3568 /* 3569 * Program CP_ME_CNTL to reset given PIPE to take 3570 * effect of CP_PFP_PRGRM_CNTR_START. 3571 */ 3572 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3573 if (pipe_id == 0) 3574 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3575 ME_PIPE0_RESET, 1); 3576 else 3577 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3578 ME_PIPE1_RESET, 1); 3579 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3580 3581 /* Clear pfp pipe0 reset bit. */ 3582 if (pipe_id == 0) 3583 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3584 ME_PIPE0_RESET, 0); 3585 else 3586 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3587 ME_PIPE1_RESET, 0); 3588 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3589 3590 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 3591 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3592 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 3593 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3594 } 3595 soc21_grbm_select(adev, 0, 0, 0, 0); 3596 mutex_unlock(&adev->srbm_mutex); 3597 3598 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3599 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3600 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3601 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3602 3603 /* Invalidate the data caches */ 3604 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3605 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3606 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3607 3608 for (i = 0; i < usec_timeout; i++) { 3609 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3610 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3611 INVALIDATE_DCACHE_COMPLETE)) 3612 break; 3613 udelay(1); 3614 } 3615 3616 if (i >= usec_timeout) { 3617 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3618 return -EINVAL; 3619 } 3620 3621 return 0; 3622 } 3623 3624 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3625 { 3626 int r; 3627 3628 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw) 3629 return -EINVAL; 3630 3631 gfx_v11_0_cp_gfx_enable(adev, false); 3632 3633 if (adev->gfx.rs64_enable) 3634 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev); 3635 else 3636 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev); 3637 if (r) { 3638 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r); 3639 return r; 3640 } 3641 3642 if (adev->gfx.rs64_enable) 3643 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev); 3644 else 3645 r = gfx_v11_0_cp_gfx_load_me_microcode(adev); 3646 if (r) { 3647 dev_err(adev->dev, "(%d) failed to load me fw\n", r); 3648 return r; 3649 } 3650 3651 return 0; 3652 } 3653 3654 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev) 3655 { 3656 struct amdgpu_ring *ring; 3657 const struct cs_section_def *sect = NULL; 3658 const struct cs_extent_def *ext = NULL; 3659 int r, i; 3660 int ctx_reg_offset; 3661 3662 /* init the CP */ 3663 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT, 3664 adev->gfx.config.max_hw_contexts - 1); 3665 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1); 3666 3667 if (!amdgpu_async_gfx_ring) 3668 gfx_v11_0_cp_gfx_enable(adev, true); 3669 3670 ring = &adev->gfx.gfx_ring[0]; 3671 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev)); 3672 if (r) { 3673 drm_err(&adev->ddev, "cp failed to lock ring (%d).\n", r); 3674 return r; 3675 } 3676 3677 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3678 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3679 3680 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3681 amdgpu_ring_write(ring, 0x80000000); 3682 amdgpu_ring_write(ring, 0x80000000); 3683 3684 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 3685 for (ext = sect->section; ext->extent != NULL; ++ext) { 3686 if (sect->id == SECT_CONTEXT) { 3687 amdgpu_ring_write(ring, 3688 PACKET3(PACKET3_SET_CONTEXT_REG, 3689 ext->reg_count)); 3690 amdgpu_ring_write(ring, ext->reg_index - 3691 PACKET3_SET_CONTEXT_REG_START); 3692 for (i = 0; i < ext->reg_count; i++) 3693 amdgpu_ring_write(ring, ext->extent[i]); 3694 } 3695 } 3696 } 3697 3698 ctx_reg_offset = 3699 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 3700 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 3701 amdgpu_ring_write(ring, ctx_reg_offset); 3702 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override); 3703 3704 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3705 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3706 3707 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3708 amdgpu_ring_write(ring, 0); 3709 3710 amdgpu_ring_commit(ring); 3711 3712 /* submit cs packet to copy state 0 to next available state */ 3713 if (adev->gfx.num_gfx_rings > 1) { 3714 /* maximum supported gfx ring is 2 */ 3715 ring = &adev->gfx.gfx_ring[1]; 3716 r = amdgpu_ring_alloc(ring, 2); 3717 if (r) { 3718 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 3719 return r; 3720 } 3721 3722 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3723 amdgpu_ring_write(ring, 0); 3724 3725 amdgpu_ring_commit(ring); 3726 } 3727 return 0; 3728 } 3729 3730 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev, 3731 CP_PIPE_ID pipe) 3732 { 3733 u32 tmp; 3734 3735 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); 3736 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe); 3737 3738 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); 3739 } 3740 3741 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev, 3742 struct amdgpu_ring *ring) 3743 { 3744 u32 tmp; 3745 3746 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); 3747 if (ring->use_doorbell) { 3748 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3749 DOORBELL_OFFSET, ring->doorbell_index); 3750 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3751 DOORBELL_EN, 1); 3752 } else { 3753 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3754 DOORBELL_EN, 0); 3755 } 3756 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp); 3757 3758 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3759 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3760 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp); 3761 3762 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 3763 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3764 } 3765 3766 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev) 3767 { 3768 struct amdgpu_ring *ring; 3769 u32 tmp; 3770 u32 rb_bufsz; 3771 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3772 3773 /* Set the write pointer delay */ 3774 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0); 3775 3776 /* set the RB to use vmid 0 */ 3777 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0); 3778 3779 /* Init gfx ring 0 for pipe 0 */ 3780 mutex_lock(&adev->srbm_mutex); 3781 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3782 3783 /* Set ring buffer size */ 3784 ring = &adev->gfx.gfx_ring[0]; 3785 rb_bufsz = order_base_2(ring->ring_size / 8); 3786 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3787 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3788 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3789 3790 /* Initialize the ring buffer's write pointers */ 3791 ring->wptr = 0; 3792 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3793 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3794 3795 /* set the wb address whether it's enabled or not */ 3796 rptr_addr = ring->rptr_gpu_addr; 3797 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3798 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3799 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3800 3801 wptr_gpu_addr = ring->wptr_gpu_addr; 3802 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3803 lower_32_bits(wptr_gpu_addr)); 3804 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3805 upper_32_bits(wptr_gpu_addr)); 3806 3807 mdelay(1); 3808 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3809 3810 rb_addr = ring->gpu_addr >> 8; 3811 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr); 3812 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3813 3814 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1); 3815 3816 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3817 mutex_unlock(&adev->srbm_mutex); 3818 3819 /* Init gfx ring 1 for pipe 1 */ 3820 if (adev->gfx.num_gfx_rings > 1) { 3821 mutex_lock(&adev->srbm_mutex); 3822 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1); 3823 /* maximum supported gfx ring is 2 */ 3824 ring = &adev->gfx.gfx_ring[1]; 3825 rb_bufsz = order_base_2(ring->ring_size / 8); 3826 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz); 3827 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2); 3828 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3829 /* Initialize the ring buffer's write pointers */ 3830 ring->wptr = 0; 3831 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr)); 3832 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr)); 3833 /* Set the wb address whether it's enabled or not */ 3834 rptr_addr = ring->rptr_gpu_addr; 3835 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr)); 3836 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3837 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3838 wptr_gpu_addr = ring->wptr_gpu_addr; 3839 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3840 lower_32_bits(wptr_gpu_addr)); 3841 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3842 upper_32_bits(wptr_gpu_addr)); 3843 3844 mdelay(1); 3845 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3846 3847 rb_addr = ring->gpu_addr >> 8; 3848 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr); 3849 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr)); 3850 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1); 3851 3852 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3853 mutex_unlock(&adev->srbm_mutex); 3854 } 3855 /* Switch to pipe 0 */ 3856 mutex_lock(&adev->srbm_mutex); 3857 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3858 mutex_unlock(&adev->srbm_mutex); 3859 3860 /* start the ring */ 3861 gfx_v11_0_cp_gfx_start(adev); 3862 3863 return 0; 3864 } 3865 3866 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3867 { 3868 u32 data; 3869 3870 if (adev->gfx.rs64_enable) { 3871 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3872 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE, 3873 enable ? 0 : 1); 3874 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 3875 enable ? 0 : 1); 3876 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 3877 enable ? 0 : 1); 3878 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 3879 enable ? 0 : 1); 3880 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 3881 enable ? 0 : 1); 3882 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE, 3883 enable ? 1 : 0); 3884 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE, 3885 enable ? 1 : 0); 3886 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE, 3887 enable ? 1 : 0); 3888 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE, 3889 enable ? 1 : 0); 3890 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT, 3891 enable ? 0 : 1); 3892 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data); 3893 } else { 3894 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); 3895 3896 if (enable) { 3897 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0); 3898 if (!adev->enable_mes_kiq) 3899 data = REG_SET_FIELD(data, CP_MEC_CNTL, 3900 MEC_ME2_HALT, 0); 3901 } else { 3902 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1); 3903 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1); 3904 } 3905 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data); 3906 } 3907 3908 udelay(50); 3909 } 3910 3911 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3912 { 3913 const struct gfx_firmware_header_v1_0 *mec_hdr; 3914 const __le32 *fw_data; 3915 unsigned i, fw_size; 3916 u32 *fw = NULL; 3917 int r; 3918 3919 if (!adev->gfx.mec_fw) 3920 return -EINVAL; 3921 3922 gfx_v11_0_cp_compute_enable(adev, false); 3923 3924 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3925 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3926 3927 fw_data = (const __le32 *) 3928 (adev->gfx.mec_fw->data + 3929 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3930 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 3931 3932 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 3933 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3934 &adev->gfx.mec.mec_fw_obj, 3935 &adev->gfx.mec.mec_fw_gpu_addr, 3936 (void **)&fw); 3937 if (r) { 3938 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r); 3939 gfx_v11_0_mec_fini(adev); 3940 return r; 3941 } 3942 3943 memcpy(fw, fw_data, fw_size); 3944 3945 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 3946 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 3947 3948 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr); 3949 3950 /* MEC1 */ 3951 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0); 3952 3953 for (i = 0; i < mec_hdr->jt_size; i++) 3954 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA, 3955 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3956 3957 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version); 3958 3959 return 0; 3960 } 3961 3962 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) 3963 { 3964 const struct gfx_firmware_header_v2_0 *mec_hdr; 3965 const __le32 *fw_ucode, *fw_data; 3966 u32 tmp, fw_ucode_size, fw_data_size; 3967 u32 i, usec_timeout = 50000; /* Wait for 50 ms */ 3968 u32 *fw_ucode_ptr, *fw_data_ptr; 3969 int r; 3970 3971 if (!adev->gfx.mec_fw) 3972 return -EINVAL; 3973 3974 gfx_v11_0_cp_compute_enable(adev, false); 3975 3976 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; 3977 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3978 3979 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data + 3980 le32_to_cpu(mec_hdr->ucode_offset_bytes)); 3981 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes); 3982 3983 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 3984 le32_to_cpu(mec_hdr->data_offset_bytes)); 3985 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes); 3986 3987 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3988 64 * 1024, 3989 AMDGPU_GEM_DOMAIN_VRAM | 3990 AMDGPU_GEM_DOMAIN_GTT, 3991 &adev->gfx.mec.mec_fw_obj, 3992 &adev->gfx.mec.mec_fw_gpu_addr, 3993 (void **)&fw_ucode_ptr); 3994 if (r) { 3995 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 3996 gfx_v11_0_mec_fini(adev); 3997 return r; 3998 } 3999 4000 r = amdgpu_bo_create_reserved(adev, fw_data_size, 4001 64 * 1024, 4002 AMDGPU_GEM_DOMAIN_VRAM | 4003 AMDGPU_GEM_DOMAIN_GTT, 4004 &adev->gfx.mec.mec_fw_data_obj, 4005 &adev->gfx.mec.mec_fw_data_gpu_addr, 4006 (void **)&fw_data_ptr); 4007 if (r) { 4008 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 4009 gfx_v11_0_mec_fini(adev); 4010 return r; 4011 } 4012 4013 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); 4014 memcpy(fw_data_ptr, fw_data, fw_data_size); 4015 4016 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 4017 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj); 4018 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 4019 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj); 4020 4021 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 4022 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 4023 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 4024 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 4025 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 4026 4027 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 4028 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 4029 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 4030 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 4031 4032 mutex_lock(&adev->srbm_mutex); 4033 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 4034 soc21_grbm_select(adev, 1, i, 0, 0); 4035 4036 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr); 4037 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 4038 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr)); 4039 4040 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 4041 mec_hdr->ucode_start_addr_lo >> 2 | 4042 mec_hdr->ucode_start_addr_hi << 30); 4043 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 4044 mec_hdr->ucode_start_addr_hi >> 2); 4045 4046 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr); 4047 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 4048 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 4049 } 4050 mutex_unlock(&adev->srbm_mutex); 4051 soc21_grbm_select(adev, 0, 0, 0, 0); 4052 4053 /* Trigger an invalidation of the L1 instruction caches */ 4054 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4055 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 4056 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 4057 4058 /* Wait for invalidation complete */ 4059 for (i = 0; i < usec_timeout; i++) { 4060 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4061 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 4062 INVALIDATE_DCACHE_COMPLETE)) 4063 break; 4064 udelay(1); 4065 } 4066 4067 if (i >= usec_timeout) { 4068 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4069 return -EINVAL; 4070 } 4071 4072 /* Trigger an invalidation of the L1 instruction caches */ 4073 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4074 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 4075 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 4076 4077 /* Wait for invalidation complete */ 4078 for (i = 0; i < usec_timeout; i++) { 4079 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4080 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 4081 INVALIDATE_CACHE_COMPLETE)) 4082 break; 4083 udelay(1); 4084 } 4085 4086 if (i >= usec_timeout) { 4087 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4088 return -EINVAL; 4089 } 4090 4091 return 0; 4092 } 4093 4094 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring) 4095 { 4096 uint32_t tmp; 4097 struct amdgpu_device *adev = ring->adev; 4098 4099 /* tell RLC which is KIQ queue */ 4100 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); 4101 tmp &= 0xffffff00; 4102 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4103 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80); 4104 } 4105 4106 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev) 4107 { 4108 /* set graphics engine doorbell range */ 4109 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, 4110 (adev->doorbell_index.gfx_ring0 * 2) << 2); 4111 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 4112 (adev->doorbell_index.gfx_userqueue_end * 2) << 2); 4113 4114 /* set compute engine doorbell range */ 4115 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4116 (adev->doorbell_index.kiq * 2) << 2); 4117 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4118 (adev->doorbell_index.userqueue_end * 2) << 2); 4119 } 4120 4121 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev, 4122 struct v11_gfx_mqd *mqd, 4123 struct amdgpu_mqd_prop *prop) 4124 { 4125 bool priority = 0; 4126 u32 tmp; 4127 4128 /* set up default queue priority level 4129 * 0x0 = low priority, 0x1 = high priority 4130 */ 4131 if (prop->hqd_queue_priority == AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM) 4132 priority = 1; 4133 4134 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT; 4135 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority); 4136 mqd->cp_gfx_hqd_queue_priority = tmp; 4137 } 4138 4139 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, 4140 struct amdgpu_mqd_prop *prop) 4141 { 4142 struct v11_gfx_mqd *mqd = m; 4143 uint64_t hqd_gpu_addr, wb_gpu_addr; 4144 uint32_t tmp; 4145 uint32_t rb_bufsz; 4146 4147 /* set up gfx hqd wptr */ 4148 mqd->cp_gfx_hqd_wptr = 0; 4149 mqd->cp_gfx_hqd_wptr_hi = 0; 4150 4151 /* set the pointer to the MQD */ 4152 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc; 4153 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4154 4155 /* set up mqd control */ 4156 tmp = regCP_GFX_MQD_CONTROL_DEFAULT; 4157 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0); 4158 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1); 4159 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0); 4160 mqd->cp_gfx_mqd_control = tmp; 4161 4162 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */ 4163 tmp = regCP_GFX_HQD_VMID_DEFAULT; 4164 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0); 4165 mqd->cp_gfx_hqd_vmid = 0; 4166 4167 /* set up gfx queue priority */ 4168 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop); 4169 4170 /* set up time quantum */ 4171 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT; 4172 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1); 4173 mqd->cp_gfx_hqd_quantum = tmp; 4174 4175 /* set up gfx hqd base. this is similar as CP_RB_BASE */ 4176 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4177 mqd->cp_gfx_hqd_base = hqd_gpu_addr; 4178 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr); 4179 4180 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */ 4181 wb_gpu_addr = prop->rptr_gpu_addr; 4182 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc; 4183 mqd->cp_gfx_hqd_rptr_addr_hi = 4184 upper_32_bits(wb_gpu_addr) & 0xffff; 4185 4186 /* set up rb_wptr_poll addr */ 4187 wb_gpu_addr = prop->wptr_gpu_addr; 4188 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4189 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4190 4191 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */ 4192 rb_bufsz = order_base_2(prop->queue_size / 4) - 1; 4193 tmp = regCP_GFX_HQD_CNTL_DEFAULT; 4194 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz); 4195 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2); 4196 #ifdef __BIG_ENDIAN 4197 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1); 4198 #endif 4199 if (prop->tmz_queue) 4200 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, TMZ_MATCH, 1); 4201 if (!prop->kernel_queue) 4202 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_NON_PRIV, 1); 4203 mqd->cp_gfx_hqd_cntl = tmp; 4204 4205 /* set up cp_doorbell_control */ 4206 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT; 4207 if (prop->use_doorbell) { 4208 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4209 DOORBELL_OFFSET, prop->doorbell_index); 4210 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4211 DOORBELL_EN, 1); 4212 } else 4213 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4214 DOORBELL_EN, 0); 4215 mqd->cp_rb_doorbell_control = tmp; 4216 4217 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4218 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT; 4219 4220 /* active the queue */ 4221 mqd->cp_gfx_hqd_active = 1; 4222 4223 /* set gfx UQ items */ 4224 mqd->shadow_base_lo = lower_32_bits(prop->shadow_addr); 4225 mqd->shadow_base_hi = upper_32_bits(prop->shadow_addr); 4226 mqd->gds_bkup_base_lo = lower_32_bits(prop->gds_bkup_addr); 4227 mqd->gds_bkup_base_hi = upper_32_bits(prop->gds_bkup_addr); 4228 mqd->fw_work_area_base_lo = lower_32_bits(prop->csa_addr); 4229 mqd->fw_work_area_base_hi = upper_32_bits(prop->csa_addr); 4230 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4231 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4232 4233 return 0; 4234 } 4235 4236 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset) 4237 { 4238 struct amdgpu_device *adev = ring->adev; 4239 struct v11_gfx_mqd *mqd = ring->mqd_ptr; 4240 int mqd_idx = ring - &adev->gfx.gfx_ring[0]; 4241 4242 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { 4243 memset((void *)mqd, 0, sizeof(*mqd)); 4244 mutex_lock(&adev->srbm_mutex); 4245 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4246 amdgpu_ring_init_mqd(ring); 4247 soc21_grbm_select(adev, 0, 0, 0, 0); 4248 mutex_unlock(&adev->srbm_mutex); 4249 if (adev->gfx.me.mqd_backup[mqd_idx]) 4250 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4251 } else { 4252 /* restore mqd with the backup copy */ 4253 if (adev->gfx.me.mqd_backup[mqd_idx]) 4254 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); 4255 /* reset the ring */ 4256 ring->wptr = 0; 4257 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4258 amdgpu_ring_clear_ring(ring); 4259 } 4260 4261 return 0; 4262 } 4263 4264 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) 4265 { 4266 int r, i; 4267 4268 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4269 r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i], false); 4270 if (r) 4271 return r; 4272 } 4273 4274 r = amdgpu_gfx_enable_kgq(adev, 0); 4275 if (r) 4276 return r; 4277 4278 return gfx_v11_0_cp_gfx_start(adev); 4279 } 4280 4281 static void gfx_v11_0_compute_mqd_set_cu_mask(struct amdgpu_device *adev, 4282 struct v11_compute_mqd *mqd, 4283 struct amdgpu_mqd_prop *prop) 4284 { 4285 uint32_t se_mask[8] = {0}; 4286 uint32_t wa_mask; 4287 bool has_wa_flag = prop->cu_flags & (AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE | 4288 AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE); 4289 4290 if (!has_wa_flag && (!prop->cu_mask || !prop->cu_mask_count)) 4291 return; 4292 4293 if (has_wa_flag) { 4294 wa_mask = (prop->cu_flags & AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE) ? 4295 0xffff : 0xffffffff; 4296 mqd->compute_static_thread_mgmt_se0 = wa_mask; 4297 mqd->compute_static_thread_mgmt_se1 = wa_mask; 4298 mqd->compute_static_thread_mgmt_se2 = wa_mask; 4299 mqd->compute_static_thread_mgmt_se3 = wa_mask; 4300 return; 4301 } 4302 4303 amdgpu_gfx_mqd_symmetrically_map_cu_mask(adev, prop->cu_mask, 4304 prop->cu_mask_count, se_mask); 4305 4306 mqd->compute_static_thread_mgmt_se0 = se_mask[0]; 4307 mqd->compute_static_thread_mgmt_se1 = se_mask[1]; 4308 mqd->compute_static_thread_mgmt_se2 = se_mask[2]; 4309 mqd->compute_static_thread_mgmt_se3 = se_mask[3]; 4310 } 4311 4312 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m, 4313 struct amdgpu_mqd_prop *prop) 4314 { 4315 struct v11_compute_mqd *mqd = m; 4316 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4317 uint32_t tmp; 4318 4319 mqd->header = 0xC0310800; 4320 mqd->compute_pipelinestat_enable = 0x00000001; 4321 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4322 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4323 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4324 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4325 mqd->compute_misc_reserved = 0x00000007; 4326 4327 eop_base_addr = prop->eop_gpu_addr >> 8; 4328 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4329 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4330 4331 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4332 tmp = regCP_HQD_EOP_CONTROL_DEFAULT; 4333 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4334 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1)); 4335 4336 mqd->cp_hqd_eop_control = tmp; 4337 4338 /* enable doorbell? */ 4339 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4340 4341 if (prop->use_doorbell) { 4342 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4343 DOORBELL_OFFSET, prop->doorbell_index); 4344 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4345 DOORBELL_EN, 1); 4346 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4347 DOORBELL_SOURCE, 0); 4348 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4349 DOORBELL_HIT, 0); 4350 } else { 4351 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4352 DOORBELL_EN, 0); 4353 } 4354 4355 mqd->cp_hqd_pq_doorbell_control = tmp; 4356 4357 /* disable the queue if it's active */ 4358 mqd->cp_hqd_dequeue_request = 0; 4359 mqd->cp_hqd_pq_rptr = 0; 4360 mqd->cp_hqd_pq_wptr_lo = 0; 4361 mqd->cp_hqd_pq_wptr_hi = 0; 4362 4363 /* set the pointer to the MQD */ 4364 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc; 4365 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4366 4367 /* set MQD vmid to 0 */ 4368 tmp = regCP_MQD_CONTROL_DEFAULT; 4369 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4370 mqd->cp_mqd_control = tmp; 4371 4372 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4373 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4374 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4375 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4376 4377 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4378 tmp = regCP_HQD_PQ_CONTROL_DEFAULT; 4379 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4380 (order_base_2(prop->queue_size / 4) - 1)); 4381 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4382 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4383 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1); 4384 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 4385 prop->allow_tunneling); 4386 if (prop->kernel_queue) { 4387 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4388 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4389 } 4390 if (prop->tmz_queue) 4391 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TMZ, 1); 4392 mqd->cp_hqd_pq_control = tmp; 4393 4394 /* set the wb address whether it's enabled or not */ 4395 wb_gpu_addr = prop->rptr_gpu_addr; 4396 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4397 mqd->cp_hqd_pq_rptr_report_addr_hi = 4398 upper_32_bits(wb_gpu_addr) & 0xffff; 4399 4400 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4401 wb_gpu_addr = prop->wptr_gpu_addr; 4402 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4403 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4404 4405 tmp = 0; 4406 /* enable the doorbell if requested */ 4407 if (prop->use_doorbell) { 4408 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4409 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4410 DOORBELL_OFFSET, prop->doorbell_index); 4411 4412 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4413 DOORBELL_EN, 1); 4414 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4415 DOORBELL_SOURCE, 0); 4416 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4417 DOORBELL_HIT, 0); 4418 } 4419 4420 mqd->cp_hqd_pq_doorbell_control = tmp; 4421 4422 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4423 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT; 4424 4425 /* set the vmid for the queue */ 4426 mqd->cp_hqd_vmid = 0; 4427 4428 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT; 4429 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55); 4430 mqd->cp_hqd_persistent_state = tmp; 4431 4432 /* set MIN_IB_AVAIL_SIZE */ 4433 tmp = regCP_HQD_IB_CONTROL_DEFAULT; 4434 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4435 mqd->cp_hqd_ib_control = tmp; 4436 4437 /* set static priority for a compute queue/ring */ 4438 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; 4439 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; 4440 4441 tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); 4442 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 4443 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); 4444 mqd->cp_hqd_quantum = tmp; 4445 4446 mqd->cp_hqd_active = prop->hqd_active; 4447 4448 /* set UQ fenceaddress */ 4449 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4450 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4451 /* set CU mask */ 4452 gfx_v11_0_compute_mqd_set_cu_mask(adev, mqd, prop); 4453 4454 return 0; 4455 } 4456 4457 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring) 4458 { 4459 struct amdgpu_device *adev = ring->adev; 4460 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4461 int j; 4462 4463 /* inactivate the queue */ 4464 if (amdgpu_sriov_vf(adev)) 4465 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); 4466 4467 /* disable wptr polling */ 4468 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4469 4470 /* write the EOP addr */ 4471 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR, 4472 mqd->cp_hqd_eop_base_addr_lo); 4473 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI, 4474 mqd->cp_hqd_eop_base_addr_hi); 4475 4476 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4477 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL, 4478 mqd->cp_hqd_eop_control); 4479 4480 /* enable doorbell? */ 4481 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4482 mqd->cp_hqd_pq_doorbell_control); 4483 4484 /* disable the queue if it's active */ 4485 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { 4486 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); 4487 for (j = 0; j < adev->usec_timeout; j++) { 4488 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) 4489 break; 4490 udelay(1); 4491 } 4492 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 4493 mqd->cp_hqd_dequeue_request); 4494 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 4495 mqd->cp_hqd_pq_rptr); 4496 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4497 mqd->cp_hqd_pq_wptr_lo); 4498 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4499 mqd->cp_hqd_pq_wptr_hi); 4500 } 4501 4502 /* set the pointer to the MQD */ 4503 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, 4504 mqd->cp_mqd_base_addr_lo); 4505 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, 4506 mqd->cp_mqd_base_addr_hi); 4507 4508 /* set MQD vmid to 0 */ 4509 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 4510 mqd->cp_mqd_control); 4511 4512 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4513 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, 4514 mqd->cp_hqd_pq_base_lo); 4515 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, 4516 mqd->cp_hqd_pq_base_hi); 4517 4518 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4519 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, 4520 mqd->cp_hqd_pq_control); 4521 4522 /* set the wb address whether it's enabled or not */ 4523 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR, 4524 mqd->cp_hqd_pq_rptr_report_addr_lo); 4525 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4526 mqd->cp_hqd_pq_rptr_report_addr_hi); 4527 4528 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4529 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR, 4530 mqd->cp_hqd_pq_wptr_poll_addr_lo); 4531 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI, 4532 mqd->cp_hqd_pq_wptr_poll_addr_hi); 4533 4534 /* enable the doorbell if requested */ 4535 if (ring->use_doorbell) { 4536 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4537 (adev->doorbell_index.kiq * 2) << 2); 4538 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4539 (adev->doorbell_index.userqueue_end * 2) << 2); 4540 } 4541 4542 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4543 mqd->cp_hqd_pq_doorbell_control); 4544 4545 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4546 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4547 mqd->cp_hqd_pq_wptr_lo); 4548 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4549 mqd->cp_hqd_pq_wptr_hi); 4550 4551 /* set the vmid for the queue */ 4552 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid); 4553 4554 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, 4555 mqd->cp_hqd_persistent_state); 4556 4557 /* activate the queue */ 4558 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 4559 mqd->cp_hqd_active); 4560 4561 if (ring->use_doorbell) 4562 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4563 4564 return 0; 4565 } 4566 4567 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) 4568 { 4569 struct amdgpu_device *adev = ring->adev; 4570 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4571 4572 gfx_v11_0_kiq_setting(ring); 4573 4574 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4575 /* reset MQD to a clean status */ 4576 if (adev->gfx.kiq[0].mqd_backup) 4577 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd)); 4578 4579 /* reset ring buffer */ 4580 ring->wptr = 0; 4581 amdgpu_ring_clear_ring(ring); 4582 4583 mutex_lock(&adev->srbm_mutex); 4584 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4585 gfx_v11_0_kiq_init_register(ring); 4586 soc21_grbm_select(adev, 0, 0, 0, 0); 4587 mutex_unlock(&adev->srbm_mutex); 4588 } else { 4589 memset((void *)mqd, 0, sizeof(*mqd)); 4590 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4591 amdgpu_ring_clear_ring(ring); 4592 mutex_lock(&adev->srbm_mutex); 4593 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4594 amdgpu_ring_init_mqd(ring); 4595 gfx_v11_0_kiq_init_register(ring); 4596 soc21_grbm_select(adev, 0, 0, 0, 0); 4597 mutex_unlock(&adev->srbm_mutex); 4598 4599 if (adev->gfx.kiq[0].mqd_backup) 4600 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd)); 4601 } 4602 4603 return 0; 4604 } 4605 4606 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset) 4607 { 4608 struct amdgpu_device *adev = ring->adev; 4609 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4610 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4611 4612 if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { 4613 memset((void *)mqd, 0, sizeof(*mqd)); 4614 mutex_lock(&adev->srbm_mutex); 4615 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4616 amdgpu_ring_init_mqd(ring); 4617 soc21_grbm_select(adev, 0, 0, 0, 0); 4618 mutex_unlock(&adev->srbm_mutex); 4619 4620 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4621 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4622 } else { 4623 /* restore MQD to a clean status */ 4624 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4625 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); 4626 /* reset ring buffer */ 4627 ring->wptr = 0; 4628 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4629 amdgpu_ring_clear_ring(ring); 4630 } 4631 4632 return 0; 4633 } 4634 4635 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev) 4636 { 4637 gfx_v11_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 4638 return 0; 4639 } 4640 4641 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev) 4642 { 4643 int i, r; 4644 4645 if (!amdgpu_async_gfx_ring) 4646 gfx_v11_0_cp_compute_enable(adev, true); 4647 4648 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4649 r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i], false); 4650 if (r) 4651 return r; 4652 } 4653 4654 return amdgpu_gfx_enable_kcq(adev, 0); 4655 } 4656 4657 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev) 4658 { 4659 int r, i; 4660 struct amdgpu_ring *ring; 4661 4662 if (!(adev->flags & AMD_IS_APU)) 4663 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 4664 4665 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4666 /* legacy firmware loading */ 4667 r = gfx_v11_0_cp_gfx_load_microcode(adev); 4668 if (r) 4669 return r; 4670 4671 if (adev->gfx.rs64_enable) 4672 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev); 4673 else 4674 r = gfx_v11_0_cp_compute_load_microcode(adev); 4675 if (r) 4676 return r; 4677 } 4678 4679 gfx_v11_0_cp_set_doorbell_range(adev); 4680 4681 if (amdgpu_async_gfx_ring) { 4682 gfx_v11_0_cp_compute_enable(adev, true); 4683 gfx_v11_0_cp_gfx_enable(adev, true); 4684 } 4685 4686 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) 4687 r = amdgpu_mes_kiq_hw_init(adev, 0); 4688 else 4689 r = gfx_v11_0_kiq_resume(adev); 4690 if (r) 4691 return r; 4692 4693 r = gfx_v11_0_kcq_resume(adev); 4694 if (r) 4695 return r; 4696 4697 if (!amdgpu_async_gfx_ring) { 4698 r = gfx_v11_0_cp_gfx_resume(adev); 4699 if (r) 4700 return r; 4701 } else { 4702 r = gfx_v11_0_cp_async_gfx_ring_resume(adev); 4703 if (r) 4704 return r; 4705 } 4706 4707 if (adev->gfx.disable_kq) { 4708 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4709 ring = &adev->gfx.gfx_ring[i]; 4710 /* we don't want to set ring->ready */ 4711 r = amdgpu_ring_test_ring(ring); 4712 if (r) 4713 return r; 4714 } 4715 if (amdgpu_async_gfx_ring) 4716 amdgpu_gfx_disable_kgq(adev, 0); 4717 } else { 4718 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4719 ring = &adev->gfx.gfx_ring[i]; 4720 r = amdgpu_ring_test_helper(ring); 4721 if (r) 4722 return r; 4723 } 4724 } 4725 4726 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4727 ring = &adev->gfx.compute_ring[i]; 4728 r = amdgpu_ring_test_helper(ring); 4729 if (r) 4730 return r; 4731 } 4732 4733 return 0; 4734 } 4735 4736 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable) 4737 { 4738 gfx_v11_0_cp_gfx_enable(adev, enable); 4739 gfx_v11_0_cp_compute_enable(adev, enable); 4740 } 4741 4742 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev) 4743 { 4744 int r; 4745 bool value; 4746 4747 r = adev->gfxhub.funcs->gart_enable(adev); 4748 if (r) 4749 return r; 4750 4751 amdgpu_device_flush_hdp(adev, NULL); 4752 4753 value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS; 4754 4755 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 4756 /* TODO investigate why this and the hdp flush above is needed, 4757 * are we missing a flush somewhere else? */ 4758 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0); 4759 4760 return 0; 4761 } 4762 4763 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev) 4764 { 4765 u32 tmp; 4766 4767 /* select RS64 */ 4768 if (adev->gfx.rs64_enable) { 4769 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL); 4770 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1); 4771 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp); 4772 4773 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL); 4774 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1); 4775 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp); 4776 } 4777 4778 if (amdgpu_emu_mode == 1) 4779 msleep(100); 4780 } 4781 4782 static int get_gb_addr_config(struct amdgpu_device * adev) 4783 { 4784 u32 gb_addr_config; 4785 4786 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG); 4787 if (gb_addr_config == 0) 4788 return -EINVAL; 4789 4790 adev->gfx.config.gb_addr_config_fields.num_pkrs = 4791 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS); 4792 4793 adev->gfx.config.gb_addr_config = gb_addr_config; 4794 4795 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 4796 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4797 GB_ADDR_CONFIG, NUM_PIPES); 4798 4799 adev->gfx.config.max_tile_pipes = 4800 adev->gfx.config.gb_addr_config_fields.num_pipes; 4801 4802 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 4803 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4804 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS); 4805 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 4806 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4807 GB_ADDR_CONFIG, NUM_RB_PER_SE); 4808 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 4809 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4810 GB_ADDR_CONFIG, NUM_SHADER_ENGINES); 4811 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 4812 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4813 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE)); 4814 4815 return 0; 4816 } 4817 4818 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev) 4819 { 4820 uint32_t data; 4821 4822 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG); 4823 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK; 4824 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data); 4825 4826 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG); 4827 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK; 4828 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); 4829 } 4830 4831 static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, 4832 bool enable) 4833 { 4834 unsigned int irq_type; 4835 int m, p, r; 4836 4837 if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { 4838 for (m = 0; m < adev->gfx.me.num_me; m++) { 4839 for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { 4840 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4841 if (enable) 4842 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4843 else 4844 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4845 if (r) { 4846 if (!enable) 4847 return r; 4848 goto err_gfx; 4849 } 4850 } 4851 } 4852 } 4853 4854 if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { 4855 for (m = 0; m < adev->gfx.mec.num_mec; ++m) { 4856 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { 4857 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4858 + (m * adev->gfx.mec.num_pipe_per_mec) 4859 + p; 4860 if (enable) 4861 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4862 else 4863 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4864 if (r) { 4865 if (!enable) 4866 return r; 4867 goto err_compute; 4868 } 4869 } 4870 } 4871 } 4872 4873 return 0; 4874 4875 err_compute: 4876 for (p--; p >= 0; p--) { 4877 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4878 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4879 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4880 } 4881 for (m--; m >= 0; m--) { 4882 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { 4883 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4884 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4885 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4886 } 4887 } 4888 m = adev->gfx.me.num_me; 4889 err_gfx: 4890 for (p--; p >= 0; p--) { 4891 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4892 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4893 } 4894 for (m--; m >= 0; m--) { 4895 for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { 4896 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4897 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4898 } 4899 } 4900 return r; 4901 } 4902 4903 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) 4904 { 4905 int r; 4906 struct amdgpu_device *adev = ip_block->adev; 4907 4908 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, 4909 adev->gfx.cleaner_shader_ptr); 4910 4911 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 4912 if (adev->gfx.imu.funcs) { 4913 /* RLC autoload sequence 1: Program rlc ram */ 4914 if (adev->gfx.imu.funcs->program_rlc_ram) 4915 adev->gfx.imu.funcs->program_rlc_ram(adev); 4916 /* rlc autoload firmware */ 4917 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev); 4918 if (r) 4919 return r; 4920 } 4921 } else { 4922 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4923 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) { 4924 if (adev->gfx.imu.funcs->load_microcode) 4925 adev->gfx.imu.funcs->load_microcode(adev); 4926 if (adev->gfx.imu.funcs->setup_imu) 4927 adev->gfx.imu.funcs->setup_imu(adev); 4928 if (adev->gfx.imu.funcs->start_imu) 4929 adev->gfx.imu.funcs->start_imu(adev); 4930 } 4931 4932 /* disable gpa mode in backdoor loading */ 4933 gfx_v11_0_disable_gpa_mode(adev); 4934 } 4935 } 4936 4937 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) || 4938 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 4939 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev); 4940 if (r) { 4941 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r); 4942 return r; 4943 } 4944 } 4945 4946 adev->gfx.is_poweron = true; 4947 4948 if(get_gb_addr_config(adev)) 4949 drm_warn(adev_to_drm(adev), "Invalid gb_addr_config !\n"); 4950 4951 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP && 4952 adev->gfx.rs64_enable) 4953 gfx_v11_0_config_gfx_rs64(adev); 4954 4955 r = gfx_v11_0_gfxhub_enable(adev); 4956 if (r) 4957 return r; 4958 4959 if (!amdgpu_emu_mode) 4960 gfx_v11_0_init_golden_registers(adev); 4961 4962 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || 4963 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) { 4964 /** 4965 * For gfx 11, rlc firmware loading relies on smu firmware is 4966 * loaded firstly, so in direct type, it has to load smc ucode 4967 * here before rlc. 4968 */ 4969 r = amdgpu_pm_load_smu_firmware(adev, NULL); 4970 if (r) 4971 return r; 4972 } 4973 4974 gfx_v11_0_constants_init(adev); 4975 4976 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 4977 gfx_v11_0_select_cp_fw_arch(adev); 4978 4979 if (adev->nbio.funcs->gc_doorbell_init) 4980 adev->nbio.funcs->gc_doorbell_init(adev); 4981 4982 r = gfx_v11_0_rlc_resume(adev); 4983 if (r) 4984 return r; 4985 4986 /* 4987 * init golden registers and rlc resume may override some registers, 4988 * reconfig them here 4989 */ 4990 gfx_v11_0_tcp_harvest(adev); 4991 4992 r = gfx_v11_0_cp_resume(adev); 4993 if (r) 4994 return r; 4995 4996 /* get IMU version from HW if it's not set */ 4997 if (!adev->gfx.imu_fw_version) 4998 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0); 4999 5000 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5001 if (r) 5002 return r; 5003 5004 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5005 if (r) 5006 goto err_priv_inst; 5007 5008 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); 5009 if (r) 5010 goto err_bad_op; 5011 5012 r = gfx_v11_0_set_userq_eop_interrupts(adev, true); 5013 if (r) 5014 goto err_userq_eop; 5015 5016 return 0; 5017 5018 err_userq_eop: 5019 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5020 err_bad_op: 5021 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5022 err_priv_inst: 5023 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5024 return r; 5025 } 5026 5027 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) 5028 { 5029 struct amdgpu_device *adev = ip_block->adev; 5030 5031 cancel_delayed_work_sync(&adev->gfx.idle_work); 5032 5033 gfx_v11_0_set_userq_eop_interrupts(adev, false); 5034 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5035 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5036 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5037 5038 if (!adev->no_hw_access) { 5039 if (amdgpu_async_gfx_ring && 5040 !adev->gfx.disable_kq) { 5041 if (amdgpu_gfx_disable_kgq(adev, 0)) 5042 DRM_ERROR("KGQ disable failed\n"); 5043 } 5044 5045 if (amdgpu_gfx_disable_kcq(adev, 0)) 5046 DRM_ERROR("KCQ disable failed\n"); 5047 5048 amdgpu_mes_kiq_hw_fini(adev, 0); 5049 } 5050 5051 if (amdgpu_sriov_vf(adev)) 5052 /* Remove the steps disabling CPG and clearing KIQ position, 5053 * so that CP could perform IDLE-SAVE during switch. Those 5054 * steps are necessary to avoid a DMAR error in gfx9 but it is 5055 * not reproduced on gfx11. 5056 */ 5057 return 0; 5058 5059 gfx_v11_0_cp_enable(adev, false); 5060 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 5061 5062 adev->gfxhub.funcs->gart_disable(adev); 5063 5064 adev->gfx.is_poweron = false; 5065 5066 return 0; 5067 } 5068 5069 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block) 5070 { 5071 return gfx_v11_0_hw_fini(ip_block); 5072 } 5073 5074 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block) 5075 { 5076 return gfx_v11_0_hw_init(ip_block); 5077 } 5078 5079 static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block) 5080 { 5081 struct amdgpu_device *adev = ip_block->adev; 5082 5083 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS), 5084 GRBM_STATUS, GUI_ACTIVE)) 5085 return false; 5086 else 5087 return true; 5088 } 5089 5090 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 5091 { 5092 unsigned i; 5093 u32 tmp; 5094 struct amdgpu_device *adev = ip_block->adev; 5095 5096 for (i = 0; i < adev->usec_timeout; i++) { 5097 /* read MC_STATUS */ 5098 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) & 5099 GRBM_STATUS__GUI_ACTIVE_MASK; 5100 5101 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE)) 5102 return 0; 5103 udelay(1); 5104 } 5105 return -ETIMEDOUT; 5106 } 5107 5108 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev, 5109 bool req) 5110 { 5111 u32 i, tmp, val; 5112 5113 for (i = 0; i < adev->usec_timeout; i++) { 5114 /* Request with MeId=2, PipeId=0 */ 5115 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req); 5116 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4); 5117 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp); 5118 5119 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX); 5120 if (req) { 5121 if (val == tmp) 5122 break; 5123 } else { 5124 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, 5125 REQUEST, 1); 5126 5127 /* unlocked or locked by firmware */ 5128 if (val != tmp) 5129 break; 5130 } 5131 udelay(1); 5132 } 5133 5134 if (i >= adev->usec_timeout) 5135 return -EINVAL; 5136 5137 return 0; 5138 } 5139 5140 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block) 5141 { 5142 u32 grbm_soft_reset = 0; 5143 u32 tmp; 5144 int r, i, j, k; 5145 struct amdgpu_device *adev = ip_block->adev; 5146 5147 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5148 5149 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5150 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0); 5151 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0); 5152 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0); 5153 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0); 5154 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5155 5156 mutex_lock(&adev->srbm_mutex); 5157 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 5158 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 5159 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 5160 soc21_grbm_select(adev, i, k, j, 0); 5161 5162 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2); 5163 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1); 5164 } 5165 } 5166 } 5167 for (i = 0; i < adev->gfx.me.num_me; ++i) { 5168 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { 5169 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 5170 soc21_grbm_select(adev, i, k, j, 0); 5171 5172 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1); 5173 } 5174 } 5175 } 5176 soc21_grbm_select(adev, 0, 0, 0, 0); 5177 mutex_unlock(&adev->srbm_mutex); 5178 5179 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */ 5180 mutex_lock(&adev->gfx.reset_sem_mutex); 5181 r = gfx_v11_0_request_gfx_index_mutex(adev, true); 5182 if (r) { 5183 mutex_unlock(&adev->gfx.reset_sem_mutex); 5184 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n"); 5185 return r; 5186 } 5187 5188 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe); 5189 5190 // Read CP_VMID_RESET register three times. 5191 // to get sufficient time for GFX_HQD_ACTIVE reach 0 5192 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5193 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5194 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5195 5196 /* release the gfx mutex */ 5197 r = gfx_v11_0_request_gfx_index_mutex(adev, false); 5198 mutex_unlock(&adev->gfx.reset_sem_mutex); 5199 if (r) { 5200 DRM_ERROR("Failed to release the gfx mutex during soft reset\n"); 5201 return r; 5202 } 5203 5204 for (i = 0; i < adev->usec_timeout; i++) { 5205 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) && 5206 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE)) 5207 break; 5208 udelay(1); 5209 } 5210 if (i >= adev->usec_timeout) { 5211 printk("Failed to wait all pipes clean\n"); 5212 return -EINVAL; 5213 } 5214 5215 /********** trigger soft reset ***********/ 5216 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5217 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5218 SOFT_RESET_CP, 1); 5219 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5220 SOFT_RESET_GFX, 1); 5221 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5222 SOFT_RESET_CPF, 1); 5223 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5224 SOFT_RESET_CPC, 1); 5225 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5226 SOFT_RESET_CPG, 1); 5227 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5228 /********** exit soft reset ***********/ 5229 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5230 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5231 SOFT_RESET_CP, 0); 5232 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5233 SOFT_RESET_GFX, 0); 5234 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5235 SOFT_RESET_CPF, 0); 5236 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5237 SOFT_RESET_CPC, 0); 5238 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5239 SOFT_RESET_CPG, 0); 5240 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5241 5242 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL); 5243 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1); 5244 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp); 5245 5246 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0); 5247 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0); 5248 5249 for (i = 0; i < adev->usec_timeout; i++) { 5250 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET)) 5251 break; 5252 udelay(1); 5253 } 5254 if (i >= adev->usec_timeout) { 5255 printk("Failed to wait CP_VMID_RESET to 0\n"); 5256 return -EINVAL; 5257 } 5258 5259 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5260 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5261 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5262 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5263 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5264 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5265 5266 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5267 5268 return gfx_v11_0_cp_resume(adev); 5269 } 5270 5271 static bool gfx_v11_0_check_soft_reset(struct amdgpu_ip_block *ip_block) 5272 { 5273 int i, r; 5274 struct amdgpu_device *adev = ip_block->adev; 5275 struct amdgpu_ring *ring; 5276 long tmo = msecs_to_jiffies(1000); 5277 5278 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 5279 ring = &adev->gfx.gfx_ring[i]; 5280 r = amdgpu_ring_test_ib(ring, tmo); 5281 if (r) 5282 return true; 5283 } 5284 5285 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 5286 ring = &adev->gfx.compute_ring[i]; 5287 r = amdgpu_ring_test_ib(ring, tmo); 5288 if (r) 5289 return true; 5290 } 5291 5292 return false; 5293 } 5294 5295 static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block) 5296 { 5297 struct amdgpu_device *adev = ip_block->adev; 5298 /** 5299 * GFX soft reset will impact MES, need resume MES when do GFX soft reset 5300 */ 5301 return amdgpu_mes_resume(adev, 0); 5302 } 5303 5304 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5305 { 5306 uint64_t clock; 5307 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after; 5308 5309 if (amdgpu_sriov_vf(adev)) { 5310 amdgpu_gfx_off_ctrl(adev, false); 5311 mutex_lock(&adev->gfx.gpu_clock_mutex); 5312 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5313 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5314 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5315 if (clock_counter_hi_pre != clock_counter_hi_after) 5316 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5317 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5318 amdgpu_gfx_off_ctrl(adev, true); 5319 } else { 5320 preempt_disable(); 5321 if (amdgpu_ip_version(adev, SMUIO_HWIP, 0) < IP_VERSION(15, 0, 0)) { 5322 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5323 regGOLDEN_TSC_COUNT_UPPER); 5324 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5325 regGOLDEN_TSC_COUNT_LOWER); 5326 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5327 regGOLDEN_TSC_COUNT_UPPER); 5328 if (clock_counter_hi_pre != clock_counter_hi_after) 5329 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5330 regGOLDEN_TSC_COUNT_LOWER); 5331 } else { 5332 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5333 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5334 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5335 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5336 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5337 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5338 if (clock_counter_hi_pre != clock_counter_hi_after) 5339 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5340 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5341 } 5342 preempt_enable(); 5343 } 5344 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL); 5345 5346 return clock; 5347 } 5348 5349 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5350 uint32_t vmid, 5351 uint32_t gds_base, uint32_t gds_size, 5352 uint32_t gws_base, uint32_t gws_size, 5353 uint32_t oa_base, uint32_t oa_size) 5354 { 5355 struct amdgpu_device *adev = ring->adev; 5356 5357 /* GDS Base */ 5358 gfx_v11_0_write_data_to_reg(ring, 0, false, 5359 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid, 5360 gds_base); 5361 5362 /* GDS Size */ 5363 gfx_v11_0_write_data_to_reg(ring, 0, false, 5364 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid, 5365 gds_size); 5366 5367 /* GWS */ 5368 gfx_v11_0_write_data_to_reg(ring, 0, false, 5369 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid, 5370 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5371 5372 /* OA */ 5373 gfx_v11_0_write_data_to_reg(ring, 0, false, 5374 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid, 5375 (1 << (oa_size + oa_base)) - (1 << oa_base)); 5376 } 5377 5378 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) 5379 { 5380 struct amdgpu_device *adev = ip_block->adev; 5381 5382 switch (amdgpu_user_queue) { 5383 case -1: 5384 case 0: 5385 default: 5386 adev->gfx.disable_kq = false; 5387 adev->gfx.disable_uq = true; 5388 break; 5389 case 1: 5390 adev->gfx.disable_kq = false; 5391 adev->gfx.disable_uq = false; 5392 break; 5393 case 2: 5394 adev->gfx.disable_kq = true; 5395 adev->gfx.disable_uq = false; 5396 break; 5397 } 5398 5399 adev->gfx.funcs = &gfx_v11_0_gfx_funcs; 5400 5401 if (adev->gfx.disable_kq) { 5402 /* We need one GFX ring temporarily to set up 5403 * the clear state. 5404 */ 5405 adev->gfx.num_gfx_rings = 1; 5406 adev->gfx.num_compute_rings = 0; 5407 } else { 5408 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS; 5409 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5410 AMDGPU_MAX_COMPUTE_RINGS); 5411 } 5412 5413 gfx_v11_0_set_kiq_pm4_funcs(adev); 5414 gfx_v11_0_set_ring_funcs(adev); 5415 gfx_v11_0_set_irq_funcs(adev); 5416 gfx_v11_0_set_gds_init(adev); 5417 gfx_v11_0_set_rlc_funcs(adev); 5418 gfx_v11_0_set_mqd_funcs(adev); 5419 gfx_v11_0_set_imu_funcs(adev); 5420 5421 gfx_v11_0_init_rlcg_reg_access_ctrl(adev); 5422 5423 return gfx_v11_0_init_microcode(adev); 5424 } 5425 5426 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev) 5427 { 5428 uint32_t rlc_cntl; 5429 5430 /* if RLC is not enabled, do nothing */ 5431 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); 5432 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; 5433 } 5434 5435 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5436 { 5437 uint32_t data; 5438 unsigned i; 5439 5440 data = RLC_SAFE_MODE__CMD_MASK; 5441 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5442 5443 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data); 5444 5445 /* wait for RLC_SAFE_MODE */ 5446 for (i = 0; i < adev->usec_timeout; i++) { 5447 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE), 5448 RLC_SAFE_MODE, CMD)) 5449 break; 5450 udelay(1); 5451 } 5452 } 5453 5454 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5455 { 5456 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK); 5457 } 5458 5459 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 5460 bool enable) 5461 { 5462 uint32_t def, data; 5463 5464 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK)) 5465 return; 5466 5467 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5468 5469 if (enable) 5470 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5471 else 5472 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5473 5474 if (def != data) 5475 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5476 } 5477 5478 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev, 5479 bool enable) 5480 { 5481 uint32_t def, data; 5482 5483 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG)) 5484 return; 5485 5486 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5487 5488 if (enable) 5489 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5490 else 5491 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5492 5493 if (def != data) 5494 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5495 } 5496 5497 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev, 5498 bool enable) 5499 { 5500 uint32_t def, data; 5501 5502 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG)) 5503 return; 5504 5505 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5506 5507 if (enable) 5508 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5509 else 5510 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5511 5512 if (def != data) 5513 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5514 } 5515 5516 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5517 bool enable) 5518 { 5519 uint32_t data, def; 5520 5521 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS))) 5522 return; 5523 5524 /* It is disabled by HW by default */ 5525 if (enable) { 5526 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5527 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 5528 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5529 5530 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5531 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5532 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5533 5534 if (def != data) 5535 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5536 } 5537 } else { 5538 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5539 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5540 5541 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5542 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5543 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5544 5545 if (def != data) 5546 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5547 } 5548 } 5549 } 5550 5551 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5552 bool enable) 5553 { 5554 uint32_t def, data; 5555 5556 if (!(adev->cg_flags & 5557 (AMD_CG_SUPPORT_GFX_CGCG | 5558 AMD_CG_SUPPORT_GFX_CGLS | 5559 AMD_CG_SUPPORT_GFX_3D_CGCG | 5560 AMD_CG_SUPPORT_GFX_3D_CGLS))) 5561 return; 5562 5563 if (enable) { 5564 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5565 5566 /* unset CGCG override */ 5567 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5568 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5569 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5570 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5571 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG || 5572 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5573 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 5574 5575 /* update CGCG override bits */ 5576 if (def != data) 5577 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5578 5579 /* enable cgcg FSM(0x0000363F) */ 5580 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5581 5582 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5583 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK; 5584 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5585 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5586 } 5587 5588 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5589 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK; 5590 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5591 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5592 } 5593 5594 if (def != data) 5595 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5596 5597 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5598 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5599 5600 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5601 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK; 5602 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5603 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5604 } 5605 5606 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5607 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK; 5608 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5609 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5610 } 5611 5612 if (def != data) 5613 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5614 5615 /* set IDLE_POLL_COUNT(0x00900100) */ 5616 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL); 5617 5618 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK); 5619 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5620 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5621 5622 if (def != data) 5623 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data); 5624 5625 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5626 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5627 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5628 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5629 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5630 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data); 5631 5632 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5633 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5634 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5635 5636 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5637 if (adev->sdma.num_instances > 1) { 5638 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5639 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5640 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5641 } 5642 } else { 5643 /* Program RLC_CGCG_CGLS_CTRL */ 5644 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5645 5646 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5647 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5648 5649 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5650 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5651 5652 if (def != data) 5653 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5654 5655 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5656 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5657 5658 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 5659 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5660 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5661 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5662 5663 if (def != data) 5664 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5665 5666 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5667 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5668 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5669 5670 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5671 if (adev->sdma.num_instances > 1) { 5672 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5673 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5674 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5675 } 5676 } 5677 } 5678 5679 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5680 bool enable) 5681 { 5682 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5683 5684 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable); 5685 5686 gfx_v11_0_update_medium_grain_clock_gating(adev, enable); 5687 5688 gfx_v11_0_update_repeater_fgcg(adev, enable); 5689 5690 gfx_v11_0_update_sram_fgcg(adev, enable); 5691 5692 gfx_v11_0_update_perf_clk(adev, enable); 5693 5694 if (adev->cg_flags & 5695 (AMD_CG_SUPPORT_GFX_MGCG | 5696 AMD_CG_SUPPORT_GFX_CGLS | 5697 AMD_CG_SUPPORT_GFX_CGCG | 5698 AMD_CG_SUPPORT_GFX_3D_CGCG | 5699 AMD_CG_SUPPORT_GFX_3D_CGLS)) 5700 gfx_v11_0_enable_gui_idle_interrupt(adev, enable); 5701 5702 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5703 5704 return 0; 5705 } 5706 5707 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5708 struct amdgpu_ring *ring, unsigned vmid) 5709 { 5710 u32 reg, pre_data, data; 5711 5712 amdgpu_gfx_off_ctrl(adev, false); 5713 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); 5714 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) 5715 pre_data = RREG32_NO_KIQ(reg); 5716 else 5717 pre_data = RREG32(reg); 5718 5719 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK); 5720 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5721 5722 if (pre_data != data) { 5723 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) { 5724 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data); 5725 } else 5726 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data); 5727 } 5728 amdgpu_gfx_off_ctrl(adev, true); 5729 5730 if (ring 5731 && amdgpu_sriov_is_pp_one_vf(adev) 5732 && (pre_data != data) 5733 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) 5734 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) { 5735 amdgpu_ring_emit_wreg(ring, reg, data); 5736 } 5737 } 5738 5739 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = { 5740 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled, 5741 .set_safe_mode = gfx_v11_0_set_safe_mode, 5742 .unset_safe_mode = gfx_v11_0_unset_safe_mode, 5743 .init = gfx_v11_0_rlc_init, 5744 .get_csb_size = gfx_v11_0_get_csb_size, 5745 .get_csb_buffer = gfx_v11_0_get_csb_buffer, 5746 .resume = gfx_v11_0_rlc_resume, 5747 .stop = gfx_v11_0_rlc_stop, 5748 .reset = gfx_v11_0_rlc_reset, 5749 .start = gfx_v11_0_rlc_start, 5750 .update_spm_vmid = gfx_v11_0_update_spm_vmid, 5751 }; 5752 5753 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable) 5754 { 5755 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 5756 5757 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) 5758 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5759 else 5760 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5761 5762 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data); 5763 5764 // Program RLC_PG_DELAY3 for CGPG hysteresis 5765 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { 5766 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5767 case IP_VERSION(11, 0, 1): 5768 case IP_VERSION(11, 0, 4): 5769 case IP_VERSION(11, 5, 0): 5770 case IP_VERSION(11, 5, 1): 5771 case IP_VERSION(11, 5, 2): 5772 case IP_VERSION(11, 5, 3): 5773 case IP_VERSION(11, 5, 4): 5774 case IP_VERSION(11, 5, 6): 5775 case IP_VERSION(11, 7, 0): 5776 case IP_VERSION(11, 7, 1): 5777 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); 5778 break; 5779 default: 5780 break; 5781 } 5782 } 5783 } 5784 5785 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable) 5786 { 5787 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5788 5789 gfx_v11_cntl_power_gating(adev, enable); 5790 5791 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5792 } 5793 5794 static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5795 enum amd_powergating_state state) 5796 { 5797 struct amdgpu_device *adev = ip_block->adev; 5798 bool enable = (state == AMD_PG_STATE_GATE); 5799 5800 if (amdgpu_sriov_vf(adev)) 5801 return 0; 5802 5803 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5804 case IP_VERSION(11, 0, 0): 5805 case IP_VERSION(11, 0, 2): 5806 case IP_VERSION(11, 0, 3): 5807 amdgpu_gfx_off_ctrl(adev, enable); 5808 break; 5809 case IP_VERSION(11, 0, 1): 5810 case IP_VERSION(11, 0, 4): 5811 case IP_VERSION(11, 5, 0): 5812 case IP_VERSION(11, 5, 1): 5813 case IP_VERSION(11, 5, 2): 5814 case IP_VERSION(11, 5, 3): 5815 case IP_VERSION(11, 5, 4): 5816 case IP_VERSION(11, 5, 6): 5817 case IP_VERSION(11, 7, 0): 5818 case IP_VERSION(11, 7, 1): 5819 if (!enable) 5820 amdgpu_gfx_off_ctrl(adev, false); 5821 5822 gfx_v11_cntl_pg(adev, enable); 5823 5824 if (enable) 5825 amdgpu_gfx_off_ctrl(adev, true); 5826 5827 break; 5828 default: 5829 break; 5830 } 5831 5832 return 0; 5833 } 5834 5835 static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5836 enum amd_clockgating_state state) 5837 { 5838 struct amdgpu_device *adev = ip_block->adev; 5839 5840 if (amdgpu_sriov_vf(adev)) 5841 return 0; 5842 5843 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5844 case IP_VERSION(11, 0, 0): 5845 case IP_VERSION(11, 0, 1): 5846 case IP_VERSION(11, 0, 2): 5847 case IP_VERSION(11, 0, 3): 5848 case IP_VERSION(11, 0, 4): 5849 case IP_VERSION(11, 5, 0): 5850 case IP_VERSION(11, 5, 1): 5851 case IP_VERSION(11, 5, 2): 5852 case IP_VERSION(11, 5, 3): 5853 case IP_VERSION(11, 5, 4): 5854 case IP_VERSION(11, 5, 6): 5855 case IP_VERSION(11, 7, 0): 5856 case IP_VERSION(11, 7, 1): 5857 gfx_v11_0_update_gfx_clock_gating(adev, 5858 state == AMD_CG_STATE_GATE); 5859 break; 5860 default: 5861 break; 5862 } 5863 5864 return 0; 5865 } 5866 5867 static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5868 { 5869 struct amdgpu_device *adev = ip_block->adev; 5870 int data; 5871 5872 /* AMD_CG_SUPPORT_GFX_MGCG */ 5873 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5874 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5875 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5876 5877 /* AMD_CG_SUPPORT_REPEATER_FGCG */ 5878 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK)) 5879 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG; 5880 5881 /* AMD_CG_SUPPORT_GFX_FGCG */ 5882 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK)) 5883 *flags |= AMD_CG_SUPPORT_GFX_FGCG; 5884 5885 /* AMD_CG_SUPPORT_GFX_PERF_CLK */ 5886 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK)) 5887 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK; 5888 5889 /* AMD_CG_SUPPORT_GFX_CGCG */ 5890 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5891 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5892 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5893 5894 /* AMD_CG_SUPPORT_GFX_CGLS */ 5895 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5896 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5897 5898 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5899 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5900 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5901 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5902 5903 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5904 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5905 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5906 } 5907 5908 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5909 { 5910 /* gfx11 is 32bit rptr*/ 5911 return *(uint32_t *)ring->rptr_cpu_addr; 5912 } 5913 5914 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5915 { 5916 struct amdgpu_device *adev = ring->adev; 5917 u64 wptr; 5918 5919 /* XXX check if swapping is necessary on BE */ 5920 if (ring->use_doorbell) { 5921 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5922 } else { 5923 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR); 5924 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32; 5925 } 5926 5927 return wptr; 5928 } 5929 5930 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5931 { 5932 struct amdgpu_device *adev = ring->adev; 5933 5934 if (ring->use_doorbell) { 5935 /* XXX check if swapping is necessary on BE */ 5936 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5937 ring->wptr); 5938 WDOORBELL64(ring->doorbell_index, ring->wptr); 5939 } else { 5940 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, 5941 lower_32_bits(ring->wptr)); 5942 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, 5943 upper_32_bits(ring->wptr)); 5944 } 5945 } 5946 5947 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5948 { 5949 /* gfx11 hardware is 32bit rptr */ 5950 return *(uint32_t *)ring->rptr_cpu_addr; 5951 } 5952 5953 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5954 { 5955 u64 wptr; 5956 5957 /* XXX check if swapping is necessary on BE */ 5958 if (ring->use_doorbell) 5959 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5960 else 5961 BUG(); 5962 return wptr; 5963 } 5964 5965 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5966 { 5967 struct amdgpu_device *adev = ring->adev; 5968 5969 /* XXX check if swapping is necessary on BE */ 5970 if (ring->use_doorbell) { 5971 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5972 ring->wptr); 5973 WDOORBELL64(ring->doorbell_index, ring->wptr); 5974 } else { 5975 BUG(); /* only DOORBELL method supported on gfx11 now */ 5976 } 5977 } 5978 5979 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5980 { 5981 struct amdgpu_device *adev = ring->adev; 5982 u32 ref_and_mask, reg_mem_engine; 5983 5984 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5985 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5986 return; 5987 } 5988 5989 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 5990 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5991 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5992 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5993 ref_and_mask, ref_and_mask, 0x20); 5994 } 5995 5996 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5997 struct amdgpu_job *job, 5998 struct amdgpu_ib *ib, 5999 uint32_t flags) 6000 { 6001 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6002 u32 header, control = 0; 6003 6004 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 6005 6006 control |= ib->length_dw | (vmid << 24); 6007 6008 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 6009 control |= INDIRECT_BUFFER_PRE_ENB(1); 6010 6011 if (flags & AMDGPU_IB_PREEMPTED) 6012 control |= INDIRECT_BUFFER_PRE_RESUME(1); 6013 6014 if (vmid && !ring->adev->gfx.rs64_enable) 6015 gfx_v11_0_ring_emit_de_meta(ring, 6016 !amdgpu_sriov_vf(ring->adev) && (flags & AMDGPU_IB_PREEMPTED)); 6017 } 6018 6019 amdgpu_ring_write(ring, header); 6020 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 6021 amdgpu_ring_write(ring, 6022 #ifdef __BIG_ENDIAN 6023 (2 << 0) | 6024 #endif 6025 lower_32_bits(ib->gpu_addr)); 6026 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 6027 amdgpu_ring_write(ring, control); 6028 } 6029 6030 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 6031 struct amdgpu_job *job, 6032 struct amdgpu_ib *ib, 6033 uint32_t flags) 6034 { 6035 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6036 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 6037 6038 /* Currently, there is a high possibility to get wave ID mismatch 6039 * between ME and GDS, leading to a hw deadlock, because ME generates 6040 * different wave IDs than the GDS expects. This situation happens 6041 * randomly when at least 5 compute pipes use GDS ordered append. 6042 * The wave IDs generated by ME are also wrong after suspend/resume. 6043 * Those are probably bugs somewhere else in the kernel driver. 6044 * 6045 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6046 * GDS to 0 for this ring (me/pipe). 6047 */ 6048 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6049 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6050 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID); 6051 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6052 } 6053 6054 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6055 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 6056 amdgpu_ring_write(ring, 6057 #ifdef __BIG_ENDIAN 6058 (2 << 0) | 6059 #endif 6060 lower_32_bits(ib->gpu_addr)); 6061 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 6062 amdgpu_ring_write(ring, control); 6063 } 6064 6065 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 6066 u64 seq, unsigned flags) 6067 { 6068 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6069 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6070 6071 /* RELEASE_MEM - flush caches, send int */ 6072 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 6073 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ | 6074 PACKET3_RELEASE_MEM_GCR_GL2_WB | 6075 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */ 6076 PACKET3_RELEASE_MEM_GCR_GLM_WB | 6077 PACKET3_RELEASE_MEM_CACHE_POLICY(3) | 6078 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6079 PACKET3_RELEASE_MEM_EVENT_INDEX(5))); 6080 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) | 6081 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0))); 6082 6083 /* 6084 * the address should be Qword aligned if 64bit write, Dword 6085 * aligned if only send 32bit data low (discard data high) 6086 */ 6087 if (write64bit) 6088 WARN_ON(addr & 0x7); 6089 else 6090 WARN_ON(addr & 0x3); 6091 amdgpu_ring_write(ring, lower_32_bits(addr)); 6092 amdgpu_ring_write(ring, upper_32_bits(addr)); 6093 amdgpu_ring_write(ring, lower_32_bits(seq)); 6094 amdgpu_ring_write(ring, upper_32_bits(seq)); 6095 amdgpu_ring_write(ring, 0); 6096 } 6097 6098 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6099 { 6100 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6101 uint32_t seq = ring->fence_drv.sync_seq; 6102 uint64_t addr = ring->fence_drv.gpu_addr; 6103 6104 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr), 6105 upper_32_bits(addr), seq, 0xffffffff, 4); 6106 } 6107 6108 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 6109 uint16_t pasid, uint32_t flush_type, 6110 bool all_hub, uint8_t dst_sel) 6111 { 6112 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 6113 amdgpu_ring_write(ring, 6114 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) | 6115 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 6116 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 6117 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 6118 } 6119 6120 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6121 unsigned vmid, uint64_t pd_addr) 6122 { 6123 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6124 6125 /* compute doesn't have PFP */ 6126 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 6127 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6128 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6129 amdgpu_ring_write(ring, 0x0); 6130 } 6131 6132 /* Make sure that we can't skip the SET_Q_MODE packets when the VM 6133 * changed in any way. 6134 */ 6135 ring->set_q_mode_offs = 0; 6136 ring->set_q_mode_ptr = NULL; 6137 } 6138 6139 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6140 u64 seq, unsigned int flags) 6141 { 6142 struct amdgpu_device *adev = ring->adev; 6143 6144 /* write fence seq to the "addr" */ 6145 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6146 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6147 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6148 amdgpu_ring_write(ring, lower_32_bits(addr)); 6149 amdgpu_ring_write(ring, upper_32_bits(addr)); 6150 amdgpu_ring_write(ring, lower_32_bits(seq)); 6151 6152 if (flags & AMDGPU_FENCE_FLAG_INT) { 6153 /* set register to trigger INT */ 6154 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6155 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6156 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6157 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS)); 6158 amdgpu_ring_write(ring, 0); 6159 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6160 } 6161 } 6162 6163 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, 6164 uint32_t flags) 6165 { 6166 uint32_t dw2 = 0; 6167 6168 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6169 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6170 /* set load_global_config & load_global_uconfig */ 6171 dw2 |= 0x8001; 6172 /* set load_cs_sh_regs */ 6173 dw2 |= 0x01000000; 6174 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6175 dw2 |= 0x10002; 6176 } 6177 6178 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6179 amdgpu_ring_write(ring, dw2); 6180 amdgpu_ring_write(ring, 0); 6181 } 6182 6183 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 6184 uint64_t addr) 6185 { 6186 unsigned ret; 6187 6188 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6189 amdgpu_ring_write(ring, lower_32_bits(addr)); 6190 amdgpu_ring_write(ring, upper_32_bits(addr)); 6191 /* discard following DWs if *cond_exec_gpu_addr==0 */ 6192 amdgpu_ring_write(ring, 0); 6193 ret = ring->wptr & ring->buf_mask; 6194 /* patch dummy value later */ 6195 amdgpu_ring_write(ring, 0); 6196 6197 return ret; 6198 } 6199 6200 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring, 6201 u64 shadow_va, u64 csa_va, 6202 u64 gds_va, bool init_shadow, 6203 int vmid) 6204 { 6205 struct amdgpu_device *adev = ring->adev; 6206 unsigned int offs, end; 6207 6208 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj) 6209 return; 6210 6211 /* 6212 * The logic here isn't easy to understand because we need to keep state 6213 * accross multiple executions of the function as well as between the 6214 * CPU and GPU. The general idea is that the newly written GPU command 6215 * has a condition on the previous one and only executed if really 6216 * necessary. 6217 */ 6218 6219 /* 6220 * The dw in the NOP controls if the next SET_Q_MODE packet should be 6221 * executed or not. Reserve 64bits just to be on the save side. 6222 */ 6223 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1)); 6224 offs = ring->wptr & ring->buf_mask; 6225 6226 /* 6227 * We start with skipping the prefix SET_Q_MODE and always executing 6228 * the postfix SET_Q_MODE packet. This is changed below with a 6229 * WRITE_DATA command when the postfix executed. 6230 */ 6231 amdgpu_ring_write(ring, shadow_va ? 1 : 0); 6232 amdgpu_ring_write(ring, 0); 6233 6234 if (ring->set_q_mode_offs) { 6235 uint64_t addr; 6236 6237 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6238 addr += ring->set_q_mode_offs << 2; 6239 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr); 6240 } 6241 6242 /* 6243 * When the postfix SET_Q_MODE packet executes we need to make sure that the 6244 * next prefix SET_Q_MODE packet executes as well. 6245 */ 6246 if (!shadow_va) { 6247 uint64_t addr; 6248 6249 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6250 addr += offs << 2; 6251 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6252 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); 6253 amdgpu_ring_write(ring, lower_32_bits(addr)); 6254 amdgpu_ring_write(ring, upper_32_bits(addr)); 6255 amdgpu_ring_write(ring, 0x1); 6256 } 6257 6258 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7)); 6259 amdgpu_ring_write(ring, lower_32_bits(shadow_va)); 6260 amdgpu_ring_write(ring, upper_32_bits(shadow_va)); 6261 amdgpu_ring_write(ring, lower_32_bits(gds_va)); 6262 amdgpu_ring_write(ring, upper_32_bits(gds_va)); 6263 amdgpu_ring_write(ring, lower_32_bits(csa_va)); 6264 amdgpu_ring_write(ring, upper_32_bits(csa_va)); 6265 amdgpu_ring_write(ring, shadow_va ? 6266 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0); 6267 amdgpu_ring_write(ring, init_shadow ? 6268 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0); 6269 6270 if (ring->set_q_mode_offs) 6271 amdgpu_ring_patch_cond_exec(ring, end); 6272 6273 if (shadow_va) { 6274 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid; 6275 6276 /* 6277 * If the tokens match try to skip the last postfix SET_Q_MODE 6278 * packet to avoid saving/restoring the state all the time. 6279 */ 6280 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token) 6281 *ring->set_q_mode_ptr = 0; 6282 6283 ring->set_q_mode_token = token; 6284 } else { 6285 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs]; 6286 } 6287 6288 ring->set_q_mode_offs = offs; 6289 } 6290 6291 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume) 6292 { 6293 struct amdgpu_device *adev = ring->adev; 6294 struct v10_de_ib_state de_payload = {0}; 6295 uint64_t offset, gds_addr, de_payload_gpu_addr; 6296 void *de_payload_cpu_addr; 6297 int cnt; 6298 6299 offset = offsetof(struct v10_gfx_meta_data, de_payload); 6300 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 6301 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 6302 6303 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 6304 AMDGPU_CSA_SIZE - adev->gds.gds_size, 6305 PAGE_SIZE); 6306 6307 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 6308 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 6309 6310 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 6311 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 6312 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 6313 WRITE_DATA_DST_SEL(8) | 6314 WR_CONFIRM) | 6315 WRITE_DATA_CACHE_POLICY(0)); 6316 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 6317 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 6318 6319 if (resume) 6320 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 6321 sizeof(de_payload) >> 2); 6322 else 6323 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 6324 sizeof(de_payload) >> 2); 6325 } 6326 6327 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 6328 bool secure) 6329 { 6330 uint32_t v = secure ? FRAME_TMZ : 0; 6331 6332 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 6333 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 6334 } 6335 6336 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6337 uint32_t reg_val_offs) 6338 { 6339 struct amdgpu_device *adev = ring->adev; 6340 6341 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6342 amdgpu_ring_write(ring, 0 | /* src: register*/ 6343 (5 << 8) | /* dst: memory */ 6344 (1 << 20)); /* write confirm */ 6345 amdgpu_ring_write(ring, reg); 6346 amdgpu_ring_write(ring, 0); 6347 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6348 reg_val_offs * 4)); 6349 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6350 reg_val_offs * 4)); 6351 } 6352 6353 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6354 uint32_t val) 6355 { 6356 uint32_t cmd = 0; 6357 6358 switch (ring->funcs->type) { 6359 case AMDGPU_RING_TYPE_GFX: 6360 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6361 break; 6362 case AMDGPU_RING_TYPE_KIQ: 6363 cmd = (1 << 16); /* no inc addr */ 6364 break; 6365 default: 6366 cmd = WR_CONFIRM; 6367 break; 6368 } 6369 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6370 amdgpu_ring_write(ring, cmd); 6371 amdgpu_ring_write(ring, reg); 6372 amdgpu_ring_write(ring, 0); 6373 amdgpu_ring_write(ring, val); 6374 } 6375 6376 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 6377 uint32_t val, uint32_t mask) 6378 { 6379 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 6380 } 6381 6382 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 6383 uint32_t reg0, uint32_t reg1, 6384 uint32_t ref, uint32_t mask) 6385 { 6386 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6387 6388 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 6389 ref, mask, 0x20); 6390 } 6391 6392 static void 6393 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6394 uint32_t me, uint32_t pipe, 6395 enum amdgpu_interrupt_state state) 6396 { 6397 uint32_t cp_int_cntl, cp_int_cntl_reg; 6398 6399 if (!me) { 6400 switch (pipe) { 6401 case 0: 6402 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 6403 break; 6404 case 1: 6405 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 6406 break; 6407 default: 6408 DRM_DEBUG("invalid pipe %d\n", pipe); 6409 return; 6410 } 6411 } else { 6412 DRM_DEBUG("invalid me %d\n", me); 6413 return; 6414 } 6415 6416 switch (state) { 6417 case AMDGPU_IRQ_STATE_DISABLE: 6418 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6419 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6420 TIME_STAMP_INT_ENABLE, 0); 6421 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6422 GENERIC0_INT_ENABLE, 0); 6423 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6424 break; 6425 case AMDGPU_IRQ_STATE_ENABLE: 6426 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6427 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6428 TIME_STAMP_INT_ENABLE, 1); 6429 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6430 GENERIC0_INT_ENABLE, 1); 6431 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6432 break; 6433 default: 6434 break; 6435 } 6436 } 6437 6438 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6439 int me, int pipe, 6440 enum amdgpu_interrupt_state state) 6441 { 6442 u32 mec_int_cntl, mec_int_cntl_reg; 6443 6444 /* 6445 * amdgpu controls only the first MEC. That's why this function only 6446 * handles the setting of interrupts for this specific MEC. All other 6447 * pipes' interrupts are set by amdkfd. 6448 */ 6449 6450 if (me == 1) { 6451 switch (pipe) { 6452 case 0: 6453 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6454 break; 6455 case 1: 6456 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 6457 break; 6458 case 2: 6459 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 6460 break; 6461 case 3: 6462 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 6463 break; 6464 default: 6465 DRM_DEBUG("invalid pipe %d\n", pipe); 6466 return; 6467 } 6468 } else { 6469 DRM_DEBUG("invalid me %d\n", me); 6470 return; 6471 } 6472 6473 switch (state) { 6474 case AMDGPU_IRQ_STATE_DISABLE: 6475 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6476 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6477 TIME_STAMP_INT_ENABLE, 0); 6478 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6479 GENERIC0_INT_ENABLE, 0); 6480 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6481 break; 6482 case AMDGPU_IRQ_STATE_ENABLE: 6483 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6484 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6485 TIME_STAMP_INT_ENABLE, 1); 6486 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6487 GENERIC0_INT_ENABLE, 1); 6488 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6489 break; 6490 default: 6491 break; 6492 } 6493 } 6494 6495 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6496 struct amdgpu_irq_src *src, 6497 unsigned type, 6498 enum amdgpu_interrupt_state state) 6499 { 6500 switch (type) { 6501 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6502 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state); 6503 break; 6504 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP: 6505 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state); 6506 break; 6507 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6508 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6509 break; 6510 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6511 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6512 break; 6513 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6514 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6515 break; 6516 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6517 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6518 break; 6519 default: 6520 break; 6521 } 6522 return 0; 6523 } 6524 6525 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, 6526 struct amdgpu_irq_src *source, 6527 struct amdgpu_iv_entry *entry) 6528 { 6529 u32 doorbell_offset = entry->src_data[0]; 6530 6531 DRM_DEBUG("IH: CP EOP\n"); 6532 6533 if (!adev->gfx.disable_kq) { 6534 u8 me_id = (entry->ring_id & 0x0c) >> 2; 6535 u8 pipe_id = (entry->ring_id & 0x03) >> 0; 6536 u8 queue_id = (entry->ring_id & 0x70) >> 4; 6537 struct amdgpu_ring *ring; 6538 int i; 6539 6540 switch (me_id) { 6541 case 0: 6542 /* 6543 * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, 6544 * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). 6545 * Require a strict (me,pipe,queue) match so userq gfx 6546 * EOPs fall through to amdgpu_userq_process_fence_irq(). 6547 */ 6548 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6549 ring = &adev->gfx.gfx_ring[i]; 6550 if ((ring->me == me_id) && 6551 (ring->pipe == pipe_id) && 6552 (ring->queue == queue_id)) { 6553 amdgpu_fence_process(ring); 6554 return 0; 6555 } 6556 } 6557 break; 6558 case 1: 6559 case 2: 6560 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6561 ring = &adev->gfx.compute_ring[i]; 6562 /* Per-queue interrupt is supported for MEC starting from VI. 6563 * The interrupt can only be enabled/disabled per pipe instead 6564 * of per queue. 6565 */ 6566 if ((ring->me == me_id) && 6567 (ring->pipe == pipe_id) && 6568 (ring->queue == queue_id)) { 6569 amdgpu_fence_process(ring); 6570 return 0; 6571 } 6572 } 6573 break; 6574 default: 6575 break; 6576 } 6577 } 6578 6579 if (adev->enable_mes && doorbell_offset) 6580 amdgpu_userq_process_fence_irq(adev, doorbell_offset); 6581 6582 return 0; 6583 } 6584 6585 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6586 struct amdgpu_irq_src *source, 6587 unsigned int type, 6588 enum amdgpu_interrupt_state state) 6589 { 6590 u32 cp_int_cntl_reg, cp_int_cntl; 6591 int i, j; 6592 6593 switch (state) { 6594 case AMDGPU_IRQ_STATE_DISABLE: 6595 case AMDGPU_IRQ_STATE_ENABLE: 6596 for (i = 0; i < adev->gfx.me.num_me; i++) { 6597 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6598 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6599 6600 if (cp_int_cntl_reg) { 6601 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6602 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6603 PRIV_REG_INT_ENABLE, 6604 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6605 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6606 } 6607 } 6608 } 6609 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6610 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6611 /* MECs start at 1 */ 6612 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6613 6614 if (cp_int_cntl_reg) { 6615 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6616 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6617 PRIV_REG_INT_ENABLE, 6618 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6619 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6620 } 6621 } 6622 } 6623 break; 6624 default: 6625 break; 6626 } 6627 6628 return 0; 6629 } 6630 6631 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev, 6632 struct amdgpu_irq_src *source, 6633 unsigned type, 6634 enum amdgpu_interrupt_state state) 6635 { 6636 u32 cp_int_cntl_reg, cp_int_cntl; 6637 int i, j; 6638 6639 switch (state) { 6640 case AMDGPU_IRQ_STATE_DISABLE: 6641 case AMDGPU_IRQ_STATE_ENABLE: 6642 for (i = 0; i < adev->gfx.me.num_me; i++) { 6643 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6644 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6645 6646 if (cp_int_cntl_reg) { 6647 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6648 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6649 OPCODE_ERROR_INT_ENABLE, 6650 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6651 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6652 } 6653 } 6654 } 6655 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6656 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6657 /* MECs start at 1 */ 6658 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6659 6660 if (cp_int_cntl_reg) { 6661 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6662 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6663 OPCODE_ERROR_INT_ENABLE, 6664 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6665 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6666 } 6667 } 6668 } 6669 break; 6670 default: 6671 break; 6672 } 6673 return 0; 6674 } 6675 6676 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6677 struct amdgpu_irq_src *source, 6678 unsigned int type, 6679 enum amdgpu_interrupt_state state) 6680 { 6681 u32 cp_int_cntl_reg, cp_int_cntl; 6682 int i, j; 6683 6684 switch (state) { 6685 case AMDGPU_IRQ_STATE_DISABLE: 6686 case AMDGPU_IRQ_STATE_ENABLE: 6687 for (i = 0; i < adev->gfx.me.num_me; i++) { 6688 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6689 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6690 6691 if (cp_int_cntl_reg) { 6692 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6693 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6694 PRIV_INSTR_INT_ENABLE, 6695 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6696 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6697 } 6698 } 6699 } 6700 break; 6701 default: 6702 break; 6703 } 6704 6705 return 0; 6706 } 6707 6708 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, 6709 struct amdgpu_iv_entry *entry) 6710 { 6711 u8 me_id, pipe_id, queue_id; 6712 struct amdgpu_ring *ring; 6713 int i; 6714 6715 me_id = (entry->ring_id & 0x0c) >> 2; 6716 pipe_id = (entry->ring_id & 0x03) >> 0; 6717 queue_id = (entry->ring_id & 0x70) >> 4; 6718 6719 if (!adev->gfx.disable_kq) { 6720 switch (me_id) { 6721 case 0: 6722 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6723 ring = &adev->gfx.gfx_ring[i]; 6724 if (ring->me == me_id && ring->pipe == pipe_id && 6725 ring->queue == queue_id) 6726 drm_sched_fault(&ring->sched); 6727 } 6728 break; 6729 case 1: 6730 case 2: 6731 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6732 ring = &adev->gfx.compute_ring[i]; 6733 if (ring->me == me_id && ring->pipe == pipe_id && 6734 ring->queue == queue_id) 6735 drm_sched_fault(&ring->sched); 6736 } 6737 break; 6738 default: 6739 BUG(); 6740 break; 6741 } 6742 } 6743 } 6744 6745 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev, 6746 struct amdgpu_irq_src *source, 6747 struct amdgpu_iv_entry *entry) 6748 { 6749 DRM_ERROR("Illegal register access in command stream\n"); 6750 gfx_v11_0_handle_priv_fault(adev, entry); 6751 return 0; 6752 } 6753 6754 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev, 6755 struct amdgpu_irq_src *source, 6756 struct amdgpu_iv_entry *entry) 6757 { 6758 DRM_ERROR("Illegal opcode in command stream\n"); 6759 gfx_v11_0_handle_priv_fault(adev, entry); 6760 return 0; 6761 } 6762 6763 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev, 6764 struct amdgpu_irq_src *source, 6765 struct amdgpu_iv_entry *entry) 6766 { 6767 DRM_ERROR("Illegal instruction in command stream\n"); 6768 gfx_v11_0_handle_priv_fault(adev, entry); 6769 return 0; 6770 } 6771 6772 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev, 6773 struct amdgpu_irq_src *source, 6774 struct amdgpu_iv_entry *entry) 6775 { 6776 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq) 6777 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry); 6778 6779 return 0; 6780 } 6781 6782 #if 0 6783 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev, 6784 struct amdgpu_irq_src *src, 6785 unsigned int type, 6786 enum amdgpu_interrupt_state state) 6787 { 6788 uint32_t tmp, target; 6789 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring); 6790 6791 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6792 target += ring->pipe; 6793 6794 switch (type) { 6795 case AMDGPU_CP_KIQ_IRQ_DRIVER0: 6796 if (state == AMDGPU_IRQ_STATE_DISABLE) { 6797 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6798 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6799 GENERIC2_INT_ENABLE, 0); 6800 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6801 6802 tmp = RREG32_SOC15_IP(GC, target); 6803 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6804 GENERIC2_INT_ENABLE, 0); 6805 WREG32_SOC15_IP(GC, target, tmp); 6806 } else { 6807 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6808 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6809 GENERIC2_INT_ENABLE, 1); 6810 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6811 6812 tmp = RREG32_SOC15_IP(GC, target); 6813 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6814 GENERIC2_INT_ENABLE, 1); 6815 WREG32_SOC15_IP(GC, target, tmp); 6816 } 6817 break; 6818 default: 6819 BUG(); /* kiq only support GENERIC2_INT now */ 6820 break; 6821 } 6822 return 0; 6823 } 6824 #endif 6825 6826 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring) 6827 { 6828 const unsigned int gcr_cntl = 6829 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) | 6830 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) | 6831 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) | 6832 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) | 6833 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) | 6834 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) | 6835 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) | 6836 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1); 6837 6838 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ 6839 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6)); 6840 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */ 6841 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6842 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6843 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6844 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6845 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6846 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ 6847 } 6848 6849 static bool gfx_v11_pipe_reset_support(struct amdgpu_device *adev) 6850 { 6851 /* Disable the pipe reset until the CPFW fully support it.*/ 6852 dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n"); 6853 return false; 6854 } 6855 6856 6857 static int gfx_v11_reset_gfx_pipe(struct amdgpu_ring *ring) 6858 { 6859 struct amdgpu_device *adev = ring->adev; 6860 uint32_t reset_pipe = 0, clean_pipe = 0; 6861 int r; 6862 6863 if (!gfx_v11_pipe_reset_support(adev)) 6864 return -EOPNOTSUPP; 6865 6866 gfx_v11_0_set_safe_mode(adev, 0); 6867 mutex_lock(&adev->srbm_mutex); 6868 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 6869 6870 switch (ring->pipe) { 6871 case 0: 6872 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, 6873 PFP_PIPE0_RESET, 1); 6874 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, 6875 ME_PIPE0_RESET, 1); 6876 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, 6877 PFP_PIPE0_RESET, 0); 6878 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, 6879 ME_PIPE0_RESET, 0); 6880 break; 6881 case 1: 6882 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, 6883 PFP_PIPE1_RESET, 1); 6884 reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, 6885 ME_PIPE1_RESET, 1); 6886 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, 6887 PFP_PIPE1_RESET, 0); 6888 clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, 6889 ME_PIPE1_RESET, 0); 6890 break; 6891 default: 6892 break; 6893 } 6894 6895 WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe); 6896 WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe); 6897 6898 r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) - 6899 RS64_FW_UC_START_ADDR_LO; 6900 soc21_grbm_select(adev, 0, 0, 0, 0); 6901 mutex_unlock(&adev->srbm_mutex); 6902 gfx_v11_0_unset_safe_mode(adev, 0); 6903 6904 dev_info(adev->dev, "The ring %s pipe reset to the ME firmware start PC: %s\n", ring->name, 6905 r == 0 ? "successfully" : "failed"); 6906 /* FIXME: Sometimes driver can't cache the ME firmware start PC correctly, 6907 * so the pipe reset status relies on the later gfx ring test result. 6908 */ 6909 return 0; 6910 } 6911 6912 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, 6913 unsigned int vmid, 6914 struct amdgpu_fence *timedout_fence) 6915 { 6916 struct amdgpu_device *adev = ring->adev; 6917 bool use_mmio = false; 6918 int r; 6919 6920 amdgpu_ring_reset_helper_begin(ring, timedout_fence); 6921 6922 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0); 6923 if (r) { 6924 6925 dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r); 6926 r = gfx_v11_reset_gfx_pipe(ring); 6927 if (r) 6928 return r; 6929 } 6930 6931 if (use_mmio) { 6932 r = gfx_v11_0_kgq_init_queue(ring, true); 6933 if (r) { 6934 dev_err(adev->dev, "failed to init kgq\n"); 6935 return r; 6936 } 6937 6938 r = amdgpu_mes_map_legacy_queue(adev, ring, 0); 6939 if (r) { 6940 dev_err(adev->dev, "failed to remap kgq\n"); 6941 return r; 6942 } 6943 } 6944 6945 return amdgpu_ring_reset_helper_end(ring, timedout_fence); 6946 } 6947 6948 static int gfx_v11_0_reset_compute_pipe(struct amdgpu_ring *ring) 6949 { 6950 6951 struct amdgpu_device *adev = ring->adev; 6952 uint32_t reset_pipe = 0, clean_pipe = 0; 6953 int r; 6954 6955 if (!gfx_v11_pipe_reset_support(adev)) 6956 return -EOPNOTSUPP; 6957 6958 gfx_v11_0_set_safe_mode(adev, 0); 6959 mutex_lock(&adev->srbm_mutex); 6960 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 6961 6962 reset_pipe = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 6963 clean_pipe = reset_pipe; 6964 6965 if (adev->gfx.rs64_enable) { 6966 6967 switch (ring->pipe) { 6968 case 0: 6969 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, 6970 MEC_PIPE0_RESET, 1); 6971 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, 6972 MEC_PIPE0_RESET, 0); 6973 break; 6974 case 1: 6975 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, 6976 MEC_PIPE1_RESET, 1); 6977 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, 6978 MEC_PIPE1_RESET, 0); 6979 break; 6980 case 2: 6981 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, 6982 MEC_PIPE2_RESET, 1); 6983 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, 6984 MEC_PIPE2_RESET, 0); 6985 break; 6986 case 3: 6987 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, 6988 MEC_PIPE3_RESET, 1); 6989 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, 6990 MEC_PIPE3_RESET, 0); 6991 break; 6992 default: 6993 break; 6994 } 6995 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_pipe); 6996 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_pipe); 6997 r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) - 6998 RS64_FW_UC_START_ADDR_LO; 6999 } else { 7000 if (ring->me == 1) { 7001 switch (ring->pipe) { 7002 case 0: 7003 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7004 MEC_ME1_PIPE0_RESET, 1); 7005 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7006 MEC_ME1_PIPE0_RESET, 0); 7007 break; 7008 case 1: 7009 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7010 MEC_ME1_PIPE1_RESET, 1); 7011 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7012 MEC_ME1_PIPE1_RESET, 0); 7013 break; 7014 case 2: 7015 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7016 MEC_ME1_PIPE2_RESET, 1); 7017 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7018 MEC_ME1_PIPE2_RESET, 0); 7019 break; 7020 case 3: 7021 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7022 MEC_ME1_PIPE3_RESET, 1); 7023 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7024 MEC_ME1_PIPE3_RESET, 0); 7025 break; 7026 default: 7027 break; 7028 } 7029 /* mec1 fw pc: CP_MEC1_INSTR_PNTR */ 7030 } else { 7031 switch (ring->pipe) { 7032 case 0: 7033 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7034 MEC_ME2_PIPE0_RESET, 1); 7035 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7036 MEC_ME2_PIPE0_RESET, 0); 7037 break; 7038 case 1: 7039 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7040 MEC_ME2_PIPE1_RESET, 1); 7041 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7042 MEC_ME2_PIPE1_RESET, 0); 7043 break; 7044 case 2: 7045 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7046 MEC_ME2_PIPE2_RESET, 1); 7047 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7048 MEC_ME2_PIPE2_RESET, 0); 7049 break; 7050 case 3: 7051 reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, 7052 MEC_ME2_PIPE3_RESET, 1); 7053 clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, 7054 MEC_ME2_PIPE3_RESET, 0); 7055 break; 7056 default: 7057 break; 7058 } 7059 /* mec2 fw pc: CP:CP_MEC2_INSTR_PNTR */ 7060 } 7061 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_pipe); 7062 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_pipe); 7063 r = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_MEC1_INSTR_PNTR)); 7064 } 7065 7066 soc21_grbm_select(adev, 0, 0, 0, 0); 7067 mutex_unlock(&adev->srbm_mutex); 7068 gfx_v11_0_unset_safe_mode(adev, 0); 7069 7070 dev_info(adev->dev, "The ring %s pipe resets to MEC FW start PC: %s\n", ring->name, 7071 r == 0 ? "successfully" : "failed"); 7072 /*FIXME:Sometimes driver can't cache the MEC firmware start PC correctly, so the pipe 7073 * reset status relies on the compute ring test result. 7074 */ 7075 return 0; 7076 } 7077 7078 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, 7079 unsigned int vmid, 7080 struct amdgpu_fence *timedout_fence) 7081 { 7082 struct amdgpu_device *adev = ring->adev; 7083 int r = 0; 7084 7085 amdgpu_ring_reset_helper_begin(ring, timedout_fence); 7086 7087 r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true, 0); 7088 if (r) { 7089 dev_warn(adev->dev, "fail(%d) to reset kcq and try pipe reset\n", r); 7090 r = gfx_v11_0_reset_compute_pipe(ring); 7091 if (r) 7092 return r; 7093 } 7094 7095 r = gfx_v11_0_kcq_init_queue(ring, true); 7096 if (r) { 7097 dev_err(adev->dev, "fail to init kcq\n"); 7098 return r; 7099 } 7100 r = amdgpu_mes_map_legacy_queue(adev, ring, 0); 7101 if (r) { 7102 dev_err(adev->dev, "failed to remap kcq\n"); 7103 return r; 7104 } 7105 7106 return amdgpu_ring_reset_helper_end(ring, timedout_fence); 7107 } 7108 7109 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 7110 { 7111 struct amdgpu_device *adev = ip_block->adev; 7112 uint32_t i, j, k, reg, index = 0; 7113 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 7114 7115 if (!adev->gfx.ip_dump_core) 7116 return; 7117 7118 for (i = 0; i < reg_count; i++) 7119 drm_printf(p, "%-50s \t 0x%08x\n", 7120 gc_reg_list_11_0[i].reg_name, 7121 adev->gfx.ip_dump_core[i]); 7122 7123 /* print compute queue registers for all instances */ 7124 if (!adev->gfx.ip_dump_compute_queues) 7125 return; 7126 7127 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 7128 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n", 7129 adev->gfx.mec.num_mec, 7130 adev->gfx.mec.num_pipe_per_mec, 7131 adev->gfx.mec.num_queue_per_pipe); 7132 7133 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 7134 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 7135 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 7136 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k); 7137 for (reg = 0; reg < reg_count; reg++) { 7138 if (i && gc_cp_reg_list_11[reg].reg_offset == regCP_MEC_ME1_HEADER_DUMP) 7139 drm_printf(p, "%-50s \t 0x%08x\n", 7140 "regCP_MEC_ME2_HEADER_DUMP", 7141 adev->gfx.ip_dump_compute_queues[index + reg]); 7142 else 7143 drm_printf(p, "%-50s \t 0x%08x\n", 7144 gc_cp_reg_list_11[reg].reg_name, 7145 adev->gfx.ip_dump_compute_queues[index + reg]); 7146 } 7147 index += reg_count; 7148 } 7149 } 7150 } 7151 7152 /* print gfx queue registers for all instances */ 7153 if (!adev->gfx.ip_dump_gfx_queues) 7154 return; 7155 7156 index = 0; 7157 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 7158 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n", 7159 adev->gfx.me.num_me, 7160 adev->gfx.me.num_pipe_per_me, 7161 adev->gfx.me.num_queue_per_pipe); 7162 7163 for (i = 0; i < adev->gfx.me.num_me; i++) { 7164 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 7165 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 7166 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k); 7167 for (reg = 0; reg < reg_count; reg++) { 7168 drm_printf(p, "%-50s \t 0x%08x\n", 7169 gc_gfx_queue_reg_list_11[reg].reg_name, 7170 adev->gfx.ip_dump_gfx_queues[index + reg]); 7171 } 7172 index += reg_count; 7173 } 7174 } 7175 } 7176 } 7177 7178 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block) 7179 { 7180 struct amdgpu_device *adev = ip_block->adev; 7181 uint32_t i, j, k, reg, index = 0; 7182 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 7183 7184 if (!adev->gfx.ip_dump_core) 7185 return; 7186 7187 amdgpu_gfx_off_ctrl(adev, false); 7188 for (i = 0; i < reg_count; i++) 7189 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i])); 7190 amdgpu_gfx_off_ctrl(adev, true); 7191 7192 /* dump compute queue registers for all instances */ 7193 if (!adev->gfx.ip_dump_compute_queues) 7194 return; 7195 7196 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 7197 amdgpu_gfx_off_ctrl(adev, false); 7198 mutex_lock(&adev->srbm_mutex); 7199 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 7200 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 7201 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 7202 /* ME0 is for GFX so start from 1 for CP */ 7203 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0); 7204 for (reg = 0; reg < reg_count; reg++) { 7205 if (i && 7206 gc_cp_reg_list_11[reg].reg_offset == 7207 regCP_MEC_ME1_HEADER_DUMP) 7208 adev->gfx.ip_dump_compute_queues[index + reg] = 7209 RREG32(SOC15_REG_OFFSET(GC, 0, 7210 regCP_MEC_ME2_HEADER_DUMP)); 7211 else 7212 adev->gfx.ip_dump_compute_queues[index + reg] = 7213 RREG32(SOC15_REG_ENTRY_OFFSET( 7214 gc_cp_reg_list_11[reg])); 7215 } 7216 index += reg_count; 7217 } 7218 } 7219 } 7220 soc21_grbm_select(adev, 0, 0, 0, 0); 7221 mutex_unlock(&adev->srbm_mutex); 7222 amdgpu_gfx_off_ctrl(adev, true); 7223 7224 /* dump gfx queue registers for all instances */ 7225 if (!adev->gfx.ip_dump_gfx_queues) 7226 return; 7227 7228 index = 0; 7229 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 7230 amdgpu_gfx_off_ctrl(adev, false); 7231 mutex_lock(&adev->srbm_mutex); 7232 for (i = 0; i < adev->gfx.me.num_me; i++) { 7233 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 7234 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 7235 soc21_grbm_select(adev, i, j, k, 0); 7236 7237 for (reg = 0; reg < reg_count; reg++) { 7238 adev->gfx.ip_dump_gfx_queues[index + reg] = 7239 RREG32(SOC15_REG_ENTRY_OFFSET( 7240 gc_gfx_queue_reg_list_11[reg])); 7241 } 7242 index += reg_count; 7243 } 7244 } 7245 } 7246 soc21_grbm_select(adev, 0, 0, 0, 0); 7247 mutex_unlock(&adev->srbm_mutex); 7248 amdgpu_gfx_off_ctrl(adev, true); 7249 } 7250 7251 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring) 7252 { 7253 /* Emit the cleaner shader */ 7254 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0)); 7255 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */ 7256 } 7257 7258 static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring) 7259 { 7260 amdgpu_gfx_profile_ring_begin_use(ring); 7261 7262 amdgpu_gfx_enforce_isolation_ring_begin_use(ring); 7263 } 7264 7265 static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring) 7266 { 7267 amdgpu_gfx_profile_ring_end_use(ring); 7268 7269 amdgpu_gfx_enforce_isolation_ring_end_use(ring); 7270 } 7271 7272 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { 7273 .name = "gfx_v11_0", 7274 .early_init = gfx_v11_0_early_init, 7275 .sw_init = gfx_v11_0_sw_init, 7276 .sw_fini = gfx_v11_0_sw_fini, 7277 .hw_init = gfx_v11_0_hw_init, 7278 .hw_fini = gfx_v11_0_hw_fini, 7279 .suspend = gfx_v11_0_suspend, 7280 .resume = gfx_v11_0_resume, 7281 .is_idle = gfx_v11_0_is_idle, 7282 .wait_for_idle = gfx_v11_0_wait_for_idle, 7283 .soft_reset = gfx_v11_0_soft_reset, 7284 .check_soft_reset = gfx_v11_0_check_soft_reset, 7285 .post_soft_reset = gfx_v11_0_post_soft_reset, 7286 .set_clockgating_state = gfx_v11_0_set_clockgating_state, 7287 .set_powergating_state = gfx_v11_0_set_powergating_state, 7288 .get_clockgating_state = gfx_v11_0_get_clockgating_state, 7289 .dump_ip_state = gfx_v11_ip_dump, 7290 .print_ip_state = gfx_v11_ip_print, 7291 }; 7292 7293 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = { 7294 .type = AMDGPU_RING_TYPE_GFX, 7295 .align_mask = 0xff, 7296 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7297 .support_64bit_ptrs = true, 7298 .secure_submission_supported = true, 7299 .get_rptr = gfx_v11_0_ring_get_rptr_gfx, 7300 .get_wptr = gfx_v11_0_ring_get_wptr_gfx, 7301 .set_wptr = gfx_v11_0_ring_set_wptr_gfx, 7302 .emit_frame_size = /* totally 247 maximum if 16 IBs */ 7303 5 + /* update_spm_vmid */ 7304 5 + /* COND_EXEC */ 7305 22 + /* SET_Q_PREEMPTION_MODE */ 7306 7 + /* PIPELINE_SYNC */ 7307 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7308 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7309 4 + /* VM_FLUSH */ 7310 8 + /* FENCE for VM_FLUSH */ 7311 20 + /* GDS switch */ 7312 5 + /* COND_EXEC */ 7313 7 + /* HDP_flush */ 7314 4 + /* VGT_flush */ 7315 31 + /* DE_META */ 7316 3 + /* CNTX_CTRL */ 7317 5 + /* HDP_INVL */ 7318 22 + /* SET_Q_PREEMPTION_MODE */ 7319 8 + 8 + /* FENCE x2 */ 7320 8 + /* gfx_v11_0_emit_mem_sync */ 7321 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7322 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */ 7323 .emit_ib = gfx_v11_0_ring_emit_ib_gfx, 7324 .emit_fence = gfx_v11_0_ring_emit_fence, 7325 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7326 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7327 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7328 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7329 .test_ring = gfx_v11_0_ring_test_ring, 7330 .test_ib = gfx_v11_0_ring_test_ib, 7331 .insert_nop = gfx_v11_ring_insert_nop, 7332 .pad_ib = amdgpu_ring_generic_pad_ib, 7333 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl, 7334 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow, 7335 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec, 7336 .preempt_ib = amdgpu_gfx_ring_preempt_ib, 7337 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl, 7338 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7339 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7340 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7341 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7342 .reset = gfx_v11_0_reset_kgq, 7343 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7344 .begin_use = gfx_v11_0_ring_begin_use, 7345 .end_use = gfx_v11_0_ring_end_use, 7346 }; 7347 7348 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = { 7349 .type = AMDGPU_RING_TYPE_COMPUTE, 7350 .align_mask = 0xff, 7351 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7352 .support_64bit_ptrs = true, 7353 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7354 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7355 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7356 .emit_frame_size = 7357 5 + /* update_spm_vmid */ 7358 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7359 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7360 5 + /* hdp invalidate */ 7361 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7362 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7363 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7364 2 + /* gfx_v11_0_ring_emit_vm_flush */ 7365 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */ 7366 8 + /* gfx_v11_0_emit_mem_sync */ 7367 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7368 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7369 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7370 .emit_fence = gfx_v11_0_ring_emit_fence, 7371 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7372 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7373 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7374 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7375 .test_ring = gfx_v11_0_ring_test_ring, 7376 .test_ib = gfx_v11_0_ring_test_ib, 7377 .insert_nop = gfx_v11_ring_insert_nop, 7378 .pad_ib = amdgpu_ring_generic_pad_ib, 7379 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7380 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7381 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7382 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7383 .reset = gfx_v11_0_reset_kcq, 7384 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7385 .begin_use = gfx_v11_0_ring_begin_use, 7386 .end_use = gfx_v11_0_ring_end_use, 7387 }; 7388 7389 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = { 7390 .type = AMDGPU_RING_TYPE_KIQ, 7391 .align_mask = 0xff, 7392 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7393 .support_64bit_ptrs = true, 7394 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7395 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7396 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7397 .emit_frame_size = 7398 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7399 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7400 5 + /*hdp invalidate */ 7401 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7402 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7403 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7404 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7405 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7406 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7407 .emit_fence = gfx_v11_0_ring_emit_fence_kiq, 7408 .test_ring = gfx_v11_0_ring_test_ring, 7409 .test_ib = gfx_v11_0_ring_test_ib, 7410 .insert_nop = amdgpu_ring_insert_nop, 7411 .pad_ib = amdgpu_ring_generic_pad_ib, 7412 .emit_rreg = gfx_v11_0_ring_emit_rreg, 7413 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7414 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7415 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7416 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7417 }; 7418 7419 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev) 7420 { 7421 int i; 7422 7423 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq; 7424 7425 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7426 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx; 7427 7428 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7429 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute; 7430 } 7431 7432 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = { 7433 .set = gfx_v11_0_set_eop_interrupt_state, 7434 .process = gfx_v11_0_eop_irq, 7435 }; 7436 7437 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = { 7438 .set = gfx_v11_0_set_priv_reg_fault_state, 7439 .process = gfx_v11_0_priv_reg_irq, 7440 }; 7441 7442 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = { 7443 .set = gfx_v11_0_set_bad_op_fault_state, 7444 .process = gfx_v11_0_bad_op_irq, 7445 }; 7446 7447 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = { 7448 .set = gfx_v11_0_set_priv_inst_fault_state, 7449 .process = gfx_v11_0_priv_inst_irq, 7450 }; 7451 7452 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = { 7453 .process = gfx_v11_0_rlc_gc_fed_irq, 7454 }; 7455 7456 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev) 7457 { 7458 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7459 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs; 7460 7461 adev->gfx.priv_reg_irq.num_types = 1; 7462 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs; 7463 7464 adev->gfx.bad_op_irq.num_types = 1; 7465 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs; 7466 7467 adev->gfx.priv_inst_irq.num_types = 1; 7468 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs; 7469 7470 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */ 7471 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs; 7472 7473 } 7474 7475 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev) 7476 { 7477 if (adev->flags & AMD_IS_APU) 7478 adev->gfx.imu.mode = MISSION_MODE; 7479 else 7480 adev->gfx.imu.mode = DEBUG_MODE; 7481 7482 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs; 7483 } 7484 7485 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev) 7486 { 7487 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs; 7488 } 7489 7490 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev) 7491 { 7492 unsigned total_cu = adev->gfx.config.max_cu_per_sh * 7493 adev->gfx.config.max_sh_per_se * 7494 adev->gfx.config.max_shader_engines; 7495 7496 adev->gds.gds_size = 0x1000; 7497 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1; 7498 adev->gds.gws_size = 64; 7499 adev->gds.oa_size = 16; 7500 } 7501 7502 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev) 7503 { 7504 /* set gfx eng mqd */ 7505 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size = 7506 sizeof(struct v11_gfx_mqd); 7507 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd = 7508 gfx_v11_0_gfx_mqd_init; 7509 /* set compute eng mqd */ 7510 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size = 7511 sizeof(struct v11_compute_mqd); 7512 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd = 7513 gfx_v11_0_compute_mqd_init; 7514 } 7515 7516 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev, 7517 u32 bitmap) 7518 { 7519 u32 data; 7520 7521 if (!bitmap) 7522 return; 7523 7524 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7525 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7526 7527 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data); 7528 } 7529 7530 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev) 7531 { 7532 u32 data, wgp_bitmask; 7533 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG); 7534 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG); 7535 7536 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7537 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7538 7539 wgp_bitmask = 7540 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1); 7541 7542 return (~data) & wgp_bitmask; 7543 } 7544 7545 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev) 7546 { 7547 u32 wgp_idx, wgp_active_bitmap; 7548 u32 cu_bitmap_per_wgp, cu_active_bitmap; 7549 7550 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev); 7551 cu_active_bitmap = 0; 7552 7553 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) { 7554 /* if there is one WGP enabled, it means 2 CUs will be enabled */ 7555 cu_bitmap_per_wgp = 3 << (2 * wgp_idx); 7556 if (wgp_active_bitmap & (1 << wgp_idx)) 7557 cu_active_bitmap |= cu_bitmap_per_wgp; 7558 } 7559 7560 return cu_active_bitmap; 7561 } 7562 7563 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 7564 struct amdgpu_cu_info *cu_info) 7565 { 7566 int i, j, k, counter, active_cu_number = 0; 7567 u32 mask, bitmap; 7568 unsigned disable_masks[8 * 2]; 7569 7570 if (!adev || !cu_info) 7571 return -EINVAL; 7572 7573 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 8, 2); 7574 7575 mutex_lock(&adev->grbm_idx_mutex); 7576 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7577 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7578 bitmap = i * adev->gfx.config.max_sh_per_se + j; 7579 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1)) 7580 continue; 7581 mask = 1; 7582 counter = 0; 7583 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7584 if (i < 8 && j < 2) 7585 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh( 7586 adev, disable_masks[i * 2 + j]); 7587 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev); 7588 7589 /** 7590 * GFX11 could support more than 4 SEs, while the bitmap 7591 * in cu_info struct is 4x4 and ioctl interface struct 7592 * drm_amdgpu_info_device should keep stable. 7593 * So we use last two columns of bitmap to store cu mask for 7594 * SEs 4 to 7, the layout of the bitmap is as below: 7595 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]} 7596 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]} 7597 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]} 7598 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]} 7599 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]} 7600 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]} 7601 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]} 7602 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]} 7603 */ 7604 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap; 7605 7606 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { 7607 if (bitmap & mask) 7608 counter++; 7609 7610 mask <<= 1; 7611 } 7612 active_cu_number += counter; 7613 } 7614 } 7615 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7616 mutex_unlock(&adev->grbm_idx_mutex); 7617 7618 cu_info->number = active_cu_number; 7619 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7620 7621 return 0; 7622 } 7623 7624 const struct amdgpu_ip_block_version gfx_v11_0_ip_block = 7625 { 7626 .type = AMD_IP_BLOCK_TYPE_GFX, 7627 .major = 11, 7628 .minor = 0, 7629 .rev = 0, 7630 .funcs = &gfx_v11_0_ip_funcs, 7631 }; 7632