1 /* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/delay.h> 24 #include <linux/kernel.h> 25 #include <linux/firmware.h> 26 #include <linux/module.h> 27 #include <linux/pci.h> 28 #include "amdgpu.h" 29 #include "amdgpu_gfx.h" 30 #include "amdgpu_psp.h" 31 #include "amdgpu_smu.h" 32 #include "imu_v11_0.h" 33 #include "soc21.h" 34 #include "nvd.h" 35 36 #include "gc/gc_11_0_0_offset.h" 37 #include "gc/gc_11_0_0_sh_mask.h" 38 #include "smuio/smuio_13_0_6_offset.h" 39 #include "smuio/smuio_13_0_6_sh_mask.h" 40 #include "navi10_enum.h" 41 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h" 42 43 #include "soc15.h" 44 #include "clearstate_gfx11.h" 45 #include "v11_structs.h" 46 #include "gfx_v11_0.h" 47 #include "gfx_v11_0_cleaner_shader.h" 48 #include "gfx_v11_0_3.h" 49 #include "nbio_v4_3.h" 50 #include "mes_v11_0.h" 51 #include "mes_userqueue.h" 52 #include "amdgpu_userq_fence.h" 53 54 #define GFX11_NUM_GFX_RINGS 2 55 #define GFX11_MEC_HPD_SIZE 2048 56 57 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 58 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388 59 60 #define regCGTT_WD_CLK_CTRL 0x5086 61 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1 62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e 63 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1 64 #define regPC_CONFIG_CNTL_1 0x194d 65 #define regPC_CONFIG_CNTL_1_BASE_IDX 1 66 67 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0 0x0030 68 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0_BASE_IDX 1 69 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0 0x0031 70 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0_BASE_IDX 1 71 72 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100 73 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000 74 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000 75 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01 76 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000 77 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000 78 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000 79 80 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006 81 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 82 #define regCP_MQD_CONTROL_DEFAULT 0x00000100 83 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509 84 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 85 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000 86 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501 87 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000 88 89 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin"); 90 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin"); 91 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin"); 92 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin"); 93 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_kicker.bin"); 94 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin"); 95 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin"); 96 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin"); 97 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin"); 98 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin"); 99 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin"); 101 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin"); 102 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin"); 103 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin"); 104 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin"); 105 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin"); 106 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin"); 107 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin"); 109 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin"); 110 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin"); 112 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin"); 113 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin"); 114 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin"); 115 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin"); 116 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin"); 117 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin"); 118 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin"); 119 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin"); 120 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin"); 122 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin"); 124 MODULE_FIRMWARE("amdgpu/gc_11_5_3_pfp.bin"); 125 MODULE_FIRMWARE("amdgpu/gc_11_5_3_me.bin"); 126 MODULE_FIRMWARE("amdgpu/gc_11_5_3_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/gc_11_5_3_rlc.bin"); 128 MODULE_FIRMWARE("amdgpu/gc_11_5_4_pfp.bin"); 129 MODULE_FIRMWARE("amdgpu/gc_11_5_4_me.bin"); 130 MODULE_FIRMWARE("amdgpu/gc_11_5_4_mec.bin"); 131 MODULE_FIRMWARE("amdgpu/gc_11_5_4_rlc.bin"); 132 MODULE_FIRMWARE("amdgpu/gc_11_5_6_pfp.bin"); 133 MODULE_FIRMWARE("amdgpu/gc_11_5_6_me.bin"); 134 MODULE_FIRMWARE("amdgpu/gc_11_5_6_mec.bin"); 135 MODULE_FIRMWARE("amdgpu/gc_11_5_6_rlc.bin"); 136 MODULE_FIRMWARE("amdgpu/gc_11_7_0_pfp.bin"); 137 MODULE_FIRMWARE("amdgpu/gc_11_7_0_me.bin"); 138 MODULE_FIRMWARE("amdgpu/gc_11_7_0_mec.bin"); 139 MODULE_FIRMWARE("amdgpu/gc_11_7_0_rlc.bin"); 140 MODULE_FIRMWARE("amdgpu/gc_11_7_1_pfp.bin"); 141 MODULE_FIRMWARE("amdgpu/gc_11_7_1_me.bin"); 142 MODULE_FIRMWARE("amdgpu/gc_11_7_1_mec.bin"); 143 MODULE_FIRMWARE("amdgpu/gc_11_7_1_rlc.bin"); 144 145 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = { 146 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), 147 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2), 148 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3), 149 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1), 150 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2), 151 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3), 152 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1), 153 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1), 154 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT), 155 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT), 156 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT), 157 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2), 158 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2), 159 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS), 160 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR), 161 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0), 162 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE), 163 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 164 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR), 165 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE), 166 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR), 167 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR), 168 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE), 169 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR), 170 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR), 171 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 172 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ), 173 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 174 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 175 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 176 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO), 177 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI), 178 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ), 179 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS), 180 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS), 181 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS), 182 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT), 183 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT), 184 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS), 185 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2), 186 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS), 187 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS), 188 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS), 189 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES), 190 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS), 191 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS), 192 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL), 193 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS), 194 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG), 195 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL), 196 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL), 197 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR), 198 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR), 199 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR), 200 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR), 201 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR), 202 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS), 203 /* cp header registers */ 204 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 205 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 206 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 207 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 208 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 209 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 210 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 211 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 212 /* SE status registers */ 213 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0), 214 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1), 215 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2), 216 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3), 217 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4), 218 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5) 219 }; 220 221 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = { 222 /* compute registers */ 223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID), 224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE), 225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY), 226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY), 227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM), 228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE), 229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI), 230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR), 231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 232 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 233 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), 234 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL), 235 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR), 236 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI), 237 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR), 238 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL), 239 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST), 240 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR), 241 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI), 242 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL), 243 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR), 244 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR), 245 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS), 246 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO), 247 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI), 248 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL), 249 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET), 250 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE), 251 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET), 252 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE), 253 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE), 254 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR), 255 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM), 256 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO), 257 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI), 258 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET), 259 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT), 260 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET), 261 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS), 262 /* cp header registers */ 263 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 264 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 265 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 266 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 267 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 268 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 269 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 270 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 271 }; 272 273 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = { 274 /* gfx queue registers */ 275 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE), 276 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID), 277 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY), 278 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM), 279 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE), 280 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI), 281 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET), 282 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL), 283 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR), 284 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR), 285 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI), 286 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST), 287 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED), 288 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL), 289 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0), 290 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0), 291 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR), 292 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI), 293 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO), 294 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI), 295 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 296 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 297 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 298 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 299 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 300 /* cp header registers */ 301 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 302 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 303 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 304 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 305 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 306 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 307 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 308 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 309 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 310 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 311 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 312 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 313 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 314 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 315 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 316 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 317 }; 318 319 static const struct soc15_reg_golden golden_settings_gc_11_0[] = { 320 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000) 321 }; 322 323 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] = 324 { 325 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010), 326 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010), 327 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200), 328 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988), 329 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007), 330 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008), 331 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100), 332 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000), 333 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a) 334 }; 335 336 #define DEFAULT_SH_MEM_CONFIG \ 337 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \ 338 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \ 339 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT)) 340 341 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev); 342 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev); 343 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev); 344 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev); 345 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev); 346 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev); 347 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev); 348 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 349 struct amdgpu_cu_info *cu_info); 350 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev); 351 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 352 u32 sh_num, u32 instance, int xcc_id); 353 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev); 354 355 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume); 356 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); 357 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 358 uint32_t val); 359 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); 360 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 361 uint16_t pasid, uint32_t flush_type, 362 bool all_hub, uint8_t dst_sel); 363 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 364 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 365 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 366 bool enable); 367 368 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask) 369 { 370 struct amdgpu_device *adev = kiq_ring->adev; 371 u64 shader_mc_addr; 372 373 /* Cleaner shader MC address */ 374 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; 375 376 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 377 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | 378 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */ 379 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ 380 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 381 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 382 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */ 383 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */ 384 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 385 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 386 } 387 388 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring, 389 struct amdgpu_ring *ring) 390 { 391 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 392 uint64_t wptr_addr = ring->wptr_gpu_addr; 393 uint32_t me = 0, eng_sel = 0; 394 395 switch (ring->funcs->type) { 396 case AMDGPU_RING_TYPE_COMPUTE: 397 me = 1; 398 eng_sel = 0; 399 break; 400 case AMDGPU_RING_TYPE_GFX: 401 me = 0; 402 eng_sel = 4; 403 break; 404 case AMDGPU_RING_TYPE_MES: 405 me = 2; 406 eng_sel = 5; 407 break; 408 default: 409 WARN_ON(1); 410 } 411 412 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 413 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 414 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 415 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 416 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 417 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 418 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 419 PACKET3_MAP_QUEUES_ME((me)) | 420 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 421 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 422 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 423 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 424 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 425 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 426 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 427 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 428 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 429 } 430 431 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 432 struct amdgpu_ring *ring, 433 enum amdgpu_unmap_queues_action action, 434 u64 gpu_addr, u64 seq) 435 { 436 struct amdgpu_device *adev = kiq_ring->adev; 437 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 438 439 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) { 440 amdgpu_mes_unmap_legacy_queue(adev, ring, action, 441 gpu_addr, seq, 0); 442 return; 443 } 444 445 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 446 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 447 PACKET3_UNMAP_QUEUES_ACTION(action) | 448 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 449 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 450 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 451 amdgpu_ring_write(kiq_ring, 452 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 453 454 if (action == PREEMPT_QUEUES_NO_UNMAP) { 455 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 456 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 457 amdgpu_ring_write(kiq_ring, seq); 458 } else { 459 amdgpu_ring_write(kiq_ring, 0); 460 amdgpu_ring_write(kiq_ring, 0); 461 amdgpu_ring_write(kiq_ring, 0); 462 } 463 } 464 465 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring, 466 struct amdgpu_ring *ring, 467 u64 addr, 468 u64 seq) 469 { 470 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 471 472 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 473 amdgpu_ring_write(kiq_ring, 474 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 475 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 476 PACKET3_QUERY_STATUS_COMMAND(2)); 477 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 478 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 479 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 480 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 481 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 482 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 483 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 484 } 485 486 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 487 uint16_t pasid, uint32_t flush_type, 488 bool all_hub) 489 { 490 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1); 491 } 492 493 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = { 494 .kiq_set_resources = gfx11_kiq_set_resources, 495 .kiq_map_queues = gfx11_kiq_map_queues, 496 .kiq_unmap_queues = gfx11_kiq_unmap_queues, 497 .kiq_query_status = gfx11_kiq_query_status, 498 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs, 499 .set_resources_size = 8, 500 .map_queues_size = 7, 501 .unmap_queues_size = 6, 502 .query_status_size = 7, 503 .invalidate_tlbs_size = 2, 504 }; 505 506 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 507 { 508 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs; 509 } 510 511 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev) 512 { 513 if (amdgpu_sriov_vf(adev)) 514 return; 515 516 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 517 case IP_VERSION(11, 0, 1): 518 case IP_VERSION(11, 0, 4): 519 soc15_program_register_sequence(adev, 520 golden_settings_gc_11_0_1, 521 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1)); 522 break; 523 default: 524 break; 525 } 526 soc15_program_register_sequence(adev, 527 golden_settings_gc_11_0, 528 (const u32)ARRAY_SIZE(golden_settings_gc_11_0)); 529 530 } 531 532 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 533 bool wc, uint32_t reg, uint32_t val) 534 { 535 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 536 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 537 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0)); 538 amdgpu_ring_write(ring, reg); 539 amdgpu_ring_write(ring, 0); 540 amdgpu_ring_write(ring, val); 541 } 542 543 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 544 int mem_space, int opt, uint32_t addr0, 545 uint32_t addr1, uint32_t ref, uint32_t mask, 546 uint32_t inv) 547 { 548 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 549 amdgpu_ring_write(ring, 550 /* memory (1) or register (0) */ 551 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 552 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 553 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 554 WAIT_REG_MEM_ENGINE(eng_sel))); 555 556 if (mem_space) 557 WARN_ON(addr0 & 0x3); /* Dword align */ 558 amdgpu_ring_write(ring, addr0); 559 amdgpu_ring_write(ring, addr1); 560 amdgpu_ring_write(ring, ref); 561 amdgpu_ring_write(ring, mask); 562 amdgpu_ring_write(ring, inv); /* poll interval */ 563 } 564 565 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop) 566 { 567 /* Header itself is a NOP packet */ 568 if (num_nop == 1) { 569 amdgpu_ring_write(ring, ring->funcs->nop); 570 return; 571 } 572 573 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 574 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); 575 576 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ 577 amdgpu_ring_insert_nop(ring, num_nop - 1); 578 } 579 580 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring) 581 { 582 struct amdgpu_device *adev = ring->adev; 583 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 584 uint32_t tmp = 0; 585 unsigned i; 586 int r; 587 588 WREG32(scratch, 0xCAFEDEAD); 589 r = amdgpu_ring_alloc(ring, 5); 590 if (r) { 591 drm_err(adev_to_drm(adev), "cp failed to lock ring %d (%d).\n", 592 ring->idx, r); 593 return r; 594 } 595 596 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) { 597 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF); 598 } else { 599 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 600 amdgpu_ring_write(ring, scratch - 601 PACKET3_SET_UCONFIG_REG_START); 602 amdgpu_ring_write(ring, 0xDEADBEEF); 603 } 604 amdgpu_ring_commit(ring); 605 606 for (i = 0; i < adev->usec_timeout; i++) { 607 tmp = RREG32(scratch); 608 if (tmp == 0xDEADBEEF) 609 break; 610 if (amdgpu_emu_mode == 1) 611 msleep(1); 612 else 613 udelay(1); 614 } 615 616 if (i >= adev->usec_timeout) 617 r = -ETIMEDOUT; 618 return r; 619 } 620 621 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 622 { 623 struct amdgpu_device *adev = ring->adev; 624 struct amdgpu_ib ib; 625 struct dma_fence *f = NULL; 626 unsigned index; 627 uint64_t gpu_addr; 628 uint32_t *cpu_ptr; 629 long r; 630 631 /* MES KIQ fw hasn't indirect buffer support for now */ 632 if (adev->enable_mes_kiq && 633 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 634 return 0; 635 636 memset(&ib, 0, sizeof(ib)); 637 638 r = amdgpu_wb_get(adev, &index); 639 if (r) 640 return r; 641 642 gpu_addr = adev->wb.gpu_addr + (index * 4); 643 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 644 cpu_ptr = &adev->wb.wb[index]; 645 646 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 647 if (r) { 648 drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r); 649 goto err1; 650 } 651 652 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 653 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 654 ib.ptr[2] = lower_32_bits(gpu_addr); 655 ib.ptr[3] = upper_32_bits(gpu_addr); 656 ib.ptr[4] = 0xDEADBEEF; 657 ib.length_dw = 5; 658 659 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 660 if (r) 661 goto err2; 662 663 r = dma_fence_wait_timeout(f, false, timeout); 664 if (r == 0) { 665 r = -ETIMEDOUT; 666 goto err2; 667 } else if (r < 0) { 668 goto err2; 669 } 670 671 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF) 672 r = 0; 673 else 674 r = -EINVAL; 675 err2: 676 amdgpu_ib_free(&ib, NULL); 677 dma_fence_put(f); 678 err1: 679 amdgpu_wb_free(adev, index); 680 return r; 681 } 682 683 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev) 684 { 685 amdgpu_ucode_release(&adev->gfx.pfp_fw); 686 amdgpu_ucode_release(&adev->gfx.me_fw); 687 amdgpu_ucode_release(&adev->gfx.rlc_fw); 688 amdgpu_ucode_release(&adev->gfx.mec_fw); 689 690 kfree(adev->gfx.rlc.register_list_format); 691 } 692 693 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix) 694 { 695 const struct psp_firmware_header_v1_0 *toc_hdr; 696 int err = 0; 697 698 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, 699 AMDGPU_UCODE_REQUIRED, 700 "amdgpu/%s_toc.bin", ucode_prefix); 701 if (err) 702 goto out; 703 704 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data; 705 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version); 706 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version); 707 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes); 708 adev->psp.toc.start_addr = (uint8_t *)toc_hdr + 709 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes); 710 return 0; 711 out: 712 amdgpu_ucode_release(&adev->psp.toc_fw); 713 return err; 714 } 715 716 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev) 717 { 718 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 719 case IP_VERSION(11, 0, 0): 720 case IP_VERSION(11, 0, 2): 721 case IP_VERSION(11, 0, 3): 722 if ((adev->gfx.me_fw_version >= 1505) && 723 (adev->gfx.pfp_fw_version >= 1600) && 724 (adev->gfx.mec_fw_version >= 512)) { 725 if (amdgpu_sriov_vf(adev)) 726 adev->gfx.cp_gfx_shadow = true; 727 else 728 adev->gfx.cp_gfx_shadow = false; 729 } 730 break; 731 default: 732 adev->gfx.cp_gfx_shadow = false; 733 break; 734 } 735 } 736 737 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) 738 { 739 char ucode_prefix[25]; 740 int err; 741 const struct rlc_firmware_header_v2_0 *rlc_hdr; 742 uint16_t version_major; 743 uint16_t version_minor; 744 745 DRM_DEBUG("\n"); 746 747 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 748 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 749 AMDGPU_UCODE_REQUIRED, 750 "amdgpu/%s_pfp.bin", ucode_prefix); 751 if (err) 752 goto out; 753 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/ 754 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version( 755 (union amdgpu_firmware_header *) 756 adev->gfx.pfp_fw->data, 2, 0); 757 if (adev->gfx.rs64_enable) { 758 dev_info(adev->dev, "CP RS64 enable\n"); 759 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP); 760 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK); 761 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK); 762 } else { 763 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 764 } 765 766 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 767 AMDGPU_UCODE_REQUIRED, 768 "amdgpu/%s_me.bin", ucode_prefix); 769 if (err) 770 goto out; 771 if (adev->gfx.rs64_enable) { 772 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME); 773 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK); 774 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK); 775 } else { 776 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 777 } 778 779 if (!amdgpu_sriov_vf(adev)) { 780 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) && 781 adev->pdev->revision == 0xCE) 782 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 783 AMDGPU_UCODE_REQUIRED, 784 "amdgpu/gc_11_0_0_rlc_1.bin"); 785 else if (amdgpu_is_kicker_fw(adev)) 786 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 787 AMDGPU_UCODE_REQUIRED, 788 "amdgpu/%s_rlc_kicker.bin", ucode_prefix); 789 else 790 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 791 AMDGPU_UCODE_REQUIRED, 792 "amdgpu/%s_rlc.bin", ucode_prefix); 793 if (err) 794 goto out; 795 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 796 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 797 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 798 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 799 if (err) 800 goto out; 801 } 802 803 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 804 AMDGPU_UCODE_REQUIRED, 805 "amdgpu/%s_mec.bin", ucode_prefix); 806 if (err) 807 goto out; 808 if (adev->gfx.rs64_enable) { 809 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC); 810 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK); 811 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK); 812 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK); 813 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK); 814 } else { 815 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 816 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 817 } 818 819 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 820 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix); 821 822 /* only one MEC for gfx 11.0.0. */ 823 adev->gfx.mec2_fw = NULL; 824 825 gfx_v11_0_check_fw_cp_gfx_shadow(adev); 826 827 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) { 828 err = adev->gfx.imu.funcs->init_microcode(adev); 829 if (err) 830 DRM_ERROR("Failed to init imu firmware!\n"); 831 return err; 832 } 833 834 out: 835 if (err) { 836 amdgpu_ucode_release(&adev->gfx.pfp_fw); 837 amdgpu_ucode_release(&adev->gfx.me_fw); 838 amdgpu_ucode_release(&adev->gfx.rlc_fw); 839 amdgpu_ucode_release(&adev->gfx.mec_fw); 840 } 841 842 return err; 843 } 844 845 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev) 846 { 847 u32 count = 0; 848 const struct cs_section_def *sect = NULL; 849 const struct cs_extent_def *ext = NULL; 850 851 /* begin clear state */ 852 count += 2; 853 /* context control state */ 854 count += 3; 855 856 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 857 for (ext = sect->section; ext->extent != NULL; ++ext) { 858 if (sect->id == SECT_CONTEXT) 859 count += 2 + ext->reg_count; 860 else 861 return 0; 862 } 863 } 864 865 /* set PA_SC_TILE_STEERING_OVERRIDE */ 866 count += 3; 867 /* end clear state */ 868 count += 2; 869 /* clear state */ 870 count += 2; 871 872 return count; 873 } 874 875 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 876 { 877 u32 count = 0; 878 int ctx_reg_offset; 879 880 if (adev->gfx.rlc.cs_data == NULL) 881 return; 882 if (buffer == NULL) 883 return; 884 885 count = amdgpu_gfx_csb_preamble_start(buffer); 886 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 887 888 ctx_reg_offset = SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 889 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 890 buffer[count++] = cpu_to_le32(ctx_reg_offset); 891 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override); 892 893 amdgpu_gfx_csb_preamble_end(buffer, count); 894 } 895 896 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev) 897 { 898 /* clear state block */ 899 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 900 &adev->gfx.rlc.clear_state_gpu_addr, 901 (void **)&adev->gfx.rlc.cs_ptr); 902 903 /* jump table block */ 904 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 905 &adev->gfx.rlc.cp_table_gpu_addr, 906 (void **)&adev->gfx.rlc.cp_table_ptr); 907 } 908 909 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 910 { 911 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 912 913 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 914 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 915 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1); 916 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2); 917 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3); 918 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL); 919 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX); 920 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0); 921 adev->gfx.rlc.rlcg_reg_access_supported = true; 922 } 923 924 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev) 925 { 926 const struct cs_section_def *cs_data; 927 int r; 928 929 adev->gfx.rlc.cs_data = gfx11_cs_data; 930 931 cs_data = adev->gfx.rlc.cs_data; 932 933 if (cs_data) { 934 /* init clear state block */ 935 r = amdgpu_gfx_rlc_init_csb(adev); 936 if (r) 937 return r; 938 } 939 940 /* init spm vmid with 0xf */ 941 if (adev->gfx.rlc.funcs->update_spm_vmid) 942 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf); 943 944 return 0; 945 } 946 947 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev) 948 { 949 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 950 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 951 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL); 952 } 953 954 static void gfx_v11_0_me_init(struct amdgpu_device *adev) 955 { 956 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 957 958 amdgpu_gfx_graphics_queue_acquire(adev); 959 } 960 961 static int gfx_v11_0_mec_init(struct amdgpu_device *adev) 962 { 963 int r; 964 u32 *hpd; 965 size_t mec_hpd_size; 966 967 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 968 969 /* take ownership of the relevant compute queues */ 970 amdgpu_gfx_compute_queue_acquire(adev); 971 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE; 972 973 if (mec_hpd_size) { 974 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 975 AMDGPU_GEM_DOMAIN_GTT, 976 &adev->gfx.mec.hpd_eop_obj, 977 &adev->gfx.mec.hpd_eop_gpu_addr, 978 (void **)&hpd); 979 if (r) { 980 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 981 gfx_v11_0_mec_fini(adev); 982 return r; 983 } 984 985 memset(hpd, 0, mec_hpd_size); 986 987 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 988 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 989 } 990 991 return 0; 992 } 993 994 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address) 995 { 996 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 997 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 998 (address << SQ_IND_INDEX__INDEX__SHIFT)); 999 return RREG32_SOC15(GC, 0, regSQ_IND_DATA); 1000 } 1001 1002 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave, 1003 uint32_t thread, uint32_t regno, 1004 uint32_t num, uint32_t *out) 1005 { 1006 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 1007 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1008 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1009 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) | 1010 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1011 while (num--) 1012 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA); 1013 } 1014 1015 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1016 { 1017 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE 1018 * field when performing a select_se_sh so it should be 1019 * zero here */ 1020 WARN_ON(simd != 0); 1021 1022 /* type 3 wave data */ 1023 dst[(*no_fields)++] = 3; 1024 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS); 1025 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO); 1026 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI); 1027 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO); 1028 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI); 1029 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1); 1030 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2); 1031 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC); 1032 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC); 1033 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS); 1034 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS); 1035 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2); 1036 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1); 1037 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0); 1038 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE); 1039 } 1040 1041 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1042 uint32_t wave, uint32_t start, 1043 uint32_t size, uint32_t *dst) 1044 { 1045 WARN_ON(simd != 0); 1046 1047 wave_read_regs( 1048 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size, 1049 dst); 1050 } 1051 1052 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1053 uint32_t wave, uint32_t thread, 1054 uint32_t start, uint32_t size, 1055 uint32_t *dst) 1056 { 1057 wave_read_regs( 1058 adev, wave, thread, 1059 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1060 } 1061 1062 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev, 1063 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1064 { 1065 soc21_grbm_select(adev, me, pipe, q, vm); 1066 } 1067 1068 /* all sizes are in bytes */ 1069 #define MQD_SHADOW_BASE_SIZE 73728 1070 #define MQD_SHADOW_BASE_ALIGNMENT 256 1071 #define MQD_FWWORKAREA_SIZE 484 1072 #define MQD_FWWORKAREA_ALIGNMENT 256 1073 1074 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev, 1075 struct amdgpu_gfx_shadow_info *shadow_info) 1076 { 1077 /* for gfx */ 1078 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE; 1079 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT; 1080 shadow_info->csa_size = MQD_FWWORKAREA_SIZE; 1081 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT; 1082 /* for compute */ 1083 shadow_info->eop_size = GFX11_MEC_HPD_SIZE; 1084 shadow_info->eop_alignment = 256; 1085 } 1086 1087 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev, 1088 struct amdgpu_gfx_shadow_info *shadow_info, 1089 bool skip_check) 1090 { 1091 if (adev->gfx.cp_gfx_shadow || skip_check) { 1092 gfx_v11_0_get_gfx_shadow_info_nocheck(adev, shadow_info); 1093 return 0; 1094 } else { 1095 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info)); 1096 return -ENOTSUPP; 1097 } 1098 } 1099 1100 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = { 1101 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter, 1102 .select_se_sh = &gfx_v11_0_select_se_sh, 1103 .read_wave_data = &gfx_v11_0_read_wave_data, 1104 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs, 1105 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs, 1106 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q, 1107 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk, 1108 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info, 1109 .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask, 1110 }; 1111 1112 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) 1113 { 1114 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1115 case IP_VERSION(11, 0, 0): 1116 case IP_VERSION(11, 0, 2): 1117 adev->gfx.config.max_hw_contexts = 8; 1118 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1119 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1120 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1121 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1122 break; 1123 case IP_VERSION(11, 0, 3): 1124 adev->gfx.ras = &gfx_v11_0_3_ras; 1125 adev->gfx.config.max_hw_contexts = 8; 1126 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1127 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1128 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1129 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1130 break; 1131 case IP_VERSION(11, 0, 1): 1132 case IP_VERSION(11, 0, 4): 1133 case IP_VERSION(11, 5, 0): 1134 case IP_VERSION(11, 5, 1): 1135 case IP_VERSION(11, 5, 2): 1136 case IP_VERSION(11, 5, 3): 1137 case IP_VERSION(11, 5, 4): 1138 case IP_VERSION(11, 5, 6): 1139 case IP_VERSION(11, 7, 0): 1140 case IP_VERSION(11, 7, 1): 1141 adev->gfx.config.max_hw_contexts = 8; 1142 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1143 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1144 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 1145 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300; 1146 break; 1147 default: 1148 BUG(); 1149 break; 1150 } 1151 1152 return 0; 1153 } 1154 1155 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id, 1156 int me, int pipe, int queue) 1157 { 1158 struct amdgpu_ring *ring; 1159 unsigned int irq_type; 1160 unsigned int hw_prio; 1161 1162 ring = &adev->gfx.gfx_ring[ring_id]; 1163 1164 ring->me = me; 1165 ring->pipe = pipe; 1166 ring->queue = queue; 1167 1168 ring->ring_obj = NULL; 1169 ring->use_doorbell = true; 1170 if (adev->gfx.disable_kq) { 1171 ring->no_scheduler = true; 1172 ring->no_user_submission = true; 1173 } 1174 1175 if (!ring_id) 1176 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 1177 else 1178 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1; 1179 ring->vm_hub = AMDGPU_GFXHUB(0); 1180 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1181 1182 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe; 1183 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ? 1184 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1185 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1186 hw_prio, NULL); 1187 } 1188 1189 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1190 int mec, int pipe, int queue) 1191 { 1192 int r; 1193 unsigned irq_type; 1194 struct amdgpu_ring *ring; 1195 unsigned int hw_prio; 1196 1197 ring = &adev->gfx.compute_ring[ring_id]; 1198 1199 /* mec0 is me1 */ 1200 ring->me = mec + 1; 1201 ring->pipe = pipe; 1202 ring->queue = queue; 1203 1204 ring->ring_obj = NULL; 1205 ring->use_doorbell = true; 1206 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 1207 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1208 + (ring_id * GFX11_MEC_HPD_SIZE); 1209 ring->vm_hub = AMDGPU_GFXHUB(0); 1210 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1211 1212 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1213 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1214 + ring->pipe; 1215 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1216 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1217 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1218 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1219 hw_prio, NULL); 1220 if (r) 1221 return r; 1222 1223 return 0; 1224 } 1225 1226 static struct { 1227 SOC21_FIRMWARE_ID id; 1228 unsigned int offset; 1229 unsigned int size; 1230 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX]; 1231 1232 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc) 1233 { 1234 RLC_TABLE_OF_CONTENT *ucode = rlc_toc; 1235 1236 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) && 1237 (ucode->id < SOC21_FIRMWARE_ID_MAX)) { 1238 rlc_autoload_info[ucode->id].id = ucode->id; 1239 rlc_autoload_info[ucode->id].offset = ucode->offset * 4; 1240 rlc_autoload_info[ucode->id].size = ucode->size * 4; 1241 1242 ucode++; 1243 } 1244 } 1245 1246 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev) 1247 { 1248 uint32_t total_size = 0; 1249 SOC21_FIRMWARE_ID id; 1250 1251 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr); 1252 1253 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++) 1254 total_size += rlc_autoload_info[id].size; 1255 1256 /* In case the offset in rlc toc ucode is aligned */ 1257 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset) 1258 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset + 1259 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size; 1260 1261 return total_size; 1262 } 1263 1264 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev) 1265 { 1266 int r; 1267 uint32_t total_size; 1268 1269 total_size = gfx_v11_0_calc_toc_total_size(adev); 1270 1271 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024, 1272 AMDGPU_GEM_DOMAIN_VRAM | 1273 AMDGPU_GEM_DOMAIN_GTT, 1274 &adev->gfx.rlc.rlc_autoload_bo, 1275 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1276 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1277 1278 if (r) { 1279 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r); 1280 return r; 1281 } 1282 1283 return 0; 1284 } 1285 1286 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev, 1287 SOC21_FIRMWARE_ID id, 1288 const void *fw_data, 1289 uint32_t fw_size, 1290 uint32_t *fw_autoload_mask) 1291 { 1292 uint32_t toc_offset; 1293 uint32_t toc_fw_size; 1294 char *ptr = adev->gfx.rlc.rlc_autoload_ptr; 1295 1296 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX) 1297 return; 1298 1299 toc_offset = rlc_autoload_info[id].offset; 1300 toc_fw_size = rlc_autoload_info[id].size; 1301 1302 if (fw_size == 0) 1303 fw_size = toc_fw_size; 1304 1305 if (fw_size > toc_fw_size) 1306 fw_size = toc_fw_size; 1307 1308 memcpy(ptr + toc_offset, fw_data, fw_size); 1309 1310 if (fw_size < toc_fw_size) 1311 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); 1312 1313 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME)) 1314 *(uint64_t *)fw_autoload_mask |= 1ULL << id; 1315 } 1316 1317 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev, 1318 uint32_t *fw_autoload_mask) 1319 { 1320 void *data; 1321 uint32_t size; 1322 uint64_t *toc_ptr; 1323 1324 *(uint64_t *)fw_autoload_mask |= 0x1; 1325 1326 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask); 1327 1328 data = adev->psp.toc.start_addr; 1329 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size; 1330 1331 toc_ptr = (uint64_t *)data + size / 8 - 1; 1332 *toc_ptr = *(uint64_t *)fw_autoload_mask; 1333 1334 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC, 1335 data, size, fw_autoload_mask); 1336 } 1337 1338 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev, 1339 uint32_t *fw_autoload_mask) 1340 { 1341 const __le32 *fw_data; 1342 uint32_t fw_size; 1343 const struct gfx_firmware_header_v1_0 *cp_hdr; 1344 const struct gfx_firmware_header_v2_0 *cpv2_hdr; 1345 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1346 const struct rlc_firmware_header_v2_2 *rlcv22_hdr; 1347 uint16_t version_major, version_minor; 1348 1349 if (adev->gfx.rs64_enable) { 1350 /* pfp ucode */ 1351 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1352 adev->gfx.pfp_fw->data; 1353 /* instruction */ 1354 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1355 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1356 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1357 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP, 1358 fw_data, fw_size, fw_autoload_mask); 1359 /* data */ 1360 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1361 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1362 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1363 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK, 1364 fw_data, fw_size, fw_autoload_mask); 1365 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK, 1366 fw_data, fw_size, fw_autoload_mask); 1367 /* me ucode */ 1368 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1369 adev->gfx.me_fw->data; 1370 /* instruction */ 1371 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1372 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1373 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1374 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME, 1375 fw_data, fw_size, fw_autoload_mask); 1376 /* data */ 1377 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1378 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1379 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1380 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK, 1381 fw_data, fw_size, fw_autoload_mask); 1382 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK, 1383 fw_data, fw_size, fw_autoload_mask); 1384 /* mec ucode */ 1385 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1386 adev->gfx.mec_fw->data; 1387 /* instruction */ 1388 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1389 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1390 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1391 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC, 1392 fw_data, fw_size, fw_autoload_mask); 1393 /* data */ 1394 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1395 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1396 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1397 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK, 1398 fw_data, fw_size, fw_autoload_mask); 1399 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK, 1400 fw_data, fw_size, fw_autoload_mask); 1401 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK, 1402 fw_data, fw_size, fw_autoload_mask); 1403 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK, 1404 fw_data, fw_size, fw_autoload_mask); 1405 } else { 1406 /* pfp ucode */ 1407 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1408 adev->gfx.pfp_fw->data; 1409 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1410 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1411 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1412 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP, 1413 fw_data, fw_size, fw_autoload_mask); 1414 1415 /* me ucode */ 1416 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1417 adev->gfx.me_fw->data; 1418 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1419 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1420 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1421 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME, 1422 fw_data, fw_size, fw_autoload_mask); 1423 1424 /* mec ucode */ 1425 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1426 adev->gfx.mec_fw->data; 1427 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1428 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1429 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1430 cp_hdr->jt_size * 4; 1431 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC, 1432 fw_data, fw_size, fw_autoload_mask); 1433 } 1434 1435 /* rlc ucode */ 1436 rlc_hdr = (const struct rlc_firmware_header_v2_0 *) 1437 adev->gfx.rlc_fw->data; 1438 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1439 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes)); 1440 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes); 1441 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE, 1442 fw_data, fw_size, fw_autoload_mask); 1443 1444 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1445 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1446 if (version_major == 2) { 1447 if (version_minor >= 2) { 1448 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 1449 1450 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1451 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes)); 1452 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes); 1453 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE, 1454 fw_data, fw_size, fw_autoload_mask); 1455 1456 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1457 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes)); 1458 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes); 1459 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT, 1460 fw_data, fw_size, fw_autoload_mask); 1461 } 1462 } 1463 } 1464 1465 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev, 1466 uint32_t *fw_autoload_mask) 1467 { 1468 const __le32 *fw_data; 1469 uint32_t fw_size; 1470 const struct sdma_firmware_header_v2_0 *sdma_hdr; 1471 1472 sdma_hdr = (const struct sdma_firmware_header_v2_0 *) 1473 adev->sdma.instance[0].fw->data; 1474 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1475 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes)); 1476 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes); 1477 1478 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1479 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask); 1480 1481 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1482 le32_to_cpu(sdma_hdr->ctl_ucode_offset)); 1483 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes); 1484 1485 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1486 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask); 1487 } 1488 1489 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev, 1490 uint32_t *fw_autoload_mask) 1491 { 1492 const __le32 *fw_data; 1493 unsigned fw_size; 1494 const struct mes_firmware_header_v1_0 *mes_hdr; 1495 int pipe, ucode_id, data_id; 1496 1497 for (pipe = 0; pipe < 2; pipe++) { 1498 if (pipe==0) { 1499 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0; 1500 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK; 1501 } else { 1502 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1; 1503 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK; 1504 } 1505 1506 mes_hdr = (const struct mes_firmware_header_v1_0 *) 1507 adev->mes.fw[pipe]->data; 1508 1509 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1510 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); 1511 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); 1512 1513 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1514 ucode_id, fw_data, fw_size, fw_autoload_mask); 1515 1516 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1517 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); 1518 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); 1519 1520 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1521 data_id, fw_data, fw_size, fw_autoload_mask); 1522 } 1523 } 1524 1525 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) 1526 { 1527 uint32_t rlc_g_offset, rlc_g_size; 1528 uint64_t gpu_addr; 1529 uint32_t autoload_fw_id[2]; 1530 1531 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2); 1532 1533 /* RLC autoload sequence 2: copy ucode */ 1534 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id); 1535 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id); 1536 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id); 1537 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id); 1538 1539 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset; 1540 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size; 1541 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset; 1542 1543 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr)); 1544 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr)); 1545 1546 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size); 1547 1548 /* RLC autoload sequence 3: load IMU fw */ 1549 if (adev->gfx.imu.funcs->load_microcode) 1550 adev->gfx.imu.funcs->load_microcode(adev); 1551 /* RLC autoload sequence 4 init IMU fw */ 1552 if (adev->gfx.imu.funcs->setup_imu) 1553 adev->gfx.imu.funcs->setup_imu(adev); 1554 if (adev->gfx.imu.funcs->start_imu) 1555 adev->gfx.imu.funcs->start_imu(adev); 1556 1557 /* RLC autoload sequence 5 disable gpa mode */ 1558 gfx_v11_0_disable_gpa_mode(adev); 1559 1560 return 0; 1561 } 1562 1563 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev) 1564 { 1565 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 1566 uint32_t *ptr; 1567 uint32_t inst; 1568 1569 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL); 1570 if (!ptr) { 1571 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 1572 adev->gfx.ip_dump_core = NULL; 1573 } else { 1574 adev->gfx.ip_dump_core = ptr; 1575 } 1576 1577 /* Allocate memory for compute queue registers for all the instances */ 1578 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 1579 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1580 adev->gfx.mec.num_queue_per_pipe; 1581 1582 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1583 if (!ptr) { 1584 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 1585 adev->gfx.ip_dump_compute_queues = NULL; 1586 } else { 1587 adev->gfx.ip_dump_compute_queues = ptr; 1588 } 1589 1590 /* Allocate memory for gfx queue registers for all the instances */ 1591 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 1592 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me * 1593 adev->gfx.me.num_queue_per_pipe; 1594 1595 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1596 if (!ptr) { 1597 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n"); 1598 adev->gfx.ip_dump_gfx_queues = NULL; 1599 } else { 1600 adev->gfx.ip_dump_gfx_queues = ptr; 1601 } 1602 } 1603 1604 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) 1605 { 1606 int i, j, k, r, ring_id; 1607 int xcc_id = 0; 1608 struct amdgpu_device *adev = ip_block->adev; 1609 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */ 1610 1611 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler); 1612 1613 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1614 case IP_VERSION(11, 0, 0): 1615 case IP_VERSION(11, 0, 1): 1616 case IP_VERSION(11, 0, 2): 1617 case IP_VERSION(11, 0, 3): 1618 case IP_VERSION(11, 0, 4): 1619 case IP_VERSION(11, 5, 0): 1620 case IP_VERSION(11, 5, 1): 1621 case IP_VERSION(11, 5, 2): 1622 case IP_VERSION(11, 5, 3): 1623 case IP_VERSION(11, 5, 4): 1624 case IP_VERSION(11, 5, 6): 1625 case IP_VERSION(11, 7, 0): 1626 case IP_VERSION(11, 7, 1): 1627 adev->gfx.me.num_me = 1; 1628 if (adev->gfx.rs64_enable) 1629 adev->gfx.me.num_pipe_per_me = 1; 1630 else 1631 adev->gfx.me.num_pipe_per_me = 2; 1632 adev->gfx.me.num_queue_per_pipe = 2; 1633 adev->gfx.mec.num_mec = 1; 1634 adev->gfx.mec.num_pipe_per_mec = 4; 1635 adev->gfx.mec.num_queue_per_pipe = 4; 1636 break; 1637 default: 1638 adev->gfx.me.num_me = 1; 1639 adev->gfx.me.num_pipe_per_me = 1; 1640 adev->gfx.me.num_queue_per_pipe = 1; 1641 adev->gfx.mec.num_mec = 1; 1642 adev->gfx.mec.num_pipe_per_mec = 4; 1643 adev->gfx.mec.num_queue_per_pipe = 8; 1644 break; 1645 } 1646 1647 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1648 case IP_VERSION(11, 0, 0): 1649 case IP_VERSION(11, 0, 2): 1650 case IP_VERSION(11, 0, 3): 1651 if (!adev->gfx.disable_uq && 1652 adev->gfx.me_fw_version >= 2420 && 1653 adev->gfx.pfp_fw_version >= 2580 && 1654 adev->gfx.mec_fw_version >= 2650 && 1655 adev->mes.fw_version[0] >= 120) { 1656 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1657 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1658 } 1659 break; 1660 case IP_VERSION(11, 0, 1): 1661 case IP_VERSION(11, 0, 4): 1662 case IP_VERSION(11, 5, 0): 1663 case IP_VERSION(11, 5, 1): 1664 case IP_VERSION(11, 5, 2): 1665 case IP_VERSION(11, 5, 3): 1666 /* add firmware version checks here */ 1667 if (0 && !adev->gfx.disable_uq) { 1668 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1669 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1670 } 1671 break; 1672 default: 1673 break; 1674 } 1675 1676 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1677 case IP_VERSION(11, 0, 0): 1678 case IP_VERSION(11, 0, 2): 1679 case IP_VERSION(11, 0, 3): 1680 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1681 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1682 if (adev->gfx.me_fw_version >= 2280 && 1683 adev->gfx.pfp_fw_version >= 2370 && 1684 adev->gfx.mec_fw_version >= 2450 && 1685 adev->mes.fw_version[0] >= 99) { 1686 adev->gfx.enable_cleaner_shader = true; 1687 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1688 if (r) { 1689 adev->gfx.enable_cleaner_shader = false; 1690 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1691 } 1692 } 1693 break; 1694 case IP_VERSION(11, 0, 1): 1695 case IP_VERSION(11, 0, 4): 1696 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1697 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1698 if (adev->gfx.pfp_fw_version >= 102 && 1699 adev->gfx.mec_fw_version >= 66 && 1700 adev->mes.fw_version[0] >= 128) { 1701 adev->gfx.enable_cleaner_shader = true; 1702 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1703 if (r) { 1704 adev->gfx.enable_cleaner_shader = false; 1705 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1706 } 1707 } 1708 break; 1709 case IP_VERSION(11, 5, 0): 1710 case IP_VERSION(11, 5, 1): 1711 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1712 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1713 if (adev->gfx.mec_fw_version >= 26 && 1714 adev->mes.fw_version[0] >= 114) { 1715 adev->gfx.enable_cleaner_shader = true; 1716 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1717 if (r) { 1718 adev->gfx.enable_cleaner_shader = false; 1719 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1720 } 1721 } 1722 break; 1723 case IP_VERSION(11, 5, 2): 1724 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1725 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1726 if (adev->gfx.me_fw_version >= 12 && 1727 adev->gfx.pfp_fw_version >= 15 && 1728 adev->gfx.mec_fw_version >= 15) { 1729 adev->gfx.enable_cleaner_shader = true; 1730 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1731 if (r) { 1732 adev->gfx.enable_cleaner_shader = false; 1733 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1734 } 1735 } 1736 break; 1737 case IP_VERSION(11, 5, 3): 1738 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1739 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1740 if (adev->gfx.me_fw_version >= 7 && 1741 adev->gfx.pfp_fw_version >= 8 && 1742 adev->gfx.mec_fw_version >= 8) { 1743 adev->gfx.enable_cleaner_shader = true; 1744 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1745 if (r) { 1746 adev->gfx.enable_cleaner_shader = false; 1747 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1748 } 1749 } 1750 break; 1751 case IP_VERSION(11, 5, 4): 1752 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1753 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1754 if (adev->gfx.me_fw_version >= 4 && 1755 adev->gfx.pfp_fw_version >= 7 && 1756 adev->gfx.mec_fw_version >= 5) { 1757 adev->gfx.enable_cleaner_shader = true; 1758 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1759 if (r) { 1760 adev->gfx.enable_cleaner_shader = false; 1761 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1762 } 1763 } 1764 break; 1765 default: 1766 adev->gfx.enable_cleaner_shader = false; 1767 break; 1768 } 1769 1770 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */ 1771 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) && 1772 amdgpu_sriov_is_pp_one_vf(adev)) 1773 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG; 1774 1775 /* EOP Event */ 1776 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1777 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT, 1778 &adev->gfx.eop_irq); 1779 if (r) 1780 return r; 1781 1782 /* Bad opcode Event */ 1783 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1784 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR, 1785 &adev->gfx.bad_op_irq); 1786 if (r) 1787 return r; 1788 1789 /* Privileged reg */ 1790 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1791 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT, 1792 &adev->gfx.priv_reg_irq); 1793 if (r) 1794 return r; 1795 1796 /* Privileged inst */ 1797 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1798 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT, 1799 &adev->gfx.priv_inst_irq); 1800 if (r) 1801 return r; 1802 1803 /* FED error */ 1804 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX, 1805 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT, 1806 &adev->gfx.rlc_gc_fed_irq); 1807 if (r) 1808 return r; 1809 1810 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1811 1812 gfx_v11_0_me_init(adev); 1813 1814 r = gfx_v11_0_rlc_init(adev); 1815 if (r) { 1816 DRM_ERROR("Failed to init rlc BOs!\n"); 1817 return r; 1818 } 1819 1820 r = gfx_v11_0_mec_init(adev); 1821 if (r) { 1822 DRM_ERROR("Failed to init MEC BOs!\n"); 1823 return r; 1824 } 1825 1826 if (adev->gfx.num_gfx_rings) { 1827 ring_id = 0; 1828 /* set up the gfx ring */ 1829 for (i = 0; i < adev->gfx.me.num_me; i++) { 1830 for (j = 0; j < num_queue_per_pipe; j++) { 1831 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 1832 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j)) 1833 continue; 1834 1835 r = gfx_v11_0_gfx_ring_init(adev, ring_id, 1836 i, k, j); 1837 if (r) 1838 return r; 1839 ring_id++; 1840 } 1841 } 1842 } 1843 } 1844 1845 if (adev->gfx.num_compute_rings) { 1846 ring_id = 0; 1847 /* set up the compute queues - allocate horizontally across pipes */ 1848 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1849 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1850 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1851 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 1852 k, j)) 1853 continue; 1854 1855 r = gfx_v11_0_compute_ring_init(adev, ring_id, 1856 i, k, j); 1857 if (r) 1858 return r; 1859 1860 ring_id++; 1861 } 1862 } 1863 } 1864 } 1865 1866 adev->gfx.gfx_supported_reset = 1867 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 1868 adev->gfx.compute_supported_reset = 1869 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 1870 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1871 case IP_VERSION(11, 0, 0): 1872 case IP_VERSION(11, 0, 2): 1873 case IP_VERSION(11, 0, 3): 1874 if ((adev->gfx.me_fw_version >= 2280) && 1875 (adev->gfx.mec_fw_version >= 2410) && 1876 !amdgpu_sriov_vf(adev) && 1877 !adev->debug_disable_gpu_ring_reset) { 1878 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1879 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1880 } 1881 break; 1882 default: 1883 if (!amdgpu_sriov_vf(adev) && 1884 !adev->debug_disable_gpu_ring_reset) { 1885 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1886 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1887 } 1888 break; 1889 } 1890 1891 if (!adev->enable_mes_kiq) { 1892 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0); 1893 if (r) { 1894 DRM_ERROR("Failed to init KIQ BOs!\n"); 1895 return r; 1896 } 1897 1898 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 1899 if (r) 1900 return r; 1901 } 1902 1903 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0); 1904 if (r) 1905 return r; 1906 1907 /* allocate visible FB for rlc auto-loading fw */ 1908 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 1909 r = gfx_v11_0_rlc_autoload_buffer_init(adev); 1910 if (r) 1911 return r; 1912 } 1913 1914 r = gfx_v11_0_gpu_early_init(adev); 1915 if (r) 1916 return r; 1917 1918 if (amdgpu_gfx_ras_sw_init(adev)) { 1919 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 1920 return -EINVAL; 1921 } 1922 1923 r = amdgpu_gfx_sysfs_init(adev); 1924 if (r) 1925 return r; 1926 1927 gfx_v11_0_alloc_ip_dump(adev); 1928 1929 adev->gfx.me.use_mmio_for_reset = false; 1930 adev->gfx.mec.use_mmio_for_reset = true; 1931 1932 mutex_init(&adev->gfx.mec.reset_mutex); 1933 1934 return 0; 1935 } 1936 1937 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev) 1938 { 1939 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj, 1940 &adev->gfx.pfp.pfp_fw_gpu_addr, 1941 (void **)&adev->gfx.pfp.pfp_fw_ptr); 1942 1943 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj, 1944 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 1945 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 1946 } 1947 1948 static void gfx_v11_0_me_fini(struct amdgpu_device *adev) 1949 { 1950 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj, 1951 &adev->gfx.me.me_fw_gpu_addr, 1952 (void **)&adev->gfx.me.me_fw_ptr); 1953 1954 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj, 1955 &adev->gfx.me.me_fw_data_gpu_addr, 1956 (void **)&adev->gfx.me.me_fw_data_ptr); 1957 } 1958 1959 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev) 1960 { 1961 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo, 1962 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1963 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1964 } 1965 1966 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block) 1967 { 1968 int i; 1969 struct amdgpu_device *adev = ip_block->adev; 1970 1971 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 1972 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 1973 for (i = 0; i < adev->gfx.num_compute_rings; i++) 1974 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 1975 1976 amdgpu_gfx_mqd_sw_fini(adev, 0); 1977 1978 if (!adev->enable_mes_kiq) { 1979 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 1980 amdgpu_gfx_kiq_fini(adev, 0); 1981 } 1982 1983 amdgpu_gfx_cleaner_shader_sw_fini(adev); 1984 1985 gfx_v11_0_pfp_fini(adev); 1986 gfx_v11_0_me_fini(adev); 1987 gfx_v11_0_rlc_fini(adev); 1988 gfx_v11_0_mec_fini(adev); 1989 1990 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 1991 gfx_v11_0_rlc_autoload_buffer_fini(adev); 1992 1993 gfx_v11_0_free_microcode(adev); 1994 1995 amdgpu_gfx_sysfs_fini(adev); 1996 1997 kfree(adev->gfx.ip_dump_core); 1998 kfree(adev->gfx.ip_dump_compute_queues); 1999 kfree(adev->gfx.ip_dump_gfx_queues); 2000 2001 return 0; 2002 } 2003 2004 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 2005 u32 sh_num, u32 instance, int xcc_id) 2006 { 2007 u32 data; 2008 2009 if (instance == 0xffffffff) 2010 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, 2011 INSTANCE_BROADCAST_WRITES, 1); 2012 else 2013 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, 2014 instance); 2015 2016 if (se_num == 0xffffffff) 2017 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 2018 1); 2019 else 2020 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2021 2022 if (sh_num == 0xffffffff) 2023 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES, 2024 1); 2025 else 2026 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num); 2027 2028 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data); 2029 } 2030 2031 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev) 2032 { 2033 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask; 2034 2035 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE); 2036 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask, 2037 CC_GC_SA_UNIT_DISABLE, 2038 SA_DISABLE); 2039 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE); 2040 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask, 2041 GC_USER_SA_UNIT_DISABLE, 2042 SA_DISABLE); 2043 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * 2044 adev->gfx.config.max_shader_engines); 2045 2046 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask)); 2047 } 2048 2049 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2050 { 2051 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask; 2052 u32 rb_mask; 2053 2054 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE); 2055 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask, 2056 CC_RB_BACKEND_DISABLE, 2057 BACKEND_DISABLE); 2058 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE); 2059 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask, 2060 GC_USER_RB_BACKEND_DISABLE, 2061 BACKEND_DISABLE); 2062 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se * 2063 adev->gfx.config.max_shader_engines); 2064 2065 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask)); 2066 } 2067 2068 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev) 2069 { 2070 u32 rb_bitmap_per_sa; 2071 u32 rb_bitmap_width_per_sa; 2072 u32 max_sa; 2073 u32 active_sa_bitmap; 2074 u32 global_active_rb_bitmap; 2075 u32 active_rb_bitmap = 0; 2076 u32 i; 2077 2078 /* query sa bitmap from SA_UNIT_DISABLE registers */ 2079 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev); 2080 /* query rb bitmap from RB_BACKEND_DISABLE registers */ 2081 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev); 2082 2083 /* generate active rb bitmap according to active sa bitmap */ 2084 max_sa = adev->gfx.config.max_shader_engines * 2085 adev->gfx.config.max_sh_per_se; 2086 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se / 2087 adev->gfx.config.max_sh_per_se; 2088 rb_bitmap_per_sa = amdgpu_gfx_create_bitmask(rb_bitmap_width_per_sa); 2089 2090 for (i = 0; i < max_sa; i++) { 2091 if (active_sa_bitmap & (1 << i)) 2092 active_rb_bitmap |= (rb_bitmap_per_sa << (i * rb_bitmap_width_per_sa)); 2093 } 2094 2095 active_rb_bitmap &= global_active_rb_bitmap; 2096 adev->gfx.config.backend_enable_mask = active_rb_bitmap; 2097 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap); 2098 } 2099 2100 #define DEFAULT_SH_MEM_BASES (0x6000) 2101 #define LDS_APP_BASE 0x1 2102 #define SCRATCH_APP_BASE 0x2 2103 2104 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev) 2105 { 2106 int i; 2107 uint32_t sh_mem_bases; 2108 uint32_t data; 2109 2110 /* 2111 * Configure apertures: 2112 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2113 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2114 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2115 */ 2116 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) | 2117 SCRATCH_APP_BASE; 2118 2119 mutex_lock(&adev->srbm_mutex); 2120 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2121 soc21_grbm_select(adev, 0, 0, 0, i); 2122 /* CP and shaders */ 2123 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2124 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases); 2125 2126 /* Enable trap for each kfd vmid. */ 2127 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL); 2128 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 2129 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data); 2130 } 2131 soc21_grbm_select(adev, 0, 0, 0, 0); 2132 mutex_unlock(&adev->srbm_mutex); 2133 2134 /* 2135 * Initialize all compute VMIDs to have no GDS, GWS, or OA 2136 * access. These should be enabled by FW for target VMIDs. 2137 */ 2138 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2139 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0); 2140 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0); 2141 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0); 2142 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0); 2143 } 2144 } 2145 2146 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev) 2147 { 2148 int vmid; 2149 2150 /* 2151 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2152 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2153 * the driver can enable them for graphics. VMID0 should maintain 2154 * access so that HWS firmware can save/restore entries. 2155 */ 2156 for (vmid = 1; vmid < 16; vmid++) { 2157 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0); 2158 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0); 2159 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0); 2160 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0); 2161 } 2162 } 2163 2164 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev) 2165 { 2166 /* TODO: harvest feature to be added later. */ 2167 } 2168 2169 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev) 2170 { 2171 /* TCCs are global (not instanced). */ 2172 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) | 2173 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE); 2174 2175 adev->gfx.config.tcc_disabled_mask = 2176 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) | 2177 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16); 2178 } 2179 2180 static void gfx_v11_0_constants_init(struct amdgpu_device *adev) 2181 { 2182 u32 tmp; 2183 int i; 2184 2185 if (!amdgpu_sriov_vf(adev)) 2186 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2187 2188 gfx_v11_0_setup_rb(adev); 2189 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info); 2190 gfx_v11_0_get_tcc_info(adev); 2191 adev->gfx.config.pa_sc_tile_steering_override = 0; 2192 2193 /* Set whether texture coordinate truncation is conformant. */ 2194 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); 2195 adev->gfx.config.ta_cntl2_truncate_coord_mode = 2196 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); 2197 /* Program DB_RING_CONTROL for multiple GFX pipes 2198 * Default power up value is 1. 2199 * Possible values: 2200 * 0 - split occlusion counters between gfx pipes 2201 * 1 - all occlusion counters to pipe 0 2202 * 2 - all occlusion counters to pipe 1 2203 */ 2204 WREG32_FIELD15_PREREG(GC, 0, DB_RING_CONTROL, COUNTER_CONTROL, 2205 (adev->gfx.me.num_pipe_per_me > 1) ? 0 : 1); 2206 2207 /* XXX SH_MEM regs */ 2208 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2209 mutex_lock(&adev->srbm_mutex); 2210 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2211 soc21_grbm_select(adev, 0, 0, 0, i); 2212 /* CP and shaders */ 2213 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2214 if (i != 0) { 2215 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2216 (adev->gmc.private_aperture_start >> 48)); 2217 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2218 (adev->gmc.shared_aperture_start >> 48)); 2219 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp); 2220 } 2221 } 2222 soc21_grbm_select(adev, 0, 0, 0, 0); 2223 2224 mutex_unlock(&adev->srbm_mutex); 2225 2226 gfx_v11_0_init_compute_vmid(adev); 2227 gfx_v11_0_init_gds_vmid(adev); 2228 } 2229 2230 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev, 2231 int me, int pipe) 2232 { 2233 if (me != 0) 2234 return 0; 2235 2236 switch (pipe) { 2237 case 0: 2238 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 2239 case 1: 2240 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 2241 default: 2242 return 0; 2243 } 2244 } 2245 2246 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev, 2247 int me, int pipe) 2248 { 2249 /* 2250 * amdgpu controls only the first MEC. That's why this function only 2251 * handles the setting of interrupts for this specific MEC. All other 2252 * pipes' interrupts are set by amdkfd. 2253 */ 2254 if (me != 1) 2255 return 0; 2256 2257 switch (pipe) { 2258 case 0: 2259 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 2260 case 1: 2261 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 2262 case 2: 2263 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 2264 case 3: 2265 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 2266 default: 2267 return 0; 2268 } 2269 } 2270 2271 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2272 bool enable) 2273 { 2274 u32 tmp, cp_int_cntl_reg; 2275 int i, j; 2276 2277 if (amdgpu_sriov_vf(adev)) 2278 return; 2279 2280 for (i = 0; i < adev->gfx.me.num_me; i++) { 2281 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 2282 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 2283 2284 if (cp_int_cntl_reg) { 2285 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 2286 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 2287 enable ? 1 : 0); 2288 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 2289 enable ? 1 : 0); 2290 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 2291 enable ? 1 : 0); 2292 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 2293 enable ? 1 : 0); 2294 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp); 2295 } 2296 } 2297 } 2298 } 2299 2300 static int gfx_v11_0_init_csb(struct amdgpu_device *adev) 2301 { 2302 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2303 2304 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI, 2305 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2306 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO, 2307 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2308 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); 2309 2310 return 0; 2311 } 2312 2313 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev) 2314 { 2315 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL); 2316 2317 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); 2318 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp); 2319 } 2320 2321 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev) 2322 { 2323 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2324 udelay(50); 2325 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2326 udelay(50); 2327 } 2328 2329 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev, 2330 bool enable) 2331 { 2332 uint32_t rlc_pg_cntl; 2333 2334 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 2335 2336 if (!enable) { 2337 /* RLC_PG_CNTL[23] = 0 (default) 2338 * RLC will wait for handshake acks with SMU 2339 * GFXOFF will be enabled 2340 * RLC_PG_CNTL[23] = 1 2341 * RLC will not issue any message to SMU 2342 * hence no handshake between SMU & RLC 2343 * GFXOFF will be disabled 2344 */ 2345 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2346 } else 2347 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2348 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl); 2349 } 2350 2351 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev) 2352 { 2353 /* TODO: enable rlc & smu handshake until smu 2354 * and gfxoff feature works as expected */ 2355 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK)) 2356 gfx_v11_0_rlc_smu_handshake_cntl(adev, false); 2357 2358 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 2359 udelay(50); 2360 } 2361 2362 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev) 2363 { 2364 uint32_t tmp; 2365 2366 /* enable Save Restore Machine */ 2367 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL)); 2368 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2369 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK; 2370 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp); 2371 } 2372 2373 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev) 2374 { 2375 const struct rlc_firmware_header_v2_0 *hdr; 2376 const __le32 *fw_data; 2377 unsigned i, fw_size; 2378 2379 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2380 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2381 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2382 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 2383 2384 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, 2385 RLCG_UCODE_LOADING_START_ADDRESS); 2386 2387 for (i = 0; i < fw_size; i++) 2388 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA, 2389 le32_to_cpup(fw_data++)); 2390 2391 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 2392 } 2393 2394 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev) 2395 { 2396 const struct rlc_firmware_header_v2_2 *hdr; 2397 const __le32 *fw_data; 2398 unsigned i, fw_size; 2399 u32 tmp; 2400 2401 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 2402 2403 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2404 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes)); 2405 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4; 2406 2407 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0); 2408 2409 for (i = 0; i < fw_size; i++) { 2410 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2411 msleep(1); 2412 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA, 2413 le32_to_cpup(fw_data++)); 2414 } 2415 2416 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2417 2418 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2419 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes)); 2420 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4; 2421 2422 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0); 2423 for (i = 0; i < fw_size; i++) { 2424 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2425 msleep(1); 2426 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA, 2427 le32_to_cpup(fw_data++)); 2428 } 2429 2430 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2431 2432 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL); 2433 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1); 2434 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0); 2435 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp); 2436 } 2437 2438 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev) 2439 { 2440 const struct rlc_firmware_header_v2_3 *hdr; 2441 const __le32 *fw_data; 2442 unsigned i, fw_size; 2443 u32 tmp; 2444 2445 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data; 2446 2447 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2448 le32_to_cpu(hdr->rlcp_ucode_offset_bytes)); 2449 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4; 2450 2451 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0); 2452 2453 for (i = 0; i < fw_size; i++) { 2454 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2455 msleep(1); 2456 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA, 2457 le32_to_cpup(fw_data++)); 2458 } 2459 2460 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version); 2461 2462 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE); 2463 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1); 2464 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp); 2465 2466 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2467 le32_to_cpu(hdr->rlcv_ucode_offset_bytes)); 2468 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4; 2469 2470 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0); 2471 2472 for (i = 0; i < fw_size; i++) { 2473 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2474 msleep(1); 2475 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA, 2476 le32_to_cpup(fw_data++)); 2477 } 2478 2479 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version); 2480 2481 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL); 2482 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1); 2483 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp); 2484 } 2485 2486 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev) 2487 { 2488 const struct rlc_firmware_header_v2_0 *hdr; 2489 uint16_t version_major; 2490 uint16_t version_minor; 2491 2492 if (!adev->gfx.rlc_fw) 2493 return -EINVAL; 2494 2495 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2496 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2497 2498 version_major = le16_to_cpu(hdr->header.header_version_major); 2499 version_minor = le16_to_cpu(hdr->header.header_version_minor); 2500 2501 if (version_major == 2) { 2502 gfx_v11_0_load_rlcg_microcode(adev); 2503 if (amdgpu_dpm == 1) { 2504 if (version_minor >= 2) 2505 gfx_v11_0_load_rlc_iram_dram_microcode(adev); 2506 if (version_minor == 3) 2507 gfx_v11_0_load_rlcp_rlcv_microcode(adev); 2508 } 2509 2510 return 0; 2511 } 2512 2513 return -EINVAL; 2514 } 2515 2516 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev) 2517 { 2518 int r; 2519 2520 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 2521 gfx_v11_0_init_csb(adev); 2522 2523 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */ 2524 gfx_v11_0_rlc_enable_srm(adev); 2525 } else { 2526 if (amdgpu_sriov_vf(adev)) { 2527 gfx_v11_0_init_csb(adev); 2528 return 0; 2529 } 2530 2531 adev->gfx.rlc.funcs->stop(adev); 2532 2533 /* disable CG */ 2534 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0); 2535 2536 /* disable PG */ 2537 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0); 2538 2539 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 2540 /* legacy rlc firmware loading */ 2541 r = gfx_v11_0_rlc_load_microcode(adev); 2542 if (r) 2543 return r; 2544 } 2545 2546 gfx_v11_0_init_csb(adev); 2547 2548 adev->gfx.rlc.funcs->start(adev); 2549 } 2550 return 0; 2551 } 2552 2553 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr) 2554 { 2555 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2556 uint32_t tmp; 2557 int i; 2558 2559 /* Trigger an invalidation of the L1 instruction caches */ 2560 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2561 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2562 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2563 2564 /* Wait for invalidation complete */ 2565 for (i = 0; i < usec_timeout; i++) { 2566 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2567 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2568 INVALIDATE_CACHE_COMPLETE)) 2569 break; 2570 udelay(1); 2571 } 2572 2573 if (i >= usec_timeout) { 2574 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2575 return -EINVAL; 2576 } 2577 2578 if (amdgpu_emu_mode == 1) 2579 amdgpu_device_flush_hdp(adev, NULL); 2580 2581 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2582 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2583 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2584 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2585 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2586 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2587 2588 /* Program me ucode address into intruction cache address register */ 2589 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2590 lower_32_bits(addr) & 0xFFFFF000); 2591 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2592 upper_32_bits(addr)); 2593 2594 return 0; 2595 } 2596 2597 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr) 2598 { 2599 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2600 uint32_t tmp; 2601 int i; 2602 2603 /* Trigger an invalidation of the L1 instruction caches */ 2604 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2605 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2606 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2607 2608 /* Wait for invalidation complete */ 2609 for (i = 0; i < usec_timeout; i++) { 2610 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2611 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2612 INVALIDATE_CACHE_COMPLETE)) 2613 break; 2614 udelay(1); 2615 } 2616 2617 if (i >= usec_timeout) { 2618 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2619 return -EINVAL; 2620 } 2621 2622 if (amdgpu_emu_mode == 1) 2623 amdgpu_device_flush_hdp(adev, NULL); 2624 2625 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2626 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2627 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2628 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2629 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2630 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2631 2632 /* Program pfp ucode address into intruction cache address register */ 2633 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2634 lower_32_bits(addr) & 0xFFFFF000); 2635 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2636 upper_32_bits(addr)); 2637 2638 return 0; 2639 } 2640 2641 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr) 2642 { 2643 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2644 uint32_t tmp; 2645 int i; 2646 2647 /* Trigger an invalidation of the L1 instruction caches */ 2648 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2649 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2650 2651 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2652 2653 /* Wait for invalidation complete */ 2654 for (i = 0; i < usec_timeout; i++) { 2655 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2656 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2657 INVALIDATE_CACHE_COMPLETE)) 2658 break; 2659 udelay(1); 2660 } 2661 2662 if (i >= usec_timeout) { 2663 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2664 return -EINVAL; 2665 } 2666 2667 if (amdgpu_emu_mode == 1) 2668 amdgpu_device_flush_hdp(adev, NULL); 2669 2670 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2671 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2672 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2673 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2674 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2675 2676 /* Program mec1 ucode address into intruction cache address register */ 2677 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, 2678 lower_32_bits(addr) & 0xFFFFF000); 2679 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2680 upper_32_bits(addr)); 2681 2682 return 0; 2683 } 2684 2685 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2686 { 2687 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2688 uint32_t tmp; 2689 unsigned i, pipe_id; 2690 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2691 2692 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 2693 adev->gfx.pfp_fw->data; 2694 2695 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2696 lower_32_bits(addr)); 2697 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2698 upper_32_bits(addr)); 2699 2700 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2701 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2702 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2703 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2704 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2705 2706 /* 2707 * Programming any of the CP_PFP_IC_BASE registers 2708 * forces invalidation of the ME L1 I$. Wait for the 2709 * invalidation complete 2710 */ 2711 for (i = 0; i < usec_timeout; i++) { 2712 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2713 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2714 INVALIDATE_CACHE_COMPLETE)) 2715 break; 2716 udelay(1); 2717 } 2718 2719 if (i >= usec_timeout) { 2720 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2721 return -EINVAL; 2722 } 2723 2724 /* Prime the L1 instruction caches */ 2725 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2726 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 2727 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2728 /* Waiting for cache primed*/ 2729 for (i = 0; i < usec_timeout; i++) { 2730 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2731 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2732 ICACHE_PRIMED)) 2733 break; 2734 udelay(1); 2735 } 2736 2737 if (i >= usec_timeout) { 2738 dev_err(adev->dev, "failed to prime instruction cache\n"); 2739 return -EINVAL; 2740 } 2741 2742 mutex_lock(&adev->srbm_mutex); 2743 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2744 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2745 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 2746 (pfp_hdr->ucode_start_addr_hi << 30) | 2747 (pfp_hdr->ucode_start_addr_lo >> 2)); 2748 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 2749 pfp_hdr->ucode_start_addr_hi >> 2); 2750 2751 /* 2752 * Program CP_ME_CNTL to reset given PIPE to take 2753 * effect of CP_PFP_PRGRM_CNTR_START. 2754 */ 2755 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2756 if (pipe_id == 0) 2757 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2758 PFP_PIPE0_RESET, 1); 2759 else 2760 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2761 PFP_PIPE1_RESET, 1); 2762 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2763 2764 /* Clear pfp pipe0 reset bit. */ 2765 if (pipe_id == 0) 2766 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2767 PFP_PIPE0_RESET, 0); 2768 else 2769 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2770 PFP_PIPE1_RESET, 0); 2771 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2772 2773 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 2774 lower_32_bits(addr2)); 2775 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 2776 upper_32_bits(addr2)); 2777 } 2778 soc21_grbm_select(adev, 0, 0, 0, 0); 2779 mutex_unlock(&adev->srbm_mutex); 2780 2781 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2782 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2783 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2784 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2785 2786 /* Invalidate the data caches */ 2787 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2788 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2789 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2790 2791 for (i = 0; i < usec_timeout; i++) { 2792 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2793 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2794 INVALIDATE_DCACHE_COMPLETE)) 2795 break; 2796 udelay(1); 2797 } 2798 2799 if (i >= usec_timeout) { 2800 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2801 return -EINVAL; 2802 } 2803 2804 return 0; 2805 } 2806 2807 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2808 { 2809 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2810 uint32_t tmp; 2811 unsigned i, pipe_id; 2812 const struct gfx_firmware_header_v2_0 *me_hdr; 2813 2814 me_hdr = (const struct gfx_firmware_header_v2_0 *) 2815 adev->gfx.me_fw->data; 2816 2817 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2818 lower_32_bits(addr)); 2819 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2820 upper_32_bits(addr)); 2821 2822 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2823 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2824 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2825 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2826 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2827 2828 /* 2829 * Programming any of the CP_ME_IC_BASE registers 2830 * forces invalidation of the ME L1 I$. Wait for the 2831 * invalidation complete 2832 */ 2833 for (i = 0; i < usec_timeout; i++) { 2834 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2835 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2836 INVALIDATE_CACHE_COMPLETE)) 2837 break; 2838 udelay(1); 2839 } 2840 2841 if (i >= usec_timeout) { 2842 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2843 return -EINVAL; 2844 } 2845 2846 /* Prime the instruction caches */ 2847 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2848 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 2849 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2850 2851 /* Waiting for instruction cache primed*/ 2852 for (i = 0; i < usec_timeout; i++) { 2853 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2854 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2855 ICACHE_PRIMED)) 2856 break; 2857 udelay(1); 2858 } 2859 2860 if (i >= usec_timeout) { 2861 dev_err(adev->dev, "failed to prime instruction cache\n"); 2862 return -EINVAL; 2863 } 2864 2865 mutex_lock(&adev->srbm_mutex); 2866 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2867 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2868 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 2869 (me_hdr->ucode_start_addr_hi << 30) | 2870 (me_hdr->ucode_start_addr_lo >> 2) ); 2871 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 2872 me_hdr->ucode_start_addr_hi>>2); 2873 2874 /* 2875 * Program CP_ME_CNTL to reset given PIPE to take 2876 * effect of CP_PFP_PRGRM_CNTR_START. 2877 */ 2878 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2879 if (pipe_id == 0) 2880 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2881 ME_PIPE0_RESET, 1); 2882 else 2883 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2884 ME_PIPE1_RESET, 1); 2885 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2886 2887 /* Clear pfp pipe0 reset bit. */ 2888 if (pipe_id == 0) 2889 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2890 ME_PIPE0_RESET, 0); 2891 else 2892 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2893 ME_PIPE1_RESET, 0); 2894 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2895 2896 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 2897 lower_32_bits(addr2)); 2898 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 2899 upper_32_bits(addr2)); 2900 } 2901 soc21_grbm_select(adev, 0, 0, 0, 0); 2902 mutex_unlock(&adev->srbm_mutex); 2903 2904 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2905 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2906 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2907 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2908 2909 /* Invalidate the data caches */ 2910 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2911 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2912 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2913 2914 for (i = 0; i < usec_timeout; i++) { 2915 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2916 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2917 INVALIDATE_DCACHE_COMPLETE)) 2918 break; 2919 udelay(1); 2920 } 2921 2922 if (i >= usec_timeout) { 2923 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2924 return -EINVAL; 2925 } 2926 2927 return 0; 2928 } 2929 2930 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2931 { 2932 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2933 uint32_t tmp; 2934 unsigned i; 2935 const struct gfx_firmware_header_v2_0 *mec_hdr; 2936 2937 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2938 adev->gfx.mec_fw->data; 2939 2940 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2941 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 2942 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2943 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2944 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2945 2946 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 2947 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 2948 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 2949 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 2950 2951 mutex_lock(&adev->srbm_mutex); 2952 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 2953 soc21_grbm_select(adev, 1, i, 0, 0); 2954 2955 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2); 2956 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 2957 upper_32_bits(addr2)); 2958 2959 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 2960 mec_hdr->ucode_start_addr_lo >> 2 | 2961 mec_hdr->ucode_start_addr_hi << 30); 2962 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2963 mec_hdr->ucode_start_addr_hi >> 2); 2964 2965 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr); 2966 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2967 upper_32_bits(addr)); 2968 } 2969 mutex_unlock(&adev->srbm_mutex); 2970 soc21_grbm_select(adev, 0, 0, 0, 0); 2971 2972 /* Trigger an invalidation of the L1 instruction caches */ 2973 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2974 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2975 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 2976 2977 /* Wait for invalidation complete */ 2978 for (i = 0; i < usec_timeout; i++) { 2979 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2980 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 2981 INVALIDATE_DCACHE_COMPLETE)) 2982 break; 2983 udelay(1); 2984 } 2985 2986 if (i >= usec_timeout) { 2987 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2988 return -EINVAL; 2989 } 2990 2991 /* Trigger an invalidation of the L1 instruction caches */ 2992 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2993 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2994 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2995 2996 /* Wait for invalidation complete */ 2997 for (i = 0; i < usec_timeout; i++) { 2998 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2999 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 3000 INVALIDATE_CACHE_COMPLETE)) 3001 break; 3002 udelay(1); 3003 } 3004 3005 if (i >= usec_timeout) { 3006 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3007 return -EINVAL; 3008 } 3009 3010 return 0; 3011 } 3012 3013 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev) 3014 { 3015 const struct gfx_firmware_header_v2_0 *pfp_hdr; 3016 const struct gfx_firmware_header_v2_0 *me_hdr; 3017 const struct gfx_firmware_header_v2_0 *mec_hdr; 3018 uint32_t pipe_id, tmp; 3019 3020 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 3021 adev->gfx.mec_fw->data; 3022 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3023 adev->gfx.me_fw->data; 3024 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3025 adev->gfx.pfp_fw->data; 3026 3027 /* config pfp program start addr */ 3028 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3029 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3030 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3031 (pfp_hdr->ucode_start_addr_hi << 30) | 3032 (pfp_hdr->ucode_start_addr_lo >> 2)); 3033 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3034 pfp_hdr->ucode_start_addr_hi >> 2); 3035 } 3036 soc21_grbm_select(adev, 0, 0, 0, 0); 3037 3038 /* reset pfp pipe */ 3039 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3040 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1); 3041 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1); 3042 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3043 3044 /* clear pfp pipe reset */ 3045 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0); 3046 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0); 3047 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3048 3049 /* config me program start addr */ 3050 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3051 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3052 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3053 (me_hdr->ucode_start_addr_hi << 30) | 3054 (me_hdr->ucode_start_addr_lo >> 2) ); 3055 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3056 me_hdr->ucode_start_addr_hi>>2); 3057 } 3058 soc21_grbm_select(adev, 0, 0, 0, 0); 3059 3060 /* reset me pipe */ 3061 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3062 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1); 3063 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1); 3064 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3065 3066 /* clear me pipe reset */ 3067 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0); 3068 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0); 3069 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3070 3071 /* config mec program start addr */ 3072 for (pipe_id = 0; pipe_id < 4; pipe_id++) { 3073 soc21_grbm_select(adev, 1, pipe_id, 0, 0); 3074 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 3075 mec_hdr->ucode_start_addr_lo >> 2 | 3076 mec_hdr->ucode_start_addr_hi << 30); 3077 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 3078 mec_hdr->ucode_start_addr_hi >> 2); 3079 } 3080 soc21_grbm_select(adev, 0, 0, 0, 0); 3081 3082 /* reset mec pipe */ 3083 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3084 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1); 3085 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1); 3086 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1); 3087 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1); 3088 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3089 3090 /* clear mec pipe reset */ 3091 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0); 3092 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0); 3093 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0); 3094 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0); 3095 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3096 } 3097 3098 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) 3099 { 3100 uint32_t cp_status; 3101 uint32_t bootload_status; 3102 int i, r; 3103 uint64_t addr, addr2; 3104 3105 for (i = 0; i < adev->usec_timeout; i++) { 3106 cp_status = RREG32_SOC15(GC, 0, regCP_STAT); 3107 3108 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 3109 IP_VERSION(11, 0, 1) || 3110 amdgpu_ip_version(adev, GC_HWIP, 0) == 3111 IP_VERSION(11, 0, 4) || 3112 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) || 3113 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) || 3114 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) || 3115 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3) || 3116 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4) || 3117 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6) || 3118 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 0) || 3119 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 7, 1)) 3120 bootload_status = RREG32_SOC15(GC, 0, 3121 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1); 3122 else 3123 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS); 3124 3125 if ((cp_status == 0) && 3126 (REG_GET_FIELD(bootload_status, 3127 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) { 3128 break; 3129 } 3130 udelay(1); 3131 } 3132 3133 if (i >= adev->usec_timeout) { 3134 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n"); 3135 return -ETIMEDOUT; 3136 } 3137 3138 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 3139 if (adev->gfx.rs64_enable) { 3140 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3141 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset; 3142 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3143 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset; 3144 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2); 3145 if (r) 3146 return r; 3147 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3148 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset; 3149 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3150 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset; 3151 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2); 3152 if (r) 3153 return r; 3154 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3155 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset; 3156 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3157 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset; 3158 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2); 3159 if (r) 3160 return r; 3161 } else { 3162 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3163 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset; 3164 r = gfx_v11_0_config_me_cache(adev, addr); 3165 if (r) 3166 return r; 3167 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3168 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset; 3169 r = gfx_v11_0_config_pfp_cache(adev, addr); 3170 if (r) 3171 return r; 3172 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3173 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset; 3174 r = gfx_v11_0_config_mec_cache(adev, addr); 3175 if (r) 3176 return r; 3177 } 3178 } 3179 3180 return 0; 3181 } 3182 3183 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3184 { 3185 int i; 3186 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3187 3188 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3189 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3190 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3191 3192 for (i = 0; i < adev->usec_timeout; i++) { 3193 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0) 3194 break; 3195 udelay(1); 3196 } 3197 3198 if (i >= adev->usec_timeout) 3199 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt"); 3200 3201 return 0; 3202 } 3203 3204 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev) 3205 { 3206 int r; 3207 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3208 const __le32 *fw_data; 3209 unsigned i, fw_size; 3210 3211 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3212 adev->gfx.pfp_fw->data; 3213 3214 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3215 3216 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3217 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3218 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes); 3219 3220 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes, 3221 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3222 &adev->gfx.pfp.pfp_fw_obj, 3223 &adev->gfx.pfp.pfp_fw_gpu_addr, 3224 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3225 if (r) { 3226 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r); 3227 gfx_v11_0_pfp_fini(adev); 3228 return r; 3229 } 3230 3231 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size); 3232 3233 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3234 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3235 3236 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr); 3237 3238 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0); 3239 3240 for (i = 0; i < pfp_hdr->jt_size; i++) 3241 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA, 3242 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i)); 3243 3244 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3245 3246 return 0; 3247 } 3248 3249 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) 3250 { 3251 int r; 3252 const struct gfx_firmware_header_v2_0 *pfp_hdr; 3253 const __le32 *fw_ucode, *fw_data; 3254 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3255 uint32_t tmp; 3256 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3257 3258 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3259 adev->gfx.pfp_fw->data; 3260 3261 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3262 3263 /* instruction */ 3264 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data + 3265 le32_to_cpu(pfp_hdr->ucode_offset_bytes)); 3266 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes); 3267 /* data */ 3268 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3269 le32_to_cpu(pfp_hdr->data_offset_bytes)); 3270 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes); 3271 3272 /* 64kb align */ 3273 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3274 64 * 1024, 3275 AMDGPU_GEM_DOMAIN_VRAM | 3276 AMDGPU_GEM_DOMAIN_GTT, 3277 &adev->gfx.pfp.pfp_fw_obj, 3278 &adev->gfx.pfp.pfp_fw_gpu_addr, 3279 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3280 if (r) { 3281 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r); 3282 gfx_v11_0_pfp_fini(adev); 3283 return r; 3284 } 3285 3286 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3287 64 * 1024, 3288 AMDGPU_GEM_DOMAIN_VRAM | 3289 AMDGPU_GEM_DOMAIN_GTT, 3290 &adev->gfx.pfp.pfp_fw_data_obj, 3291 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 3292 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 3293 if (r) { 3294 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r); 3295 gfx_v11_0_pfp_fini(adev); 3296 return r; 3297 } 3298 3299 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size); 3300 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); 3301 3302 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3303 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj); 3304 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3305 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj); 3306 3307 if (amdgpu_emu_mode == 1) 3308 amdgpu_device_flush_hdp(adev, NULL); 3309 3310 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 3311 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3312 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 3313 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3314 3315 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 3316 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 3317 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 3318 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 3319 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 3320 3321 /* 3322 * Programming any of the CP_PFP_IC_BASE registers 3323 * forces invalidation of the ME L1 I$. Wait for the 3324 * invalidation complete 3325 */ 3326 for (i = 0; i < usec_timeout; i++) { 3327 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3328 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3329 INVALIDATE_CACHE_COMPLETE)) 3330 break; 3331 udelay(1); 3332 } 3333 3334 if (i >= usec_timeout) { 3335 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3336 return -EINVAL; 3337 } 3338 3339 /* Prime the L1 instruction caches */ 3340 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3341 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 3342 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 3343 /* Waiting for cache primed*/ 3344 for (i = 0; i < usec_timeout; i++) { 3345 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3346 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3347 ICACHE_PRIMED)) 3348 break; 3349 udelay(1); 3350 } 3351 3352 if (i >= usec_timeout) { 3353 dev_err(adev->dev, "failed to prime instruction cache\n"); 3354 return -EINVAL; 3355 } 3356 3357 mutex_lock(&adev->srbm_mutex); 3358 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3359 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3360 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3361 (pfp_hdr->ucode_start_addr_hi << 30) | 3362 (pfp_hdr->ucode_start_addr_lo >> 2) ); 3363 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3364 pfp_hdr->ucode_start_addr_hi>>2); 3365 3366 /* 3367 * Program CP_ME_CNTL to reset given PIPE to take 3368 * effect of CP_PFP_PRGRM_CNTR_START. 3369 */ 3370 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3371 if (pipe_id == 0) 3372 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3373 PFP_PIPE0_RESET, 1); 3374 else 3375 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3376 PFP_PIPE1_RESET, 1); 3377 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3378 3379 /* Clear pfp pipe0 reset bit. */ 3380 if (pipe_id == 0) 3381 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3382 PFP_PIPE0_RESET, 0); 3383 else 3384 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3385 PFP_PIPE1_RESET, 0); 3386 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3387 3388 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 3389 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3390 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 3391 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3392 } 3393 soc21_grbm_select(adev, 0, 0, 0, 0); 3394 mutex_unlock(&adev->srbm_mutex); 3395 3396 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3397 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3398 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3399 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3400 3401 /* Invalidate the data caches */ 3402 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3403 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3404 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3405 3406 for (i = 0; i < usec_timeout; i++) { 3407 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3408 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3409 INVALIDATE_DCACHE_COMPLETE)) 3410 break; 3411 udelay(1); 3412 } 3413 3414 if (i >= usec_timeout) { 3415 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3416 return -EINVAL; 3417 } 3418 3419 return 0; 3420 } 3421 3422 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev) 3423 { 3424 int r; 3425 const struct gfx_firmware_header_v1_0 *me_hdr; 3426 const __le32 *fw_data; 3427 unsigned i, fw_size; 3428 3429 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3430 adev->gfx.me_fw->data; 3431 3432 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3433 3434 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3435 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3436 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes); 3437 3438 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes, 3439 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3440 &adev->gfx.me.me_fw_obj, 3441 &adev->gfx.me.me_fw_gpu_addr, 3442 (void **)&adev->gfx.me.me_fw_ptr); 3443 if (r) { 3444 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r); 3445 gfx_v11_0_me_fini(adev); 3446 return r; 3447 } 3448 3449 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size); 3450 3451 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3452 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3453 3454 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr); 3455 3456 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0); 3457 3458 for (i = 0; i < me_hdr->jt_size; i++) 3459 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA, 3460 le32_to_cpup(fw_data + me_hdr->jt_offset + i)); 3461 3462 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version); 3463 3464 return 0; 3465 } 3466 3467 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) 3468 { 3469 int r; 3470 const struct gfx_firmware_header_v2_0 *me_hdr; 3471 const __le32 *fw_ucode, *fw_data; 3472 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3473 uint32_t tmp; 3474 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3475 3476 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3477 adev->gfx.me_fw->data; 3478 3479 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3480 3481 /* instruction */ 3482 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data + 3483 le32_to_cpu(me_hdr->ucode_offset_bytes)); 3484 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes); 3485 /* data */ 3486 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3487 le32_to_cpu(me_hdr->data_offset_bytes)); 3488 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes); 3489 3490 /* 64kb align*/ 3491 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3492 64 * 1024, 3493 AMDGPU_GEM_DOMAIN_VRAM | 3494 AMDGPU_GEM_DOMAIN_GTT, 3495 &adev->gfx.me.me_fw_obj, 3496 &adev->gfx.me.me_fw_gpu_addr, 3497 (void **)&adev->gfx.me.me_fw_ptr); 3498 if (r) { 3499 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r); 3500 gfx_v11_0_me_fini(adev); 3501 return r; 3502 } 3503 3504 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3505 64 * 1024, 3506 AMDGPU_GEM_DOMAIN_VRAM | 3507 AMDGPU_GEM_DOMAIN_GTT, 3508 &adev->gfx.me.me_fw_data_obj, 3509 &adev->gfx.me.me_fw_data_gpu_addr, 3510 (void **)&adev->gfx.me.me_fw_data_ptr); 3511 if (r) { 3512 dev_err(adev->dev, "(%d) failed to create me data bo\n", r); 3513 gfx_v11_0_pfp_fini(adev); 3514 return r; 3515 } 3516 3517 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size); 3518 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); 3519 3520 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3521 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj); 3522 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3523 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj); 3524 3525 if (amdgpu_emu_mode == 1) 3526 amdgpu_device_flush_hdp(adev, NULL); 3527 3528 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 3529 lower_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3530 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 3531 upper_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3532 3533 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 3534 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 3535 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 3536 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 3537 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 3538 3539 /* 3540 * Programming any of the CP_ME_IC_BASE registers 3541 * forces invalidation of the ME L1 I$. Wait for the 3542 * invalidation complete 3543 */ 3544 for (i = 0; i < usec_timeout; i++) { 3545 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3546 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3547 INVALIDATE_CACHE_COMPLETE)) 3548 break; 3549 udelay(1); 3550 } 3551 3552 if (i >= usec_timeout) { 3553 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3554 return -EINVAL; 3555 } 3556 3557 /* Prime the instruction caches */ 3558 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3559 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 3560 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 3561 3562 /* Waiting for instruction cache primed*/ 3563 for (i = 0; i < usec_timeout; i++) { 3564 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3565 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3566 ICACHE_PRIMED)) 3567 break; 3568 udelay(1); 3569 } 3570 3571 if (i >= usec_timeout) { 3572 dev_err(adev->dev, "failed to prime instruction cache\n"); 3573 return -EINVAL; 3574 } 3575 3576 mutex_lock(&adev->srbm_mutex); 3577 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3578 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3579 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3580 (me_hdr->ucode_start_addr_hi << 30) | 3581 (me_hdr->ucode_start_addr_lo >> 2) ); 3582 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3583 me_hdr->ucode_start_addr_hi>>2); 3584 3585 /* 3586 * Program CP_ME_CNTL to reset given PIPE to take 3587 * effect of CP_PFP_PRGRM_CNTR_START. 3588 */ 3589 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3590 if (pipe_id == 0) 3591 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3592 ME_PIPE0_RESET, 1); 3593 else 3594 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3595 ME_PIPE1_RESET, 1); 3596 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3597 3598 /* Clear pfp pipe0 reset bit. */ 3599 if (pipe_id == 0) 3600 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3601 ME_PIPE0_RESET, 0); 3602 else 3603 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3604 ME_PIPE1_RESET, 0); 3605 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3606 3607 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 3608 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3609 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 3610 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3611 } 3612 soc21_grbm_select(adev, 0, 0, 0, 0); 3613 mutex_unlock(&adev->srbm_mutex); 3614 3615 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3616 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3617 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3618 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3619 3620 /* Invalidate the data caches */ 3621 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3622 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3623 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3624 3625 for (i = 0; i < usec_timeout; i++) { 3626 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3627 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3628 INVALIDATE_DCACHE_COMPLETE)) 3629 break; 3630 udelay(1); 3631 } 3632 3633 if (i >= usec_timeout) { 3634 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3635 return -EINVAL; 3636 } 3637 3638 return 0; 3639 } 3640 3641 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3642 { 3643 int r; 3644 3645 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw) 3646 return -EINVAL; 3647 3648 gfx_v11_0_cp_gfx_enable(adev, false); 3649 3650 if (adev->gfx.rs64_enable) 3651 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev); 3652 else 3653 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev); 3654 if (r) { 3655 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r); 3656 return r; 3657 } 3658 3659 if (adev->gfx.rs64_enable) 3660 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev); 3661 else 3662 r = gfx_v11_0_cp_gfx_load_me_microcode(adev); 3663 if (r) { 3664 dev_err(adev->dev, "(%d) failed to load me fw\n", r); 3665 return r; 3666 } 3667 3668 return 0; 3669 } 3670 3671 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev) 3672 { 3673 struct amdgpu_ring *ring; 3674 const struct cs_section_def *sect = NULL; 3675 const struct cs_extent_def *ext = NULL; 3676 int r, i; 3677 int ctx_reg_offset; 3678 3679 /* init the CP */ 3680 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT, 3681 adev->gfx.config.max_hw_contexts - 1); 3682 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1); 3683 3684 if (!amdgpu_async_gfx_ring) 3685 gfx_v11_0_cp_gfx_enable(adev, true); 3686 3687 ring = &adev->gfx.gfx_ring[0]; 3688 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev)); 3689 if (r) { 3690 drm_err(&adev->ddev, "cp failed to lock ring (%d).\n", r); 3691 return r; 3692 } 3693 3694 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3695 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3696 3697 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3698 amdgpu_ring_write(ring, 0x80000000); 3699 amdgpu_ring_write(ring, 0x80000000); 3700 3701 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 3702 for (ext = sect->section; ext->extent != NULL; ++ext) { 3703 if (sect->id == SECT_CONTEXT) { 3704 amdgpu_ring_write(ring, 3705 PACKET3(PACKET3_SET_CONTEXT_REG, 3706 ext->reg_count)); 3707 amdgpu_ring_write(ring, ext->reg_index - 3708 PACKET3_SET_CONTEXT_REG_START); 3709 for (i = 0; i < ext->reg_count; i++) 3710 amdgpu_ring_write(ring, ext->extent[i]); 3711 } 3712 } 3713 } 3714 3715 ctx_reg_offset = 3716 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 3717 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 3718 amdgpu_ring_write(ring, ctx_reg_offset); 3719 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override); 3720 3721 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3722 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3723 3724 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3725 amdgpu_ring_write(ring, 0); 3726 3727 amdgpu_ring_commit(ring); 3728 3729 /* submit cs packet to copy state 0 to next available state */ 3730 if (adev->gfx.num_gfx_rings > 1) { 3731 /* maximum supported gfx ring is 2 */ 3732 ring = &adev->gfx.gfx_ring[1]; 3733 r = amdgpu_ring_alloc(ring, 2); 3734 if (r) { 3735 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 3736 return r; 3737 } 3738 3739 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3740 amdgpu_ring_write(ring, 0); 3741 3742 amdgpu_ring_commit(ring); 3743 } 3744 return 0; 3745 } 3746 3747 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev, 3748 CP_PIPE_ID pipe) 3749 { 3750 u32 tmp; 3751 3752 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); 3753 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe); 3754 3755 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); 3756 } 3757 3758 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev, 3759 struct amdgpu_ring *ring) 3760 { 3761 u32 tmp; 3762 3763 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); 3764 if (ring->use_doorbell) { 3765 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3766 DOORBELL_OFFSET, ring->doorbell_index); 3767 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3768 DOORBELL_EN, 1); 3769 } else { 3770 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3771 DOORBELL_EN, 0); 3772 } 3773 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp); 3774 3775 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3776 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3777 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp); 3778 3779 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 3780 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3781 } 3782 3783 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev) 3784 { 3785 struct amdgpu_ring *ring; 3786 u32 tmp; 3787 u32 rb_bufsz; 3788 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3789 3790 /* Set the write pointer delay */ 3791 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0); 3792 3793 /* set the RB to use vmid 0 */ 3794 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0); 3795 3796 /* Init gfx ring 0 for pipe 0 */ 3797 mutex_lock(&adev->srbm_mutex); 3798 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3799 3800 /* Set ring buffer size */ 3801 ring = &adev->gfx.gfx_ring[0]; 3802 rb_bufsz = order_base_2(ring->ring_size / 8); 3803 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3804 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3805 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3806 3807 /* Initialize the ring buffer's write pointers */ 3808 ring->wptr = 0; 3809 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3810 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3811 3812 /* set the wb address whether it's enabled or not */ 3813 rptr_addr = ring->rptr_gpu_addr; 3814 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3815 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3816 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3817 3818 wptr_gpu_addr = ring->wptr_gpu_addr; 3819 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3820 lower_32_bits(wptr_gpu_addr)); 3821 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3822 upper_32_bits(wptr_gpu_addr)); 3823 3824 mdelay(1); 3825 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3826 3827 rb_addr = ring->gpu_addr >> 8; 3828 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr); 3829 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3830 3831 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1); 3832 3833 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3834 mutex_unlock(&adev->srbm_mutex); 3835 3836 /* Init gfx ring 1 for pipe 1 */ 3837 if (adev->gfx.num_gfx_rings > 1) { 3838 mutex_lock(&adev->srbm_mutex); 3839 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1); 3840 /* maximum supported gfx ring is 2 */ 3841 ring = &adev->gfx.gfx_ring[1]; 3842 rb_bufsz = order_base_2(ring->ring_size / 8); 3843 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz); 3844 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2); 3845 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3846 /* Initialize the ring buffer's write pointers */ 3847 ring->wptr = 0; 3848 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr)); 3849 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr)); 3850 /* Set the wb address whether it's enabled or not */ 3851 rptr_addr = ring->rptr_gpu_addr; 3852 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr)); 3853 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3854 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3855 wptr_gpu_addr = ring->wptr_gpu_addr; 3856 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3857 lower_32_bits(wptr_gpu_addr)); 3858 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3859 upper_32_bits(wptr_gpu_addr)); 3860 3861 mdelay(1); 3862 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3863 3864 rb_addr = ring->gpu_addr >> 8; 3865 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr); 3866 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr)); 3867 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1); 3868 3869 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3870 mutex_unlock(&adev->srbm_mutex); 3871 } 3872 /* Switch to pipe 0 */ 3873 mutex_lock(&adev->srbm_mutex); 3874 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3875 mutex_unlock(&adev->srbm_mutex); 3876 3877 /* start the ring */ 3878 gfx_v11_0_cp_gfx_start(adev); 3879 3880 return 0; 3881 } 3882 3883 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3884 { 3885 u32 data; 3886 3887 if (adev->gfx.rs64_enable) { 3888 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3889 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE, 3890 enable ? 0 : 1); 3891 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 3892 enable ? 0 : 1); 3893 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 3894 enable ? 0 : 1); 3895 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 3896 enable ? 0 : 1); 3897 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 3898 enable ? 0 : 1); 3899 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE, 3900 enable ? 1 : 0); 3901 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE, 3902 enable ? 1 : 0); 3903 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE, 3904 enable ? 1 : 0); 3905 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE, 3906 enable ? 1 : 0); 3907 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT, 3908 enable ? 0 : 1); 3909 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data); 3910 } else { 3911 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); 3912 3913 if (enable) { 3914 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0); 3915 if (!adev->enable_mes_kiq) 3916 data = REG_SET_FIELD(data, CP_MEC_CNTL, 3917 MEC_ME2_HALT, 0); 3918 } else { 3919 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1); 3920 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1); 3921 } 3922 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data); 3923 } 3924 3925 udelay(50); 3926 } 3927 3928 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3929 { 3930 const struct gfx_firmware_header_v1_0 *mec_hdr; 3931 const __le32 *fw_data; 3932 unsigned i, fw_size; 3933 u32 *fw = NULL; 3934 int r; 3935 3936 if (!adev->gfx.mec_fw) 3937 return -EINVAL; 3938 3939 gfx_v11_0_cp_compute_enable(adev, false); 3940 3941 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3942 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3943 3944 fw_data = (const __le32 *) 3945 (adev->gfx.mec_fw->data + 3946 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3947 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 3948 3949 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 3950 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3951 &adev->gfx.mec.mec_fw_obj, 3952 &adev->gfx.mec.mec_fw_gpu_addr, 3953 (void **)&fw); 3954 if (r) { 3955 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r); 3956 gfx_v11_0_mec_fini(adev); 3957 return r; 3958 } 3959 3960 memcpy(fw, fw_data, fw_size); 3961 3962 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 3963 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 3964 3965 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr); 3966 3967 /* MEC1 */ 3968 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0); 3969 3970 for (i = 0; i < mec_hdr->jt_size; i++) 3971 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA, 3972 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3973 3974 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version); 3975 3976 return 0; 3977 } 3978 3979 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) 3980 { 3981 const struct gfx_firmware_header_v2_0 *mec_hdr; 3982 const __le32 *fw_ucode, *fw_data; 3983 u32 tmp, fw_ucode_size, fw_data_size; 3984 u32 i, usec_timeout = 50000; /* Wait for 50 ms */ 3985 u32 *fw_ucode_ptr, *fw_data_ptr; 3986 int r; 3987 3988 if (!adev->gfx.mec_fw) 3989 return -EINVAL; 3990 3991 gfx_v11_0_cp_compute_enable(adev, false); 3992 3993 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; 3994 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3995 3996 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data + 3997 le32_to_cpu(mec_hdr->ucode_offset_bytes)); 3998 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes); 3999 4000 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 4001 le32_to_cpu(mec_hdr->data_offset_bytes)); 4002 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes); 4003 4004 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 4005 64 * 1024, 4006 AMDGPU_GEM_DOMAIN_VRAM | 4007 AMDGPU_GEM_DOMAIN_GTT, 4008 &adev->gfx.mec.mec_fw_obj, 4009 &adev->gfx.mec.mec_fw_gpu_addr, 4010 (void **)&fw_ucode_ptr); 4011 if (r) { 4012 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 4013 gfx_v11_0_mec_fini(adev); 4014 return r; 4015 } 4016 4017 r = amdgpu_bo_create_reserved(adev, fw_data_size, 4018 64 * 1024, 4019 AMDGPU_GEM_DOMAIN_VRAM | 4020 AMDGPU_GEM_DOMAIN_GTT, 4021 &adev->gfx.mec.mec_fw_data_obj, 4022 &adev->gfx.mec.mec_fw_data_gpu_addr, 4023 (void **)&fw_data_ptr); 4024 if (r) { 4025 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 4026 gfx_v11_0_mec_fini(adev); 4027 return r; 4028 } 4029 4030 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); 4031 memcpy(fw_data_ptr, fw_data, fw_data_size); 4032 4033 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 4034 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj); 4035 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 4036 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj); 4037 4038 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 4039 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 4040 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 4041 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 4042 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 4043 4044 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 4045 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 4046 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 4047 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 4048 4049 mutex_lock(&adev->srbm_mutex); 4050 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 4051 soc21_grbm_select(adev, 1, i, 0, 0); 4052 4053 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr); 4054 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 4055 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr)); 4056 4057 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 4058 mec_hdr->ucode_start_addr_lo >> 2 | 4059 mec_hdr->ucode_start_addr_hi << 30); 4060 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 4061 mec_hdr->ucode_start_addr_hi >> 2); 4062 4063 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr); 4064 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 4065 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 4066 } 4067 mutex_unlock(&adev->srbm_mutex); 4068 soc21_grbm_select(adev, 0, 0, 0, 0); 4069 4070 /* Trigger an invalidation of the L1 instruction caches */ 4071 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4072 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 4073 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 4074 4075 /* Wait for invalidation complete */ 4076 for (i = 0; i < usec_timeout; i++) { 4077 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4078 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 4079 INVALIDATE_DCACHE_COMPLETE)) 4080 break; 4081 udelay(1); 4082 } 4083 4084 if (i >= usec_timeout) { 4085 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4086 return -EINVAL; 4087 } 4088 4089 /* Trigger an invalidation of the L1 instruction caches */ 4090 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4091 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 4092 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 4093 4094 /* Wait for invalidation complete */ 4095 for (i = 0; i < usec_timeout; i++) { 4096 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4097 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 4098 INVALIDATE_CACHE_COMPLETE)) 4099 break; 4100 udelay(1); 4101 } 4102 4103 if (i >= usec_timeout) { 4104 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4105 return -EINVAL; 4106 } 4107 4108 return 0; 4109 } 4110 4111 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring) 4112 { 4113 uint32_t tmp; 4114 struct amdgpu_device *adev = ring->adev; 4115 4116 /* tell RLC which is KIQ queue */ 4117 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); 4118 tmp &= 0xffffff00; 4119 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4120 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80); 4121 } 4122 4123 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev) 4124 { 4125 /* set graphics engine doorbell range */ 4126 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, 4127 (adev->doorbell_index.gfx_ring0 * 2) << 2); 4128 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 4129 (adev->doorbell_index.gfx_userqueue_end * 2) << 2); 4130 4131 /* set compute engine doorbell range */ 4132 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4133 (adev->doorbell_index.kiq * 2) << 2); 4134 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4135 (adev->doorbell_index.userqueue_end * 2) << 2); 4136 } 4137 4138 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev, 4139 struct v11_gfx_mqd *mqd, 4140 struct amdgpu_mqd_prop *prop) 4141 { 4142 bool priority = 0; 4143 u32 tmp; 4144 4145 /* set up default queue priority level 4146 * 0x0 = low priority, 0x1 = high priority 4147 */ 4148 if (prop->hqd_queue_priority == AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM) 4149 priority = 1; 4150 4151 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT; 4152 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority); 4153 mqd->cp_gfx_hqd_queue_priority = tmp; 4154 } 4155 4156 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, 4157 struct amdgpu_mqd_prop *prop) 4158 { 4159 struct v11_gfx_mqd *mqd = m; 4160 uint64_t hqd_gpu_addr, wb_gpu_addr; 4161 uint32_t tmp; 4162 uint32_t rb_bufsz; 4163 4164 /* set up gfx hqd wptr */ 4165 mqd->cp_gfx_hqd_wptr = 0; 4166 mqd->cp_gfx_hqd_wptr_hi = 0; 4167 4168 /* set the pointer to the MQD */ 4169 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc; 4170 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4171 4172 /* set up mqd control */ 4173 tmp = regCP_GFX_MQD_CONTROL_DEFAULT; 4174 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0); 4175 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1); 4176 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0); 4177 mqd->cp_gfx_mqd_control = tmp; 4178 4179 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */ 4180 tmp = regCP_GFX_HQD_VMID_DEFAULT; 4181 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0); 4182 mqd->cp_gfx_hqd_vmid = 0; 4183 4184 /* set up gfx queue priority */ 4185 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop); 4186 4187 /* set up time quantum */ 4188 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT; 4189 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1); 4190 mqd->cp_gfx_hqd_quantum = tmp; 4191 4192 /* set up gfx hqd base. this is similar as CP_RB_BASE */ 4193 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4194 mqd->cp_gfx_hqd_base = hqd_gpu_addr; 4195 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr); 4196 4197 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */ 4198 wb_gpu_addr = prop->rptr_gpu_addr; 4199 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc; 4200 mqd->cp_gfx_hqd_rptr_addr_hi = 4201 upper_32_bits(wb_gpu_addr) & 0xffff; 4202 4203 /* set up rb_wptr_poll addr */ 4204 wb_gpu_addr = prop->wptr_gpu_addr; 4205 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4206 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4207 4208 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */ 4209 rb_bufsz = order_base_2(prop->queue_size / 4) - 1; 4210 tmp = regCP_GFX_HQD_CNTL_DEFAULT; 4211 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz); 4212 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2); 4213 #ifdef __BIG_ENDIAN 4214 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1); 4215 #endif 4216 if (prop->tmz_queue) 4217 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, TMZ_MATCH, 1); 4218 if (!prop->kernel_queue) 4219 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_NON_PRIV, 1); 4220 mqd->cp_gfx_hqd_cntl = tmp; 4221 4222 /* set up cp_doorbell_control */ 4223 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT; 4224 if (prop->use_doorbell) { 4225 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4226 DOORBELL_OFFSET, prop->doorbell_index); 4227 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4228 DOORBELL_EN, 1); 4229 } else 4230 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4231 DOORBELL_EN, 0); 4232 mqd->cp_rb_doorbell_control = tmp; 4233 4234 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4235 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT; 4236 4237 /* active the queue */ 4238 mqd->cp_gfx_hqd_active = 1; 4239 4240 /* set gfx UQ items */ 4241 mqd->shadow_base_lo = lower_32_bits(prop->shadow_addr); 4242 mqd->shadow_base_hi = upper_32_bits(prop->shadow_addr); 4243 mqd->gds_bkup_base_lo = lower_32_bits(prop->gds_bkup_addr); 4244 mqd->gds_bkup_base_hi = upper_32_bits(prop->gds_bkup_addr); 4245 mqd->fw_work_area_base_lo = lower_32_bits(prop->csa_addr); 4246 mqd->fw_work_area_base_hi = upper_32_bits(prop->csa_addr); 4247 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4248 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4249 4250 return 0; 4251 } 4252 4253 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring) 4254 { 4255 struct amdgpu_device *adev = ring->adev; 4256 struct v11_gfx_mqd *mqd = ring->mqd_ptr; 4257 int mqd_idx = ring - &adev->gfx.gfx_ring[0]; 4258 4259 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4260 memset((void *)mqd, 0, sizeof(*mqd)); 4261 mutex_lock(&adev->srbm_mutex); 4262 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4263 amdgpu_ring_init_mqd(ring); 4264 soc21_grbm_select(adev, 0, 0, 0, 0); 4265 mutex_unlock(&adev->srbm_mutex); 4266 if (adev->gfx.me.mqd_backup[mqd_idx]) 4267 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4268 } else { 4269 /* restore mqd with the backup copy */ 4270 if (adev->gfx.me.mqd_backup[mqd_idx]) 4271 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); 4272 /* reset the ring */ 4273 ring->wptr = 0; 4274 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4275 amdgpu_ring_clear_ring(ring); 4276 } 4277 4278 return 0; 4279 } 4280 4281 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) 4282 { 4283 int r, i; 4284 4285 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4286 r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i]); 4287 if (r) 4288 return r; 4289 } 4290 4291 r = amdgpu_gfx_enable_kgq(adev, 0); 4292 if (r) 4293 return r; 4294 4295 return gfx_v11_0_cp_gfx_start(adev); 4296 } 4297 4298 static void gfx_v11_0_compute_mqd_set_cu_mask(struct amdgpu_device *adev, 4299 struct v11_compute_mqd *mqd, 4300 struct amdgpu_mqd_prop *prop) 4301 { 4302 uint32_t se_mask[8] = {0}; 4303 uint32_t wa_mask; 4304 bool has_wa_flag = prop->cu_flags & (AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE | 4305 AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE); 4306 4307 if (!has_wa_flag && (!prop->cu_mask || !prop->cu_mask_count)) 4308 return; 4309 4310 if (has_wa_flag) { 4311 wa_mask = (prop->cu_flags & AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE) ? 4312 0xffff : 0xffffffff; 4313 mqd->compute_static_thread_mgmt_se0 = wa_mask; 4314 mqd->compute_static_thread_mgmt_se1 = wa_mask; 4315 mqd->compute_static_thread_mgmt_se2 = wa_mask; 4316 mqd->compute_static_thread_mgmt_se3 = wa_mask; 4317 return; 4318 } 4319 4320 amdgpu_gfx_mqd_symmetrically_map_cu_mask(adev, prop->cu_mask, 4321 prop->cu_mask_count, se_mask); 4322 4323 mqd->compute_static_thread_mgmt_se0 = se_mask[0]; 4324 mqd->compute_static_thread_mgmt_se1 = se_mask[1]; 4325 mqd->compute_static_thread_mgmt_se2 = se_mask[2]; 4326 mqd->compute_static_thread_mgmt_se3 = se_mask[3]; 4327 } 4328 4329 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m, 4330 struct amdgpu_mqd_prop *prop) 4331 { 4332 struct v11_compute_mqd *mqd = m; 4333 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4334 uint32_t tmp; 4335 4336 mqd->header = 0xC0310800; 4337 mqd->compute_pipelinestat_enable = 0x00000001; 4338 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4339 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4340 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4341 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4342 mqd->compute_misc_reserved = 0x00000007; 4343 4344 eop_base_addr = prop->eop_gpu_addr >> 8; 4345 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4346 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4347 4348 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4349 tmp = regCP_HQD_EOP_CONTROL_DEFAULT; 4350 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4351 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1)); 4352 4353 mqd->cp_hqd_eop_control = tmp; 4354 4355 /* enable doorbell? */ 4356 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4357 4358 if (prop->use_doorbell) { 4359 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4360 DOORBELL_OFFSET, prop->doorbell_index); 4361 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4362 DOORBELL_EN, 1); 4363 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4364 DOORBELL_SOURCE, 0); 4365 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4366 DOORBELL_HIT, 0); 4367 } else { 4368 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4369 DOORBELL_EN, 0); 4370 } 4371 4372 mqd->cp_hqd_pq_doorbell_control = tmp; 4373 4374 /* disable the queue if it's active */ 4375 mqd->cp_hqd_dequeue_request = 0; 4376 mqd->cp_hqd_pq_rptr = 0; 4377 mqd->cp_hqd_pq_wptr_lo = 0; 4378 mqd->cp_hqd_pq_wptr_hi = 0; 4379 4380 /* set the pointer to the MQD */ 4381 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc; 4382 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4383 4384 /* set MQD vmid to 0 */ 4385 tmp = regCP_MQD_CONTROL_DEFAULT; 4386 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4387 mqd->cp_mqd_control = tmp; 4388 4389 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4390 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4391 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4392 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4393 4394 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4395 tmp = regCP_HQD_PQ_CONTROL_DEFAULT; 4396 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4397 (order_base_2(prop->queue_size / 4) - 1)); 4398 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4399 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4400 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1); 4401 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 4402 prop->allow_tunneling); 4403 if (prop->kernel_queue) { 4404 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4405 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4406 } 4407 if (prop->tmz_queue) 4408 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TMZ, 1); 4409 mqd->cp_hqd_pq_control = tmp; 4410 4411 /* set the wb address whether it's enabled or not */ 4412 wb_gpu_addr = prop->rptr_gpu_addr; 4413 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4414 mqd->cp_hqd_pq_rptr_report_addr_hi = 4415 upper_32_bits(wb_gpu_addr) & 0xffff; 4416 4417 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4418 wb_gpu_addr = prop->wptr_gpu_addr; 4419 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4420 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4421 4422 tmp = 0; 4423 /* enable the doorbell if requested */ 4424 if (prop->use_doorbell) { 4425 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4426 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4427 DOORBELL_OFFSET, prop->doorbell_index); 4428 4429 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4430 DOORBELL_EN, 1); 4431 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4432 DOORBELL_SOURCE, 0); 4433 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4434 DOORBELL_HIT, 0); 4435 } 4436 4437 mqd->cp_hqd_pq_doorbell_control = tmp; 4438 4439 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4440 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT; 4441 4442 /* set the vmid for the queue */ 4443 mqd->cp_hqd_vmid = 0; 4444 4445 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT; 4446 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55); 4447 mqd->cp_hqd_persistent_state = tmp; 4448 4449 /* set MIN_IB_AVAIL_SIZE */ 4450 tmp = regCP_HQD_IB_CONTROL_DEFAULT; 4451 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4452 mqd->cp_hqd_ib_control = tmp; 4453 4454 /* set static priority for a compute queue/ring */ 4455 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; 4456 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; 4457 4458 tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); 4459 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 4460 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); 4461 mqd->cp_hqd_quantum = tmp; 4462 4463 mqd->cp_hqd_active = prop->hqd_active; 4464 4465 /* set UQ fenceaddress */ 4466 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4467 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4468 /* set CU mask */ 4469 gfx_v11_0_compute_mqd_set_cu_mask(adev, mqd, prop); 4470 4471 return 0; 4472 } 4473 4474 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring) 4475 { 4476 struct amdgpu_device *adev = ring->adev; 4477 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4478 int j; 4479 4480 /* inactivate the queue */ 4481 if (amdgpu_sriov_vf(adev)) 4482 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); 4483 4484 /* disable wptr polling */ 4485 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4486 4487 /* write the EOP addr */ 4488 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR, 4489 mqd->cp_hqd_eop_base_addr_lo); 4490 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI, 4491 mqd->cp_hqd_eop_base_addr_hi); 4492 4493 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4494 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL, 4495 mqd->cp_hqd_eop_control); 4496 4497 /* enable doorbell? */ 4498 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4499 mqd->cp_hqd_pq_doorbell_control); 4500 4501 /* disable the queue if it's active */ 4502 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { 4503 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); 4504 for (j = 0; j < adev->usec_timeout; j++) { 4505 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) 4506 break; 4507 udelay(1); 4508 } 4509 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 4510 mqd->cp_hqd_dequeue_request); 4511 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 4512 mqd->cp_hqd_pq_rptr); 4513 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4514 mqd->cp_hqd_pq_wptr_lo); 4515 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4516 mqd->cp_hqd_pq_wptr_hi); 4517 } 4518 4519 /* set the pointer to the MQD */ 4520 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, 4521 mqd->cp_mqd_base_addr_lo); 4522 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, 4523 mqd->cp_mqd_base_addr_hi); 4524 4525 /* set MQD vmid to 0 */ 4526 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 4527 mqd->cp_mqd_control); 4528 4529 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4530 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, 4531 mqd->cp_hqd_pq_base_lo); 4532 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, 4533 mqd->cp_hqd_pq_base_hi); 4534 4535 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4536 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, 4537 mqd->cp_hqd_pq_control); 4538 4539 /* set the wb address whether it's enabled or not */ 4540 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR, 4541 mqd->cp_hqd_pq_rptr_report_addr_lo); 4542 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4543 mqd->cp_hqd_pq_rptr_report_addr_hi); 4544 4545 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4546 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR, 4547 mqd->cp_hqd_pq_wptr_poll_addr_lo); 4548 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI, 4549 mqd->cp_hqd_pq_wptr_poll_addr_hi); 4550 4551 /* enable the doorbell if requested */ 4552 if (ring->use_doorbell) { 4553 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4554 (adev->doorbell_index.kiq * 2) << 2); 4555 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4556 (adev->doorbell_index.userqueue_end * 2) << 2); 4557 } 4558 4559 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4560 mqd->cp_hqd_pq_doorbell_control); 4561 4562 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4563 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4564 mqd->cp_hqd_pq_wptr_lo); 4565 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4566 mqd->cp_hqd_pq_wptr_hi); 4567 4568 /* set the vmid for the queue */ 4569 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid); 4570 4571 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, 4572 mqd->cp_hqd_persistent_state); 4573 4574 /* activate the queue */ 4575 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 4576 mqd->cp_hqd_active); 4577 4578 if (ring->use_doorbell) 4579 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4580 4581 return 0; 4582 } 4583 4584 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) 4585 { 4586 struct amdgpu_device *adev = ring->adev; 4587 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4588 4589 gfx_v11_0_kiq_setting(ring); 4590 4591 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4592 /* reset MQD to a clean status */ 4593 if (adev->gfx.kiq[0].mqd_backup) 4594 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd)); 4595 4596 /* reset ring buffer */ 4597 ring->wptr = 0; 4598 amdgpu_ring_clear_ring(ring); 4599 4600 mutex_lock(&adev->srbm_mutex); 4601 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4602 gfx_v11_0_kiq_init_register(ring); 4603 soc21_grbm_select(adev, 0, 0, 0, 0); 4604 mutex_unlock(&adev->srbm_mutex); 4605 } else { 4606 memset((void *)mqd, 0, sizeof(*mqd)); 4607 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4608 amdgpu_ring_clear_ring(ring); 4609 mutex_lock(&adev->srbm_mutex); 4610 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4611 amdgpu_ring_init_mqd(ring); 4612 gfx_v11_0_kiq_init_register(ring); 4613 soc21_grbm_select(adev, 0, 0, 0, 0); 4614 mutex_unlock(&adev->srbm_mutex); 4615 4616 if (adev->gfx.kiq[0].mqd_backup) 4617 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd)); 4618 } 4619 4620 return 0; 4621 } 4622 4623 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring) 4624 { 4625 struct amdgpu_device *adev = ring->adev; 4626 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4627 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4628 4629 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4630 memset((void *)mqd, 0, sizeof(*mqd)); 4631 mutex_lock(&adev->srbm_mutex); 4632 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4633 amdgpu_ring_init_mqd(ring); 4634 soc21_grbm_select(adev, 0, 0, 0, 0); 4635 mutex_unlock(&adev->srbm_mutex); 4636 4637 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4638 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4639 } else { 4640 /* restore MQD to a clean status */ 4641 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4642 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); 4643 /* reset ring buffer */ 4644 ring->wptr = 0; 4645 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4646 amdgpu_ring_clear_ring(ring); 4647 } 4648 4649 return 0; 4650 } 4651 4652 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev) 4653 { 4654 gfx_v11_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 4655 return 0; 4656 } 4657 4658 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev) 4659 { 4660 int i, r; 4661 4662 if (!amdgpu_async_gfx_ring) 4663 gfx_v11_0_cp_compute_enable(adev, true); 4664 4665 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4666 r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i]); 4667 if (r) 4668 return r; 4669 } 4670 4671 return amdgpu_gfx_enable_kcq(adev, 0); 4672 } 4673 4674 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev) 4675 { 4676 int r, i; 4677 struct amdgpu_ring *ring; 4678 4679 if (!(adev->flags & AMD_IS_APU)) 4680 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 4681 4682 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4683 /* legacy firmware loading */ 4684 r = gfx_v11_0_cp_gfx_load_microcode(adev); 4685 if (r) 4686 return r; 4687 4688 if (adev->gfx.rs64_enable) 4689 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev); 4690 else 4691 r = gfx_v11_0_cp_compute_load_microcode(adev); 4692 if (r) 4693 return r; 4694 } 4695 4696 gfx_v11_0_cp_set_doorbell_range(adev); 4697 4698 if (amdgpu_async_gfx_ring) { 4699 gfx_v11_0_cp_compute_enable(adev, true); 4700 gfx_v11_0_cp_gfx_enable(adev, true); 4701 } 4702 4703 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) 4704 r = amdgpu_mes_kiq_hw_init(adev, 0); 4705 else 4706 r = gfx_v11_0_kiq_resume(adev); 4707 if (r) 4708 return r; 4709 4710 r = gfx_v11_0_kcq_resume(adev); 4711 if (r) 4712 return r; 4713 4714 if (!amdgpu_async_gfx_ring) { 4715 r = gfx_v11_0_cp_gfx_resume(adev); 4716 if (r) 4717 return r; 4718 } else { 4719 r = gfx_v11_0_cp_async_gfx_ring_resume(adev); 4720 if (r) 4721 return r; 4722 } 4723 4724 if (adev->gfx.disable_kq) { 4725 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4726 ring = &adev->gfx.gfx_ring[i]; 4727 /* we don't want to set ring->ready */ 4728 r = amdgpu_ring_test_ring(ring); 4729 if (r) 4730 return r; 4731 } 4732 if (amdgpu_async_gfx_ring) 4733 amdgpu_gfx_disable_kgq(adev, 0); 4734 } else { 4735 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4736 ring = &adev->gfx.gfx_ring[i]; 4737 r = amdgpu_ring_test_helper(ring); 4738 if (r) 4739 return r; 4740 } 4741 } 4742 4743 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4744 ring = &adev->gfx.compute_ring[i]; 4745 r = amdgpu_ring_test_helper(ring); 4746 if (r) 4747 return r; 4748 } 4749 4750 return 0; 4751 } 4752 4753 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable) 4754 { 4755 gfx_v11_0_cp_gfx_enable(adev, enable); 4756 gfx_v11_0_cp_compute_enable(adev, enable); 4757 } 4758 4759 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev) 4760 { 4761 int r; 4762 bool value; 4763 4764 r = adev->gfxhub.funcs->gart_enable(adev); 4765 if (r) 4766 return r; 4767 4768 amdgpu_device_flush_hdp(adev, NULL); 4769 4770 value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS; 4771 4772 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 4773 /* TODO investigate why this and the hdp flush above is needed, 4774 * are we missing a flush somewhere else? */ 4775 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0); 4776 4777 return 0; 4778 } 4779 4780 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev) 4781 { 4782 u32 tmp; 4783 4784 /* select RS64 */ 4785 if (adev->gfx.rs64_enable) { 4786 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL); 4787 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1); 4788 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp); 4789 4790 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL); 4791 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1); 4792 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp); 4793 } 4794 4795 if (amdgpu_emu_mode == 1) 4796 msleep(100); 4797 } 4798 4799 static int get_gb_addr_config(struct amdgpu_device * adev) 4800 { 4801 u32 gb_addr_config; 4802 4803 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG); 4804 if (gb_addr_config == 0) 4805 return -EINVAL; 4806 4807 adev->gfx.config.gb_addr_config_fields.num_pkrs = 4808 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS); 4809 4810 adev->gfx.config.gb_addr_config = gb_addr_config; 4811 4812 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 4813 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4814 GB_ADDR_CONFIG, NUM_PIPES); 4815 4816 adev->gfx.config.max_tile_pipes = 4817 adev->gfx.config.gb_addr_config_fields.num_pipes; 4818 4819 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 4820 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4821 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS); 4822 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 4823 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4824 GB_ADDR_CONFIG, NUM_RB_PER_SE); 4825 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 4826 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4827 GB_ADDR_CONFIG, NUM_SHADER_ENGINES); 4828 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 4829 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4830 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE)); 4831 4832 return 0; 4833 } 4834 4835 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev) 4836 { 4837 uint32_t data; 4838 4839 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG); 4840 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK; 4841 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data); 4842 4843 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG); 4844 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK; 4845 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); 4846 } 4847 4848 static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, 4849 bool enable) 4850 { 4851 unsigned int irq_type; 4852 int m, p, r; 4853 4854 if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { 4855 for (m = 0; m < adev->gfx.me.num_me; m++) { 4856 for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { 4857 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4858 if (enable) 4859 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4860 else 4861 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4862 if (r) { 4863 if (!enable) 4864 return r; 4865 goto err_gfx; 4866 } 4867 } 4868 } 4869 } 4870 4871 if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { 4872 for (m = 0; m < adev->gfx.mec.num_mec; ++m) { 4873 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { 4874 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4875 + (m * adev->gfx.mec.num_pipe_per_mec) 4876 + p; 4877 if (enable) 4878 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4879 else 4880 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4881 if (r) { 4882 if (!enable) 4883 return r; 4884 goto err_compute; 4885 } 4886 } 4887 } 4888 } 4889 4890 return 0; 4891 4892 err_compute: 4893 for (p--; p >= 0; p--) { 4894 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4895 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4896 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4897 } 4898 for (m--; m >= 0; m--) { 4899 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { 4900 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4901 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4902 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4903 } 4904 } 4905 m = adev->gfx.me.num_me; 4906 err_gfx: 4907 for (p--; p >= 0; p--) { 4908 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4909 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4910 } 4911 for (m--; m >= 0; m--) { 4912 for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { 4913 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4914 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4915 } 4916 } 4917 return r; 4918 } 4919 4920 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) 4921 { 4922 int r; 4923 struct amdgpu_device *adev = ip_block->adev; 4924 4925 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, 4926 adev->gfx.cleaner_shader_ptr); 4927 4928 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 4929 if (adev->gfx.imu.funcs) { 4930 /* RLC autoload sequence 1: Program rlc ram */ 4931 if (adev->gfx.imu.funcs->program_rlc_ram) 4932 adev->gfx.imu.funcs->program_rlc_ram(adev); 4933 /* rlc autoload firmware */ 4934 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev); 4935 if (r) 4936 return r; 4937 } 4938 } else { 4939 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4940 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) { 4941 if (adev->gfx.imu.funcs->load_microcode) 4942 adev->gfx.imu.funcs->load_microcode(adev); 4943 if (adev->gfx.imu.funcs->setup_imu) 4944 adev->gfx.imu.funcs->setup_imu(adev); 4945 if (adev->gfx.imu.funcs->start_imu) 4946 adev->gfx.imu.funcs->start_imu(adev); 4947 } 4948 4949 /* disable gpa mode in backdoor loading */ 4950 gfx_v11_0_disable_gpa_mode(adev); 4951 } 4952 } 4953 4954 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) || 4955 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 4956 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev); 4957 if (r) { 4958 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r); 4959 return r; 4960 } 4961 } 4962 4963 adev->gfx.is_poweron = true; 4964 4965 if(get_gb_addr_config(adev)) 4966 drm_warn(adev_to_drm(adev), "Invalid gb_addr_config !\n"); 4967 4968 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP && 4969 adev->gfx.rs64_enable) 4970 gfx_v11_0_config_gfx_rs64(adev); 4971 4972 r = gfx_v11_0_gfxhub_enable(adev); 4973 if (r) 4974 return r; 4975 4976 if (!amdgpu_emu_mode) 4977 gfx_v11_0_init_golden_registers(adev); 4978 4979 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || 4980 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) { 4981 /** 4982 * For gfx 11, rlc firmware loading relies on smu firmware is 4983 * loaded firstly, so in direct type, it has to load smc ucode 4984 * here before rlc. 4985 */ 4986 r = amdgpu_pm_load_smu_firmware(adev, NULL); 4987 if (r) 4988 return r; 4989 } 4990 4991 gfx_v11_0_constants_init(adev); 4992 4993 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 4994 gfx_v11_0_select_cp_fw_arch(adev); 4995 4996 if (adev->nbio.funcs->gc_doorbell_init) 4997 adev->nbio.funcs->gc_doorbell_init(adev); 4998 4999 r = gfx_v11_0_rlc_resume(adev); 5000 if (r) 5001 return r; 5002 5003 /* 5004 * init golden registers and rlc resume may override some registers, 5005 * reconfig them here 5006 */ 5007 gfx_v11_0_tcp_harvest(adev); 5008 5009 r = gfx_v11_0_cp_resume(adev); 5010 if (r) 5011 return r; 5012 5013 /* get IMU version from HW if it's not set */ 5014 if (!adev->gfx.imu_fw_version) 5015 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0); 5016 5017 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 5018 if (r) 5019 return r; 5020 5021 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 5022 if (r) 5023 goto err_priv_inst; 5024 5025 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); 5026 if (r) 5027 goto err_bad_op; 5028 5029 r = gfx_v11_0_set_userq_eop_interrupts(adev, true); 5030 if (r) 5031 goto err_userq_eop; 5032 5033 return 0; 5034 5035 err_userq_eop: 5036 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5037 err_bad_op: 5038 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5039 err_priv_inst: 5040 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5041 return r; 5042 } 5043 5044 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) 5045 { 5046 struct amdgpu_device *adev = ip_block->adev; 5047 5048 cancel_delayed_work_sync(&adev->gfx.idle_work); 5049 5050 gfx_v11_0_set_userq_eop_interrupts(adev, false); 5051 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5052 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5053 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5054 5055 if (!adev->no_hw_access) { 5056 if (amdgpu_async_gfx_ring && 5057 !adev->gfx.disable_kq) { 5058 if (amdgpu_gfx_disable_kgq(adev, 0)) 5059 DRM_ERROR("KGQ disable failed\n"); 5060 } 5061 5062 if (amdgpu_gfx_disable_kcq(adev, 0)) 5063 DRM_ERROR("KCQ disable failed\n"); 5064 5065 amdgpu_mes_kiq_hw_fini(adev, 0); 5066 } 5067 5068 if (amdgpu_sriov_vf(adev)) 5069 /* Remove the steps disabling CPG and clearing KIQ position, 5070 * so that CP could perform IDLE-SAVE during switch. Those 5071 * steps are necessary to avoid a DMAR error in gfx9 but it is 5072 * not reproduced on gfx11. 5073 */ 5074 return 0; 5075 5076 gfx_v11_0_cp_enable(adev, false); 5077 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 5078 5079 adev->gfxhub.funcs->gart_disable(adev); 5080 5081 adev->gfx.is_poweron = false; 5082 5083 return 0; 5084 } 5085 5086 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block) 5087 { 5088 return gfx_v11_0_hw_fini(ip_block); 5089 } 5090 5091 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block) 5092 { 5093 return gfx_v11_0_hw_init(ip_block); 5094 } 5095 5096 static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block) 5097 { 5098 struct amdgpu_device *adev = ip_block->adev; 5099 5100 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS), 5101 GRBM_STATUS, GUI_ACTIVE)) 5102 return false; 5103 else 5104 return true; 5105 } 5106 5107 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 5108 { 5109 unsigned i; 5110 u32 tmp; 5111 struct amdgpu_device *adev = ip_block->adev; 5112 5113 for (i = 0; i < adev->usec_timeout; i++) { 5114 /* read MC_STATUS */ 5115 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) & 5116 GRBM_STATUS__GUI_ACTIVE_MASK; 5117 5118 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE)) 5119 return 0; 5120 udelay(1); 5121 } 5122 return -ETIMEDOUT; 5123 } 5124 5125 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev, 5126 bool req) 5127 { 5128 u32 i, tmp, val; 5129 5130 for (i = 0; i < adev->usec_timeout; i++) { 5131 /* Request with MeId=2, PipeId=0 */ 5132 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req); 5133 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4); 5134 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp); 5135 5136 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX); 5137 if (req) { 5138 if (val == tmp) 5139 break; 5140 } else { 5141 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, 5142 REQUEST, 1); 5143 5144 /* unlocked or locked by firmware */ 5145 if (val != tmp) 5146 break; 5147 } 5148 udelay(1); 5149 } 5150 5151 if (i >= adev->usec_timeout) 5152 return -EINVAL; 5153 5154 return 0; 5155 } 5156 5157 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block) 5158 { 5159 u32 grbm_soft_reset = 0; 5160 u32 tmp; 5161 int r, i, j, k; 5162 struct amdgpu_device *adev = ip_block->adev; 5163 5164 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5165 5166 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5167 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0); 5168 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0); 5169 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0); 5170 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0); 5171 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5172 5173 mutex_lock(&adev->srbm_mutex); 5174 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 5175 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 5176 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 5177 soc21_grbm_select(adev, i, k, j, 0); 5178 5179 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2); 5180 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1); 5181 } 5182 } 5183 } 5184 for (i = 0; i < adev->gfx.me.num_me; ++i) { 5185 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { 5186 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 5187 soc21_grbm_select(adev, i, k, j, 0); 5188 5189 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1); 5190 } 5191 } 5192 } 5193 soc21_grbm_select(adev, 0, 0, 0, 0); 5194 mutex_unlock(&adev->srbm_mutex); 5195 5196 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */ 5197 mutex_lock(&adev->gfx.reset_sem_mutex); 5198 r = gfx_v11_0_request_gfx_index_mutex(adev, true); 5199 if (r) { 5200 mutex_unlock(&adev->gfx.reset_sem_mutex); 5201 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n"); 5202 return r; 5203 } 5204 5205 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe); 5206 5207 // Read CP_VMID_RESET register three times. 5208 // to get sufficient time for GFX_HQD_ACTIVE reach 0 5209 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5210 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5211 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5212 5213 /* release the gfx mutex */ 5214 r = gfx_v11_0_request_gfx_index_mutex(adev, false); 5215 mutex_unlock(&adev->gfx.reset_sem_mutex); 5216 if (r) { 5217 DRM_ERROR("Failed to release the gfx mutex during soft reset\n"); 5218 return r; 5219 } 5220 5221 for (i = 0; i < adev->usec_timeout; i++) { 5222 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) && 5223 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE)) 5224 break; 5225 udelay(1); 5226 } 5227 if (i >= adev->usec_timeout) { 5228 printk("Failed to wait all pipes clean\n"); 5229 return -EINVAL; 5230 } 5231 5232 /********** trigger soft reset ***********/ 5233 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5234 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5235 SOFT_RESET_CP, 1); 5236 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5237 SOFT_RESET_GFX, 1); 5238 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5239 SOFT_RESET_CPF, 1); 5240 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5241 SOFT_RESET_CPC, 1); 5242 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5243 SOFT_RESET_CPG, 1); 5244 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5245 /********** exit soft reset ***********/ 5246 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5247 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5248 SOFT_RESET_CP, 0); 5249 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5250 SOFT_RESET_GFX, 0); 5251 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5252 SOFT_RESET_CPF, 0); 5253 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5254 SOFT_RESET_CPC, 0); 5255 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5256 SOFT_RESET_CPG, 0); 5257 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5258 5259 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL); 5260 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1); 5261 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp); 5262 5263 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0); 5264 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0); 5265 5266 for (i = 0; i < adev->usec_timeout; i++) { 5267 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET)) 5268 break; 5269 udelay(1); 5270 } 5271 if (i >= adev->usec_timeout) { 5272 printk("Failed to wait CP_VMID_RESET to 0\n"); 5273 return -EINVAL; 5274 } 5275 5276 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5277 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5278 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5279 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5280 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5281 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5282 5283 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5284 5285 r = gfx_v11_0_cp_resume(adev); 5286 if (r) 5287 return r; 5288 5289 /** 5290 * GFX soft reset impacts MES, resume MES after GFX soft reset is finished 5291 */ 5292 return amdgpu_mes_resume(adev, 0); 5293 } 5294 5295 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5296 { 5297 uint64_t clock; 5298 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after; 5299 5300 if (amdgpu_sriov_vf(adev)) { 5301 amdgpu_gfx_off_ctrl(adev, false); 5302 mutex_lock(&adev->gfx.gpu_clock_mutex); 5303 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5304 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5305 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5306 if (clock_counter_hi_pre != clock_counter_hi_after) 5307 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5308 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5309 amdgpu_gfx_off_ctrl(adev, true); 5310 } else { 5311 preempt_disable(); 5312 if (amdgpu_ip_version(adev, SMUIO_HWIP, 0) < IP_VERSION(15, 0, 0)) { 5313 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5314 regGOLDEN_TSC_COUNT_UPPER); 5315 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5316 regGOLDEN_TSC_COUNT_LOWER); 5317 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5318 regGOLDEN_TSC_COUNT_UPPER); 5319 if (clock_counter_hi_pre != clock_counter_hi_after) 5320 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5321 regGOLDEN_TSC_COUNT_LOWER); 5322 } else { 5323 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5324 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5325 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5326 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5327 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5328 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5329 if (clock_counter_hi_pre != clock_counter_hi_after) 5330 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5331 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5332 } 5333 preempt_enable(); 5334 } 5335 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL); 5336 5337 return clock; 5338 } 5339 5340 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5341 uint32_t vmid, 5342 uint32_t gds_base, uint32_t gds_size, 5343 uint32_t gws_base, uint32_t gws_size, 5344 uint32_t oa_base, uint32_t oa_size) 5345 { 5346 struct amdgpu_device *adev = ring->adev; 5347 5348 /* GDS Base */ 5349 gfx_v11_0_write_data_to_reg(ring, 0, false, 5350 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid, 5351 gds_base); 5352 5353 /* GDS Size */ 5354 gfx_v11_0_write_data_to_reg(ring, 0, false, 5355 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid, 5356 gds_size); 5357 5358 /* GWS */ 5359 gfx_v11_0_write_data_to_reg(ring, 0, false, 5360 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid, 5361 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5362 5363 /* OA */ 5364 gfx_v11_0_write_data_to_reg(ring, 0, false, 5365 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid, 5366 (1 << (oa_size + oa_base)) - (1 << oa_base)); 5367 } 5368 5369 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) 5370 { 5371 struct amdgpu_device *adev = ip_block->adev; 5372 int r; 5373 5374 switch (amdgpu_user_queue) { 5375 case -1: 5376 case 0: 5377 default: 5378 adev->gfx.disable_kq = false; 5379 adev->gfx.disable_uq = true; 5380 break; 5381 case 1: 5382 adev->gfx.disable_kq = false; 5383 adev->gfx.disable_uq = false; 5384 break; 5385 case 2: 5386 adev->gfx.disable_kq = true; 5387 adev->gfx.disable_uq = false; 5388 break; 5389 } 5390 5391 adev->gfx.funcs = &gfx_v11_0_gfx_funcs; 5392 5393 gfx_v11_0_set_imu_funcs(adev); 5394 r = gfx_v11_0_init_microcode(adev); 5395 if (r) 5396 return r; 5397 5398 if (adev->gfx.disable_kq) { 5399 /* We need one GFX ring temporarily to set up 5400 * the clear state. 5401 */ 5402 adev->gfx.num_gfx_rings = 1; 5403 adev->gfx.num_compute_rings = 0; 5404 } else { 5405 /* rs64 only supports one gfx pipe */ 5406 if (adev->gfx.rs64_enable) 5407 adev->gfx.num_gfx_rings = 1; 5408 else 5409 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS; 5410 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5411 AMDGPU_MAX_COMPUTE_RINGS); 5412 } 5413 5414 gfx_v11_0_set_kiq_pm4_funcs(adev); 5415 gfx_v11_0_set_ring_funcs(adev); 5416 gfx_v11_0_set_irq_funcs(adev); 5417 gfx_v11_0_set_gds_init(adev); 5418 gfx_v11_0_set_rlc_funcs(adev); 5419 gfx_v11_0_set_mqd_funcs(adev); 5420 5421 gfx_v11_0_init_rlcg_reg_access_ctrl(adev); 5422 5423 amdgpu_init_rlc_reg_funcs(adev); 5424 5425 return 0; 5426 } 5427 5428 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev) 5429 { 5430 uint32_t rlc_cntl; 5431 5432 /* if RLC is not enabled, do nothing */ 5433 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); 5434 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; 5435 } 5436 5437 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5438 { 5439 uint32_t data; 5440 unsigned i; 5441 5442 data = RLC_SAFE_MODE__CMD_MASK; 5443 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5444 5445 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data); 5446 5447 /* wait for RLC_SAFE_MODE */ 5448 for (i = 0; i < adev->usec_timeout; i++) { 5449 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE), 5450 RLC_SAFE_MODE, CMD)) 5451 break; 5452 udelay(1); 5453 } 5454 } 5455 5456 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5457 { 5458 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK); 5459 } 5460 5461 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 5462 bool enable) 5463 { 5464 uint32_t def, data; 5465 5466 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK)) 5467 return; 5468 5469 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5470 5471 if (enable) 5472 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5473 else 5474 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5475 5476 if (def != data) 5477 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5478 } 5479 5480 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev, 5481 bool enable) 5482 { 5483 uint32_t def, data; 5484 5485 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG)) 5486 return; 5487 5488 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5489 5490 if (enable) 5491 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5492 else 5493 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5494 5495 if (def != data) 5496 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5497 } 5498 5499 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev, 5500 bool enable) 5501 { 5502 uint32_t def, data; 5503 5504 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG)) 5505 return; 5506 5507 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5508 5509 if (enable) 5510 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5511 else 5512 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5513 5514 if (def != data) 5515 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5516 } 5517 5518 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5519 bool enable) 5520 { 5521 uint32_t data, def; 5522 5523 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS))) 5524 return; 5525 5526 /* It is disabled by HW by default */ 5527 if (enable) { 5528 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5529 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 5530 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5531 5532 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5533 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5534 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5535 5536 if (def != data) 5537 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5538 } 5539 } else { 5540 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5541 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5542 5543 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5544 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5545 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5546 5547 if (def != data) 5548 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5549 } 5550 } 5551 } 5552 5553 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5554 bool enable) 5555 { 5556 uint32_t def, data; 5557 5558 if (!(adev->cg_flags & 5559 (AMD_CG_SUPPORT_GFX_CGCG | 5560 AMD_CG_SUPPORT_GFX_CGLS | 5561 AMD_CG_SUPPORT_GFX_3D_CGCG | 5562 AMD_CG_SUPPORT_GFX_3D_CGLS))) 5563 return; 5564 5565 if (enable) { 5566 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5567 5568 /* unset CGCG override */ 5569 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5570 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5571 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5572 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5573 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG || 5574 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5575 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 5576 5577 /* update CGCG override bits */ 5578 if (def != data) 5579 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5580 5581 /* enable cgcg FSM(0x0000363F) */ 5582 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5583 5584 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5585 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK; 5586 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5587 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5588 } 5589 5590 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5591 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK; 5592 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5593 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5594 } 5595 5596 if (def != data) 5597 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5598 5599 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5600 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5601 5602 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5603 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK; 5604 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5605 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5606 } 5607 5608 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5609 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK; 5610 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5611 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5612 } 5613 5614 if (def != data) 5615 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5616 5617 /* set IDLE_POLL_COUNT(0x00900100) */ 5618 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL); 5619 5620 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK); 5621 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5622 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5623 5624 if (def != data) 5625 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data); 5626 5627 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5628 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5629 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5630 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5631 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5632 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data); 5633 5634 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5635 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5636 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5637 5638 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5639 if (adev->sdma.num_instances > 1) { 5640 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5641 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5642 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5643 } 5644 } else { 5645 /* Program RLC_CGCG_CGLS_CTRL */ 5646 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5647 5648 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5649 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5650 5651 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5652 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5653 5654 if (def != data) 5655 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5656 5657 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5658 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5659 5660 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 5661 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5662 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5663 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5664 5665 if (def != data) 5666 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5667 5668 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5669 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5670 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5671 5672 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5673 if (adev->sdma.num_instances > 1) { 5674 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5675 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5676 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5677 } 5678 } 5679 } 5680 5681 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5682 bool enable) 5683 { 5684 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5685 5686 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable); 5687 5688 gfx_v11_0_update_medium_grain_clock_gating(adev, enable); 5689 5690 gfx_v11_0_update_repeater_fgcg(adev, enable); 5691 5692 gfx_v11_0_update_sram_fgcg(adev, enable); 5693 5694 gfx_v11_0_update_perf_clk(adev, enable); 5695 5696 if (adev->cg_flags & 5697 (AMD_CG_SUPPORT_GFX_MGCG | 5698 AMD_CG_SUPPORT_GFX_CGLS | 5699 AMD_CG_SUPPORT_GFX_CGCG | 5700 AMD_CG_SUPPORT_GFX_3D_CGCG | 5701 AMD_CG_SUPPORT_GFX_3D_CGLS)) 5702 gfx_v11_0_enable_gui_idle_interrupt(adev, enable); 5703 5704 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5705 5706 return 0; 5707 } 5708 5709 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5710 struct amdgpu_ring *ring, unsigned vmid) 5711 { 5712 u32 reg, pre_data, data; 5713 5714 amdgpu_gfx_off_ctrl(adev, false); 5715 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); 5716 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) 5717 pre_data = RREG32_NO_KIQ(reg); 5718 else 5719 pre_data = RREG32(reg); 5720 5721 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK); 5722 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5723 5724 if (pre_data != data) { 5725 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) { 5726 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data); 5727 } else 5728 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data); 5729 } 5730 amdgpu_gfx_off_ctrl(adev, true); 5731 5732 if (ring 5733 && amdgpu_sriov_is_pp_one_vf(adev) 5734 && (pre_data != data) 5735 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) 5736 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) { 5737 amdgpu_ring_emit_wreg(ring, reg, data); 5738 } 5739 } 5740 5741 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = { 5742 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled, 5743 .set_safe_mode = gfx_v11_0_set_safe_mode, 5744 .unset_safe_mode = gfx_v11_0_unset_safe_mode, 5745 .init = gfx_v11_0_rlc_init, 5746 .get_csb_size = gfx_v11_0_get_csb_size, 5747 .get_csb_buffer = gfx_v11_0_get_csb_buffer, 5748 .resume = gfx_v11_0_rlc_resume, 5749 .stop = gfx_v11_0_rlc_stop, 5750 .reset = gfx_v11_0_rlc_reset, 5751 .start = gfx_v11_0_rlc_start, 5752 .update_spm_vmid = gfx_v11_0_update_spm_vmid, 5753 }; 5754 5755 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable) 5756 { 5757 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 5758 5759 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) 5760 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5761 else 5762 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5763 5764 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data); 5765 5766 // Program RLC_PG_DELAY3 for CGPG hysteresis 5767 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { 5768 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5769 case IP_VERSION(11, 0, 1): 5770 case IP_VERSION(11, 0, 4): 5771 case IP_VERSION(11, 5, 0): 5772 case IP_VERSION(11, 5, 1): 5773 case IP_VERSION(11, 5, 2): 5774 case IP_VERSION(11, 5, 3): 5775 case IP_VERSION(11, 5, 4): 5776 case IP_VERSION(11, 5, 6): 5777 case IP_VERSION(11, 7, 0): 5778 case IP_VERSION(11, 7, 1): 5779 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); 5780 break; 5781 default: 5782 break; 5783 } 5784 } 5785 } 5786 5787 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable) 5788 { 5789 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5790 5791 gfx_v11_cntl_power_gating(adev, enable); 5792 5793 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5794 } 5795 5796 static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5797 enum amd_powergating_state state) 5798 { 5799 struct amdgpu_device *adev = ip_block->adev; 5800 bool enable = (state == AMD_PG_STATE_GATE); 5801 5802 if (amdgpu_sriov_vf(adev)) 5803 return 0; 5804 5805 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5806 case IP_VERSION(11, 0, 0): 5807 case IP_VERSION(11, 0, 2): 5808 case IP_VERSION(11, 0, 3): 5809 amdgpu_gfx_off_ctrl(adev, enable); 5810 break; 5811 case IP_VERSION(11, 0, 1): 5812 case IP_VERSION(11, 0, 4): 5813 case IP_VERSION(11, 5, 0): 5814 case IP_VERSION(11, 5, 1): 5815 case IP_VERSION(11, 5, 2): 5816 case IP_VERSION(11, 5, 3): 5817 case IP_VERSION(11, 5, 4): 5818 case IP_VERSION(11, 5, 6): 5819 case IP_VERSION(11, 7, 0): 5820 case IP_VERSION(11, 7, 1): 5821 if (!enable) 5822 amdgpu_gfx_off_ctrl(adev, false); 5823 5824 gfx_v11_cntl_pg(adev, enable); 5825 5826 if (enable) 5827 amdgpu_gfx_off_ctrl(adev, true); 5828 5829 break; 5830 default: 5831 break; 5832 } 5833 5834 return 0; 5835 } 5836 5837 static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5838 enum amd_clockgating_state state) 5839 { 5840 struct amdgpu_device *adev = ip_block->adev; 5841 5842 if (amdgpu_sriov_vf(adev)) 5843 return 0; 5844 5845 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5846 case IP_VERSION(11, 0, 0): 5847 case IP_VERSION(11, 0, 1): 5848 case IP_VERSION(11, 0, 2): 5849 case IP_VERSION(11, 0, 3): 5850 case IP_VERSION(11, 0, 4): 5851 case IP_VERSION(11, 5, 0): 5852 case IP_VERSION(11, 5, 1): 5853 case IP_VERSION(11, 5, 2): 5854 case IP_VERSION(11, 5, 3): 5855 case IP_VERSION(11, 5, 4): 5856 case IP_VERSION(11, 5, 6): 5857 case IP_VERSION(11, 7, 0): 5858 case IP_VERSION(11, 7, 1): 5859 gfx_v11_0_update_gfx_clock_gating(adev, 5860 state == AMD_CG_STATE_GATE); 5861 break; 5862 default: 5863 break; 5864 } 5865 5866 return 0; 5867 } 5868 5869 static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5870 { 5871 struct amdgpu_device *adev = ip_block->adev; 5872 int data; 5873 5874 /* AMD_CG_SUPPORT_GFX_MGCG */ 5875 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5876 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5877 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5878 5879 /* AMD_CG_SUPPORT_REPEATER_FGCG */ 5880 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK)) 5881 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG; 5882 5883 /* AMD_CG_SUPPORT_GFX_FGCG */ 5884 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK)) 5885 *flags |= AMD_CG_SUPPORT_GFX_FGCG; 5886 5887 /* AMD_CG_SUPPORT_GFX_PERF_CLK */ 5888 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK)) 5889 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK; 5890 5891 /* AMD_CG_SUPPORT_GFX_CGCG */ 5892 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5893 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5894 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5895 5896 /* AMD_CG_SUPPORT_GFX_CGLS */ 5897 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5898 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5899 5900 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5901 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5902 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5903 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5904 5905 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5906 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5907 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5908 } 5909 5910 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5911 { 5912 /* gfx11 is 32bit rptr*/ 5913 return *(uint32_t *)ring->rptr_cpu_addr; 5914 } 5915 5916 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5917 { 5918 struct amdgpu_device *adev = ring->adev; 5919 u64 wptr; 5920 5921 /* XXX check if swapping is necessary on BE */ 5922 if (ring->use_doorbell) { 5923 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5924 } else { 5925 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR); 5926 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32; 5927 } 5928 5929 return wptr; 5930 } 5931 5932 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5933 { 5934 struct amdgpu_device *adev = ring->adev; 5935 5936 if (ring->use_doorbell) { 5937 /* XXX check if swapping is necessary on BE */ 5938 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5939 ring->wptr); 5940 WDOORBELL64(ring->doorbell_index, ring->wptr); 5941 } else { 5942 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, 5943 lower_32_bits(ring->wptr)); 5944 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, 5945 upper_32_bits(ring->wptr)); 5946 } 5947 } 5948 5949 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5950 { 5951 /* gfx11 hardware is 32bit rptr */ 5952 return *(uint32_t *)ring->rptr_cpu_addr; 5953 } 5954 5955 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5956 { 5957 u64 wptr; 5958 5959 /* XXX check if swapping is necessary on BE */ 5960 if (ring->use_doorbell) 5961 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5962 else 5963 BUG(); 5964 return wptr; 5965 } 5966 5967 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5968 { 5969 struct amdgpu_device *adev = ring->adev; 5970 5971 /* XXX check if swapping is necessary on BE */ 5972 if (ring->use_doorbell) { 5973 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5974 ring->wptr); 5975 WDOORBELL64(ring->doorbell_index, ring->wptr); 5976 } else { 5977 BUG(); /* only DOORBELL method supported on gfx11 now */ 5978 } 5979 } 5980 5981 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5982 { 5983 struct amdgpu_device *adev = ring->adev; 5984 u32 ref_and_mask, reg_mem_engine; 5985 5986 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5987 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5988 return; 5989 } 5990 5991 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 5992 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5993 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5994 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5995 ref_and_mask, ref_and_mask, 0x20); 5996 } 5997 5998 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5999 struct amdgpu_job *job, 6000 struct amdgpu_ib *ib, 6001 uint32_t flags) 6002 { 6003 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6004 u32 header, control = 0; 6005 6006 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 6007 6008 control |= ib->length_dw | (vmid << 24); 6009 6010 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 6011 control |= INDIRECT_BUFFER_PRE_ENB(1); 6012 6013 if (flags & AMDGPU_IB_PREEMPTED) 6014 control |= INDIRECT_BUFFER_PRE_RESUME(1); 6015 6016 if (vmid && !ring->adev->gfx.rs64_enable) 6017 gfx_v11_0_ring_emit_de_meta(ring, 6018 !amdgpu_sriov_vf(ring->adev) && (flags & AMDGPU_IB_PREEMPTED)); 6019 } 6020 6021 amdgpu_ring_write(ring, header); 6022 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 6023 amdgpu_ring_write(ring, 6024 #ifdef __BIG_ENDIAN 6025 (2 << 0) | 6026 #endif 6027 lower_32_bits(ib->gpu_addr)); 6028 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 6029 amdgpu_ring_write(ring, control); 6030 } 6031 6032 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 6033 struct amdgpu_job *job, 6034 struct amdgpu_ib *ib, 6035 uint32_t flags) 6036 { 6037 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 6038 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 6039 6040 /* Currently, there is a high possibility to get wave ID mismatch 6041 * between ME and GDS, leading to a hw deadlock, because ME generates 6042 * different wave IDs than the GDS expects. This situation happens 6043 * randomly when at least 5 compute pipes use GDS ordered append. 6044 * The wave IDs generated by ME are also wrong after suspend/resume. 6045 * Those are probably bugs somewhere else in the kernel driver. 6046 * 6047 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6048 * GDS to 0 for this ring (me/pipe). 6049 */ 6050 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6051 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6052 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID); 6053 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6054 } 6055 6056 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6057 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 6058 amdgpu_ring_write(ring, 6059 #ifdef __BIG_ENDIAN 6060 (2 << 0) | 6061 #endif 6062 lower_32_bits(ib->gpu_addr)); 6063 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 6064 amdgpu_ring_write(ring, control); 6065 } 6066 6067 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 6068 u64 seq, unsigned flags) 6069 { 6070 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6071 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6072 6073 /* RELEASE_MEM - flush caches, send int */ 6074 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 6075 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ | 6076 PACKET3_RELEASE_MEM_GCR_GL2_WB | 6077 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */ 6078 PACKET3_RELEASE_MEM_GCR_GLM_WB | 6079 PACKET3_RELEASE_MEM_CACHE_POLICY(3) | 6080 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6081 PACKET3_RELEASE_MEM_EVENT_INDEX(5))); 6082 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) | 6083 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0))); 6084 6085 /* 6086 * the address should be Qword aligned if 64bit write, Dword 6087 * aligned if only send 32bit data low (discard data high) 6088 */ 6089 if (write64bit) 6090 WARN_ON(addr & 0x7); 6091 else 6092 WARN_ON(addr & 0x3); 6093 amdgpu_ring_write(ring, lower_32_bits(addr)); 6094 amdgpu_ring_write(ring, upper_32_bits(addr)); 6095 amdgpu_ring_write(ring, lower_32_bits(seq)); 6096 amdgpu_ring_write(ring, upper_32_bits(seq)); 6097 amdgpu_ring_write(ring, 0); 6098 } 6099 6100 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6101 { 6102 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6103 uint32_t seq = ring->fence_drv.sync_seq; 6104 uint64_t addr = ring->fence_drv.gpu_addr; 6105 6106 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr), 6107 upper_32_bits(addr), seq, 0xffffffff, 4); 6108 } 6109 6110 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 6111 uint16_t pasid, uint32_t flush_type, 6112 bool all_hub, uint8_t dst_sel) 6113 { 6114 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 6115 amdgpu_ring_write(ring, 6116 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) | 6117 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 6118 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 6119 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 6120 } 6121 6122 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6123 unsigned vmid, uint64_t pd_addr) 6124 { 6125 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6126 6127 /* compute doesn't have PFP */ 6128 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 6129 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6130 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6131 amdgpu_ring_write(ring, 0x0); 6132 } 6133 6134 /* Make sure that we can't skip the SET_Q_MODE packets when the VM 6135 * changed in any way. 6136 */ 6137 ring->set_q_mode_offs = 0; 6138 ring->set_q_mode_ptr = NULL; 6139 } 6140 6141 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6142 u64 seq, unsigned int flags) 6143 { 6144 struct amdgpu_device *adev = ring->adev; 6145 6146 /* write fence seq to the "addr" */ 6147 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6148 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6149 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6150 amdgpu_ring_write(ring, lower_32_bits(addr)); 6151 amdgpu_ring_write(ring, upper_32_bits(addr)); 6152 amdgpu_ring_write(ring, lower_32_bits(seq)); 6153 6154 if (flags & AMDGPU_FENCE_FLAG_INT) { 6155 /* set register to trigger INT */ 6156 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6157 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6158 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6159 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS)); 6160 amdgpu_ring_write(ring, 0); 6161 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6162 } 6163 } 6164 6165 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, 6166 uint32_t flags) 6167 { 6168 uint32_t dw2 = 0; 6169 6170 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6171 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6172 /* set load_global_config & load_global_uconfig */ 6173 dw2 |= 0x8001; 6174 /* set load_cs_sh_regs */ 6175 dw2 |= 0x01000000; 6176 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6177 dw2 |= 0x10002; 6178 } 6179 6180 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6181 amdgpu_ring_write(ring, dw2); 6182 amdgpu_ring_write(ring, 0); 6183 } 6184 6185 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 6186 uint64_t addr) 6187 { 6188 unsigned ret; 6189 6190 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6191 amdgpu_ring_write(ring, lower_32_bits(addr)); 6192 amdgpu_ring_write(ring, upper_32_bits(addr)); 6193 /* discard following DWs if *cond_exec_gpu_addr==0 */ 6194 amdgpu_ring_write(ring, 0); 6195 ret = ring->wptr & ring->buf_mask; 6196 /* patch dummy value later */ 6197 amdgpu_ring_write(ring, 0); 6198 6199 return ret; 6200 } 6201 6202 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring, 6203 u64 shadow_va, u64 csa_va, 6204 u64 gds_va, bool init_shadow, 6205 int vmid) 6206 { 6207 struct amdgpu_device *adev = ring->adev; 6208 unsigned int offs, end; 6209 6210 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj) 6211 return; 6212 6213 /* 6214 * The logic here isn't easy to understand because we need to keep state 6215 * accross multiple executions of the function as well as between the 6216 * CPU and GPU. The general idea is that the newly written GPU command 6217 * has a condition on the previous one and only executed if really 6218 * necessary. 6219 */ 6220 6221 /* 6222 * The dw in the NOP controls if the next SET_Q_MODE packet should be 6223 * executed or not. Reserve 64bits just to be on the save side. 6224 */ 6225 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1)); 6226 offs = ring->wptr & ring->buf_mask; 6227 6228 /* 6229 * We start with skipping the prefix SET_Q_MODE and always executing 6230 * the postfix SET_Q_MODE packet. This is changed below with a 6231 * WRITE_DATA command when the postfix executed. 6232 */ 6233 amdgpu_ring_write(ring, shadow_va ? 1 : 0); 6234 amdgpu_ring_write(ring, 0); 6235 6236 if (ring->set_q_mode_offs) { 6237 uint64_t addr; 6238 6239 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6240 addr += ring->set_q_mode_offs << 2; 6241 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr); 6242 } 6243 6244 /* 6245 * When the postfix SET_Q_MODE packet executes we need to make sure that the 6246 * next prefix SET_Q_MODE packet executes as well. 6247 */ 6248 if (!shadow_va) { 6249 uint64_t addr; 6250 6251 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6252 addr += offs << 2; 6253 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6254 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); 6255 amdgpu_ring_write(ring, lower_32_bits(addr)); 6256 amdgpu_ring_write(ring, upper_32_bits(addr)); 6257 amdgpu_ring_write(ring, 0x1); 6258 } 6259 6260 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7)); 6261 amdgpu_ring_write(ring, lower_32_bits(shadow_va)); 6262 amdgpu_ring_write(ring, upper_32_bits(shadow_va)); 6263 amdgpu_ring_write(ring, lower_32_bits(gds_va)); 6264 amdgpu_ring_write(ring, upper_32_bits(gds_va)); 6265 amdgpu_ring_write(ring, lower_32_bits(csa_va)); 6266 amdgpu_ring_write(ring, upper_32_bits(csa_va)); 6267 amdgpu_ring_write(ring, shadow_va ? 6268 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0); 6269 amdgpu_ring_write(ring, init_shadow ? 6270 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0); 6271 6272 if (ring->set_q_mode_offs) 6273 amdgpu_ring_patch_cond_exec(ring, end); 6274 6275 if (shadow_va) { 6276 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid; 6277 6278 /* 6279 * If the tokens match try to skip the last postfix SET_Q_MODE 6280 * packet to avoid saving/restoring the state all the time. 6281 */ 6282 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token) 6283 *ring->set_q_mode_ptr = 0; 6284 6285 ring->set_q_mode_token = token; 6286 } else { 6287 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs]; 6288 } 6289 6290 ring->set_q_mode_offs = offs; 6291 } 6292 6293 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume) 6294 { 6295 struct amdgpu_device *adev = ring->adev; 6296 struct v10_de_ib_state de_payload = {0}; 6297 uint64_t offset, gds_addr, de_payload_gpu_addr; 6298 void *de_payload_cpu_addr; 6299 int cnt; 6300 6301 offset = offsetof(struct v10_gfx_meta_data, de_payload); 6302 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 6303 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 6304 6305 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 6306 AMDGPU_CSA_SIZE - adev->gds.gds_size, 6307 PAGE_SIZE); 6308 6309 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 6310 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 6311 6312 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 6313 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 6314 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 6315 WRITE_DATA_DST_SEL(8) | 6316 WR_CONFIRM) | 6317 WRITE_DATA_CACHE_POLICY(0)); 6318 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 6319 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 6320 6321 if (resume) 6322 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 6323 sizeof(de_payload) >> 2); 6324 else 6325 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 6326 sizeof(de_payload) >> 2); 6327 } 6328 6329 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 6330 bool secure) 6331 { 6332 uint32_t v = secure ? FRAME_TMZ : 0; 6333 6334 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 6335 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 6336 } 6337 6338 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6339 uint32_t reg_val_offs) 6340 { 6341 struct amdgpu_device *adev = ring->adev; 6342 6343 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6344 amdgpu_ring_write(ring, 0 | /* src: register*/ 6345 (5 << 8) | /* dst: memory */ 6346 (1 << 20)); /* write confirm */ 6347 amdgpu_ring_write(ring, reg); 6348 amdgpu_ring_write(ring, 0); 6349 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6350 reg_val_offs * 4)); 6351 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6352 reg_val_offs * 4)); 6353 } 6354 6355 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6356 uint32_t val) 6357 { 6358 uint32_t cmd = 0; 6359 6360 switch (ring->funcs->type) { 6361 case AMDGPU_RING_TYPE_GFX: 6362 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6363 break; 6364 case AMDGPU_RING_TYPE_KIQ: 6365 cmd = (1 << 16); /* no inc addr */ 6366 break; 6367 default: 6368 cmd = WR_CONFIRM; 6369 break; 6370 } 6371 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6372 amdgpu_ring_write(ring, cmd); 6373 amdgpu_ring_write(ring, reg); 6374 amdgpu_ring_write(ring, 0); 6375 amdgpu_ring_write(ring, val); 6376 } 6377 6378 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 6379 uint32_t val, uint32_t mask) 6380 { 6381 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 6382 } 6383 6384 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 6385 uint32_t reg0, uint32_t reg1, 6386 uint32_t ref, uint32_t mask) 6387 { 6388 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6389 6390 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 6391 ref, mask, 0x20); 6392 } 6393 6394 static void 6395 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6396 uint32_t me, uint32_t pipe, 6397 enum amdgpu_interrupt_state state) 6398 { 6399 uint32_t cp_int_cntl, cp_int_cntl_reg; 6400 6401 if (!me) { 6402 switch (pipe) { 6403 case 0: 6404 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 6405 break; 6406 case 1: 6407 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 6408 break; 6409 default: 6410 DRM_DEBUG("invalid pipe %d\n", pipe); 6411 return; 6412 } 6413 } else { 6414 DRM_DEBUG("invalid me %d\n", me); 6415 return; 6416 } 6417 6418 switch (state) { 6419 case AMDGPU_IRQ_STATE_DISABLE: 6420 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6421 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6422 TIME_STAMP_INT_ENABLE, 0); 6423 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6424 GENERIC0_INT_ENABLE, 0); 6425 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6426 break; 6427 case AMDGPU_IRQ_STATE_ENABLE: 6428 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6429 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6430 TIME_STAMP_INT_ENABLE, 1); 6431 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6432 GENERIC0_INT_ENABLE, 1); 6433 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6434 break; 6435 default: 6436 break; 6437 } 6438 } 6439 6440 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6441 int me, int pipe, 6442 enum amdgpu_interrupt_state state) 6443 { 6444 u32 mec_int_cntl, mec_int_cntl_reg; 6445 6446 /* 6447 * amdgpu controls only the first MEC. That's why this function only 6448 * handles the setting of interrupts for this specific MEC. All other 6449 * pipes' interrupts are set by amdkfd. 6450 */ 6451 6452 if (me == 1) { 6453 switch (pipe) { 6454 case 0: 6455 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6456 break; 6457 case 1: 6458 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 6459 break; 6460 case 2: 6461 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 6462 break; 6463 case 3: 6464 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 6465 break; 6466 default: 6467 DRM_DEBUG("invalid pipe %d\n", pipe); 6468 return; 6469 } 6470 } else { 6471 DRM_DEBUG("invalid me %d\n", me); 6472 return; 6473 } 6474 6475 switch (state) { 6476 case AMDGPU_IRQ_STATE_DISABLE: 6477 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6478 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6479 TIME_STAMP_INT_ENABLE, 0); 6480 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6481 GENERIC0_INT_ENABLE, 0); 6482 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6483 break; 6484 case AMDGPU_IRQ_STATE_ENABLE: 6485 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6486 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6487 TIME_STAMP_INT_ENABLE, 1); 6488 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6489 GENERIC0_INT_ENABLE, 1); 6490 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6491 break; 6492 default: 6493 break; 6494 } 6495 } 6496 6497 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6498 struct amdgpu_irq_src *src, 6499 unsigned type, 6500 enum amdgpu_interrupt_state state) 6501 { 6502 switch (type) { 6503 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6504 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state); 6505 break; 6506 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP: 6507 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state); 6508 break; 6509 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6510 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6511 break; 6512 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6513 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6514 break; 6515 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6516 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6517 break; 6518 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6519 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6520 break; 6521 default: 6522 break; 6523 } 6524 return 0; 6525 } 6526 6527 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, 6528 struct amdgpu_irq_src *source, 6529 struct amdgpu_iv_entry *entry) 6530 { 6531 u32 doorbell_offset = entry->src_data[0]; 6532 6533 DRM_DEBUG("IH: CP EOP\n"); 6534 6535 if (!adev->gfx.disable_kq) { 6536 u8 me_id = (entry->ring_id & 0x0c) >> 2; 6537 u8 pipe_id = (entry->ring_id & 0x03) >> 0; 6538 u8 queue_id = (entry->ring_id & 0x70) >> 4; 6539 struct amdgpu_ring *ring; 6540 int i; 6541 6542 switch (me_id) { 6543 case 0: 6544 /* 6545 * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, 6546 * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). 6547 * Require a strict (me,pipe,queue) match so userq gfx 6548 * EOPs fall through to amdgpu_userq_process_fence_irq(). 6549 */ 6550 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6551 ring = &adev->gfx.gfx_ring[i]; 6552 if ((ring->me == me_id) && 6553 (ring->pipe == pipe_id) && 6554 (ring->queue == queue_id)) { 6555 amdgpu_fence_process(ring); 6556 return 0; 6557 } 6558 } 6559 break; 6560 case 1: 6561 case 2: 6562 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6563 ring = &adev->gfx.compute_ring[i]; 6564 /* Per-queue interrupt is supported for MEC starting from VI. 6565 * The interrupt can only be enabled/disabled per pipe instead 6566 * of per queue. 6567 */ 6568 if ((ring->me == me_id) && 6569 (ring->pipe == pipe_id) && 6570 (ring->queue == queue_id)) { 6571 amdgpu_fence_process(ring); 6572 return 0; 6573 } 6574 } 6575 break; 6576 default: 6577 break; 6578 } 6579 } 6580 6581 if (adev->enable_mes && doorbell_offset) 6582 amdgpu_userq_process_fence_irq(adev, doorbell_offset); 6583 6584 return 0; 6585 } 6586 6587 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6588 struct amdgpu_irq_src *source, 6589 unsigned int type, 6590 enum amdgpu_interrupt_state state) 6591 { 6592 u32 cp_int_cntl_reg, cp_int_cntl; 6593 int i, j; 6594 6595 switch (state) { 6596 case AMDGPU_IRQ_STATE_DISABLE: 6597 case AMDGPU_IRQ_STATE_ENABLE: 6598 for (i = 0; i < adev->gfx.me.num_me; i++) { 6599 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6600 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6601 6602 if (cp_int_cntl_reg) { 6603 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6604 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6605 PRIV_REG_INT_ENABLE, 6606 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6607 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6608 } 6609 } 6610 } 6611 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6612 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6613 /* MECs start at 1 */ 6614 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6615 6616 if (cp_int_cntl_reg) { 6617 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6618 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6619 PRIV_REG_INT_ENABLE, 6620 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6621 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6622 } 6623 } 6624 } 6625 break; 6626 default: 6627 break; 6628 } 6629 6630 return 0; 6631 } 6632 6633 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev, 6634 struct amdgpu_irq_src *source, 6635 unsigned type, 6636 enum amdgpu_interrupt_state state) 6637 { 6638 u32 cp_int_cntl_reg, cp_int_cntl; 6639 int i, j; 6640 6641 switch (state) { 6642 case AMDGPU_IRQ_STATE_DISABLE: 6643 case AMDGPU_IRQ_STATE_ENABLE: 6644 for (i = 0; i < adev->gfx.me.num_me; i++) { 6645 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6646 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6647 6648 if (cp_int_cntl_reg) { 6649 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6650 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6651 OPCODE_ERROR_INT_ENABLE, 6652 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6653 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6654 } 6655 } 6656 } 6657 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6658 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6659 /* MECs start at 1 */ 6660 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6661 6662 if (cp_int_cntl_reg) { 6663 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6664 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6665 OPCODE_ERROR_INT_ENABLE, 6666 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6667 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6668 } 6669 } 6670 } 6671 break; 6672 default: 6673 break; 6674 } 6675 return 0; 6676 } 6677 6678 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6679 struct amdgpu_irq_src *source, 6680 unsigned int type, 6681 enum amdgpu_interrupt_state state) 6682 { 6683 u32 cp_int_cntl_reg, cp_int_cntl; 6684 int i, j; 6685 6686 switch (state) { 6687 case AMDGPU_IRQ_STATE_DISABLE: 6688 case AMDGPU_IRQ_STATE_ENABLE: 6689 for (i = 0; i < adev->gfx.me.num_me; i++) { 6690 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6691 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6692 6693 if (cp_int_cntl_reg) { 6694 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6695 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6696 PRIV_INSTR_INT_ENABLE, 6697 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6698 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6699 } 6700 } 6701 } 6702 break; 6703 default: 6704 break; 6705 } 6706 6707 return 0; 6708 } 6709 6710 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, 6711 struct amdgpu_iv_entry *entry) 6712 { 6713 u32 doorbell_offset = entry->src_data[0] & AMDGPU_CTXID0_DOORBELL_ID_MASK; 6714 6715 /* 6716 * Try KQ first by ring_id (HW slot is authoritative). The 6717 * KMD compute_hqd_mask contract guarantees KCQ and user queues 6718 * never share a HW slot. 6719 */ 6720 if (!adev->gfx.disable_kq) { 6721 u8 me_id = (entry->ring_id & 0x0c) >> 2; 6722 u8 pipe_id = (entry->ring_id & 0x03) >> 0; 6723 u8 queue_id = (entry->ring_id & 0x70) >> 4; 6724 struct amdgpu_ring *ring; 6725 int i; 6726 6727 switch (me_id) { 6728 case 0: 6729 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6730 ring = &adev->gfx.gfx_ring[i]; 6731 if (ring->me == me_id && ring->pipe == pipe_id && 6732 ring->queue == queue_id) { 6733 drm_sched_fault(&ring->sched); 6734 return; 6735 } 6736 } 6737 break; 6738 case 1: 6739 case 2: 6740 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6741 ring = &adev->gfx.compute_ring[i]; 6742 if (ring->me == me_id && ring->pipe == pipe_id && 6743 ring->queue == queue_id) { 6744 drm_sched_fault(&ring->sched); 6745 return; 6746 } 6747 } 6748 break; 6749 default: 6750 BUG(); 6751 break; 6752 } 6753 } 6754 6755 /* No KQ matched: HW slot is a MES-scheduled user queue. */ 6756 if (adev->enable_mes && doorbell_offset) 6757 amdgpu_userq_process_reset_irq(adev, entry->pasid, 6758 doorbell_offset); 6759 } 6760 6761 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev, 6762 struct amdgpu_irq_src *source, 6763 struct amdgpu_iv_entry *entry) 6764 { 6765 DRM_ERROR("Illegal register access in command stream\n"); 6766 gfx_v11_0_handle_priv_fault(adev, entry); 6767 return 0; 6768 } 6769 6770 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev, 6771 struct amdgpu_irq_src *source, 6772 struct amdgpu_iv_entry *entry) 6773 { 6774 DRM_ERROR("Illegal opcode in command stream\n"); 6775 gfx_v11_0_handle_priv_fault(adev, entry); 6776 return 0; 6777 } 6778 6779 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev, 6780 struct amdgpu_irq_src *source, 6781 struct amdgpu_iv_entry *entry) 6782 { 6783 DRM_ERROR("Illegal instruction in command stream\n"); 6784 gfx_v11_0_handle_priv_fault(adev, entry); 6785 return 0; 6786 } 6787 6788 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev, 6789 struct amdgpu_irq_src *source, 6790 struct amdgpu_iv_entry *entry) 6791 { 6792 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq) 6793 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry); 6794 6795 return 0; 6796 } 6797 6798 #if 0 6799 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev, 6800 struct amdgpu_irq_src *src, 6801 unsigned int type, 6802 enum amdgpu_interrupt_state state) 6803 { 6804 uint32_t tmp, target; 6805 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring); 6806 6807 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6808 target += ring->pipe; 6809 6810 switch (type) { 6811 case AMDGPU_CP_KIQ_IRQ_DRIVER0: 6812 if (state == AMDGPU_IRQ_STATE_DISABLE) { 6813 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6814 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6815 GENERIC2_INT_ENABLE, 0); 6816 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6817 6818 tmp = RREG32_SOC15_IP(GC, target); 6819 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6820 GENERIC2_INT_ENABLE, 0); 6821 WREG32_SOC15_IP(GC, target, tmp); 6822 } else { 6823 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6824 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6825 GENERIC2_INT_ENABLE, 1); 6826 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6827 6828 tmp = RREG32_SOC15_IP(GC, target); 6829 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6830 GENERIC2_INT_ENABLE, 1); 6831 WREG32_SOC15_IP(GC, target, tmp); 6832 } 6833 break; 6834 default: 6835 BUG(); /* kiq only support GENERIC2_INT now */ 6836 break; 6837 } 6838 return 0; 6839 } 6840 #endif 6841 6842 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring) 6843 { 6844 const unsigned int gcr_cntl = 6845 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) | 6846 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) | 6847 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) | 6848 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) | 6849 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) | 6850 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) | 6851 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) | 6852 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1); 6853 6854 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ 6855 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6)); 6856 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */ 6857 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6858 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6859 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6860 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6861 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6862 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ 6863 } 6864 6865 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, 6866 unsigned int vmid, 6867 struct amdgpu_fence *timedout_fence) 6868 { 6869 struct amdgpu_device *adev = ring->adev; 6870 bool use_mmio = adev->gfx.me.use_mmio_for_reset; 6871 6872 return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio); 6873 } 6874 6875 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, 6876 unsigned int vmid, 6877 struct amdgpu_fence *timedout_fence) 6878 { 6879 struct amdgpu_device *adev = ring->adev; 6880 6881 return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL); 6882 } 6883 6884 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 6885 { 6886 struct amdgpu_device *adev = ip_block->adev; 6887 uint32_t i, j, k, reg, index = 0; 6888 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6889 6890 if (!adev->gfx.ip_dump_core) 6891 return; 6892 6893 for (i = 0; i < reg_count; i++) 6894 drm_printf(p, "%-50s \t 0x%08x\n", 6895 gc_reg_list_11_0[i].reg_name, 6896 adev->gfx.ip_dump_core[i]); 6897 6898 /* print compute queue registers for all instances */ 6899 if (!adev->gfx.ip_dump_compute_queues) 6900 return; 6901 6902 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6903 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n", 6904 adev->gfx.mec.num_mec, 6905 adev->gfx.mec.num_pipe_per_mec, 6906 adev->gfx.mec.num_queue_per_pipe); 6907 6908 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6909 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6910 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6911 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k); 6912 for (reg = 0; reg < reg_count; reg++) { 6913 if (i && gc_cp_reg_list_11[reg].reg_offset == regCP_MEC_ME1_HEADER_DUMP) 6914 drm_printf(p, "%-50s \t 0x%08x\n", 6915 "regCP_MEC_ME2_HEADER_DUMP", 6916 adev->gfx.ip_dump_compute_queues[index + reg]); 6917 else 6918 drm_printf(p, "%-50s \t 0x%08x\n", 6919 gc_cp_reg_list_11[reg].reg_name, 6920 adev->gfx.ip_dump_compute_queues[index + reg]); 6921 } 6922 index += reg_count; 6923 } 6924 } 6925 } 6926 6927 /* print gfx queue registers for all instances */ 6928 if (!adev->gfx.ip_dump_gfx_queues) 6929 return; 6930 6931 index = 0; 6932 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 6933 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n", 6934 adev->gfx.me.num_me, 6935 adev->gfx.me.num_pipe_per_me, 6936 adev->gfx.me.num_queue_per_pipe); 6937 6938 for (i = 0; i < adev->gfx.me.num_me; i++) { 6939 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6940 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 6941 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k); 6942 for (reg = 0; reg < reg_count; reg++) { 6943 drm_printf(p, "%-50s \t 0x%08x\n", 6944 gc_gfx_queue_reg_list_11[reg].reg_name, 6945 adev->gfx.ip_dump_gfx_queues[index + reg]); 6946 } 6947 index += reg_count; 6948 } 6949 } 6950 } 6951 } 6952 6953 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block) 6954 { 6955 struct amdgpu_device *adev = ip_block->adev; 6956 uint32_t i, j, k, reg, index = 0; 6957 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6958 6959 if (!adev->gfx.ip_dump_core) 6960 return; 6961 6962 amdgpu_gfx_off_ctrl(adev, false); 6963 for (i = 0; i < reg_count; i++) 6964 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i])); 6965 amdgpu_gfx_off_ctrl(adev, true); 6966 6967 /* dump compute queue registers for all instances */ 6968 if (!adev->gfx.ip_dump_compute_queues) 6969 return; 6970 6971 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6972 amdgpu_gfx_off_ctrl(adev, false); 6973 mutex_lock(&adev->srbm_mutex); 6974 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6975 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6976 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6977 /* ME0 is for GFX so start from 1 for CP */ 6978 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0); 6979 for (reg = 0; reg < reg_count; reg++) { 6980 if (i && 6981 gc_cp_reg_list_11[reg].reg_offset == 6982 regCP_MEC_ME1_HEADER_DUMP) 6983 adev->gfx.ip_dump_compute_queues[index + reg] = 6984 RREG32(SOC15_REG_OFFSET(GC, 0, 6985 regCP_MEC_ME2_HEADER_DUMP)); 6986 else 6987 adev->gfx.ip_dump_compute_queues[index + reg] = 6988 RREG32(SOC15_REG_ENTRY_OFFSET( 6989 gc_cp_reg_list_11[reg])); 6990 } 6991 index += reg_count; 6992 } 6993 } 6994 } 6995 soc21_grbm_select(adev, 0, 0, 0, 0); 6996 mutex_unlock(&adev->srbm_mutex); 6997 amdgpu_gfx_off_ctrl(adev, true); 6998 6999 /* dump gfx queue registers for all instances */ 7000 if (!adev->gfx.ip_dump_gfx_queues) 7001 return; 7002 7003 index = 0; 7004 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 7005 amdgpu_gfx_off_ctrl(adev, false); 7006 mutex_lock(&adev->srbm_mutex); 7007 for (i = 0; i < adev->gfx.me.num_me; i++) { 7008 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 7009 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 7010 soc21_grbm_select(adev, i, j, k, 0); 7011 7012 for (reg = 0; reg < reg_count; reg++) { 7013 adev->gfx.ip_dump_gfx_queues[index + reg] = 7014 RREG32(SOC15_REG_ENTRY_OFFSET( 7015 gc_gfx_queue_reg_list_11[reg])); 7016 } 7017 index += reg_count; 7018 } 7019 } 7020 } 7021 soc21_grbm_select(adev, 0, 0, 0, 0); 7022 mutex_unlock(&adev->srbm_mutex); 7023 amdgpu_gfx_off_ctrl(adev, true); 7024 } 7025 7026 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring) 7027 { 7028 /* Emit the cleaner shader */ 7029 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0)); 7030 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */ 7031 } 7032 7033 static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring) 7034 { 7035 amdgpu_gfx_profile_ring_begin_use(ring); 7036 7037 amdgpu_gfx_enforce_isolation_ring_begin_use(ring); 7038 } 7039 7040 static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring) 7041 { 7042 amdgpu_gfx_profile_ring_end_use(ring); 7043 7044 amdgpu_gfx_enforce_isolation_ring_end_use(ring); 7045 } 7046 7047 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { 7048 .name = "gfx_v11_0", 7049 .early_init = gfx_v11_0_early_init, 7050 .sw_init = gfx_v11_0_sw_init, 7051 .sw_fini = gfx_v11_0_sw_fini, 7052 .hw_init = gfx_v11_0_hw_init, 7053 .hw_fini = gfx_v11_0_hw_fini, 7054 .suspend = gfx_v11_0_suspend, 7055 .resume = gfx_v11_0_resume, 7056 .is_idle = gfx_v11_0_is_idle, 7057 .wait_for_idle = gfx_v11_0_wait_for_idle, 7058 .soft_reset = gfx_v11_0_soft_reset, 7059 .set_clockgating_state = gfx_v11_0_set_clockgating_state, 7060 .set_powergating_state = gfx_v11_0_set_powergating_state, 7061 .get_clockgating_state = gfx_v11_0_get_clockgating_state, 7062 .dump_ip_state = gfx_v11_ip_dump, 7063 .print_ip_state = gfx_v11_ip_print, 7064 }; 7065 7066 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = { 7067 .type = AMDGPU_RING_TYPE_GFX, 7068 .align_mask = 0xff, 7069 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7070 .support_64bit_ptrs = true, 7071 .secure_submission_supported = true, 7072 .get_rptr = gfx_v11_0_ring_get_rptr_gfx, 7073 .get_wptr = gfx_v11_0_ring_get_wptr_gfx, 7074 .set_wptr = gfx_v11_0_ring_set_wptr_gfx, 7075 .emit_frame_size = /* totally 247 maximum if 16 IBs */ 7076 5 + /* update_spm_vmid */ 7077 5 + /* COND_EXEC */ 7078 22 + /* SET_Q_PREEMPTION_MODE */ 7079 7 + /* PIPELINE_SYNC */ 7080 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7081 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7082 4 + /* VM_FLUSH */ 7083 8 + /* FENCE for VM_FLUSH */ 7084 20 + /* GDS switch */ 7085 5 + /* COND_EXEC */ 7086 7 + /* HDP_flush */ 7087 4 + /* VGT_flush */ 7088 31 + /* DE_META */ 7089 3 + /* CNTX_CTRL */ 7090 5 + /* HDP_INVL */ 7091 22 + /* SET_Q_PREEMPTION_MODE */ 7092 8 + 8 + /* FENCE x2 */ 7093 8 + /* gfx_v11_0_emit_mem_sync */ 7094 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7095 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */ 7096 .emit_ib = gfx_v11_0_ring_emit_ib_gfx, 7097 .emit_fence = gfx_v11_0_ring_emit_fence, 7098 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7099 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7100 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7101 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7102 .test_ring = gfx_v11_0_ring_test_ring, 7103 .test_ib = gfx_v11_0_ring_test_ib, 7104 .insert_nop = gfx_v11_ring_insert_nop, 7105 .pad_ib = amdgpu_ring_generic_pad_ib, 7106 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl, 7107 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow, 7108 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec, 7109 .preempt_ib = amdgpu_gfx_ring_preempt_ib, 7110 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl, 7111 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7112 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7113 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7114 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7115 .reset = gfx_v11_0_reset_kgq, 7116 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7117 .begin_use = gfx_v11_0_ring_begin_use, 7118 .end_use = gfx_v11_0_ring_end_use, 7119 }; 7120 7121 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = { 7122 .type = AMDGPU_RING_TYPE_COMPUTE, 7123 .align_mask = 0xff, 7124 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7125 .support_64bit_ptrs = true, 7126 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7127 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7128 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7129 .emit_frame_size = 7130 5 + /* update_spm_vmid */ 7131 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7132 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7133 5 + /* hdp invalidate */ 7134 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7135 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7136 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7137 2 + /* gfx_v11_0_ring_emit_vm_flush */ 7138 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */ 7139 8 + /* gfx_v11_0_emit_mem_sync */ 7140 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7141 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7142 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7143 .emit_fence = gfx_v11_0_ring_emit_fence, 7144 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7145 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7146 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7147 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7148 .test_ring = gfx_v11_0_ring_test_ring, 7149 .test_ib = gfx_v11_0_ring_test_ib, 7150 .insert_nop = gfx_v11_ring_insert_nop, 7151 .pad_ib = amdgpu_ring_generic_pad_ib, 7152 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7153 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7154 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7155 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7156 .reset = gfx_v11_0_reset_kcq, 7157 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7158 .begin_use = gfx_v11_0_ring_begin_use, 7159 .end_use = gfx_v11_0_ring_end_use, 7160 }; 7161 7162 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = { 7163 .type = AMDGPU_RING_TYPE_KIQ, 7164 .align_mask = 0xff, 7165 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7166 .support_64bit_ptrs = true, 7167 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7168 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7169 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7170 .emit_frame_size = 7171 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7172 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7173 5 + /*hdp invalidate */ 7174 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7175 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7176 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7177 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7178 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7179 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7180 .emit_fence = gfx_v11_0_ring_emit_fence_kiq, 7181 .test_ring = gfx_v11_0_ring_test_ring, 7182 .test_ib = gfx_v11_0_ring_test_ib, 7183 .insert_nop = amdgpu_ring_insert_nop, 7184 .pad_ib = amdgpu_ring_generic_pad_ib, 7185 .emit_rreg = gfx_v11_0_ring_emit_rreg, 7186 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7187 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7188 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7189 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7190 }; 7191 7192 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev) 7193 { 7194 int i; 7195 7196 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq; 7197 7198 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7199 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx; 7200 7201 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7202 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute; 7203 } 7204 7205 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = { 7206 .set = gfx_v11_0_set_eop_interrupt_state, 7207 .process = gfx_v11_0_eop_irq, 7208 }; 7209 7210 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = { 7211 .set = gfx_v11_0_set_priv_reg_fault_state, 7212 .process = gfx_v11_0_priv_reg_irq, 7213 }; 7214 7215 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = { 7216 .set = gfx_v11_0_set_bad_op_fault_state, 7217 .process = gfx_v11_0_bad_op_irq, 7218 }; 7219 7220 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = { 7221 .set = gfx_v11_0_set_priv_inst_fault_state, 7222 .process = gfx_v11_0_priv_inst_irq, 7223 }; 7224 7225 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = { 7226 .process = gfx_v11_0_rlc_gc_fed_irq, 7227 }; 7228 7229 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev) 7230 { 7231 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7232 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs; 7233 7234 adev->gfx.priv_reg_irq.num_types = 1; 7235 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs; 7236 7237 adev->gfx.bad_op_irq.num_types = 1; 7238 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs; 7239 7240 adev->gfx.priv_inst_irq.num_types = 1; 7241 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs; 7242 7243 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */ 7244 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs; 7245 7246 } 7247 7248 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev) 7249 { 7250 if (adev->flags & AMD_IS_APU) 7251 adev->gfx.imu.mode = MISSION_MODE; 7252 else 7253 adev->gfx.imu.mode = DEBUG_MODE; 7254 7255 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs; 7256 } 7257 7258 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev) 7259 { 7260 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs; 7261 } 7262 7263 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev) 7264 { 7265 unsigned total_cu = adev->gfx.config.max_cu_per_sh * 7266 adev->gfx.config.max_sh_per_se * 7267 adev->gfx.config.max_shader_engines; 7268 7269 adev->gds.gds_size = 0x1000; 7270 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1; 7271 adev->gds.gws_size = 64; 7272 adev->gds.oa_size = 16; 7273 } 7274 7275 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev) 7276 { 7277 /* set gfx eng mqd */ 7278 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size = 7279 sizeof(struct v11_gfx_mqd); 7280 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd = 7281 gfx_v11_0_gfx_mqd_init; 7282 /* set compute eng mqd */ 7283 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size = 7284 sizeof(struct v11_compute_mqd); 7285 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd = 7286 gfx_v11_0_compute_mqd_init; 7287 } 7288 7289 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev, 7290 u32 bitmap) 7291 { 7292 u32 data; 7293 7294 if (!bitmap) 7295 return; 7296 7297 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7298 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7299 7300 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data); 7301 } 7302 7303 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev) 7304 { 7305 u32 data, wgp_bitmask; 7306 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG); 7307 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG); 7308 7309 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7310 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7311 7312 wgp_bitmask = 7313 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1); 7314 7315 return (~data) & wgp_bitmask; 7316 } 7317 7318 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev) 7319 { 7320 u32 wgp_idx, wgp_active_bitmap; 7321 u32 cu_bitmap_per_wgp, cu_active_bitmap; 7322 7323 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev); 7324 cu_active_bitmap = 0; 7325 7326 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) { 7327 /* if there is one WGP enabled, it means 2 CUs will be enabled */ 7328 cu_bitmap_per_wgp = 3 << (2 * wgp_idx); 7329 if (wgp_active_bitmap & (1 << wgp_idx)) 7330 cu_active_bitmap |= cu_bitmap_per_wgp; 7331 } 7332 7333 return cu_active_bitmap; 7334 } 7335 7336 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 7337 struct amdgpu_cu_info *cu_info) 7338 { 7339 int i, j, k, counter, active_cu_number = 0; 7340 u32 mask, bitmap; 7341 unsigned disable_masks[8 * 2]; 7342 7343 if (!adev || !cu_info) 7344 return -EINVAL; 7345 7346 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 8, 2); 7347 7348 mutex_lock(&adev->grbm_idx_mutex); 7349 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7350 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7351 bitmap = i * adev->gfx.config.max_sh_per_se + j; 7352 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1)) 7353 continue; 7354 mask = 1; 7355 counter = 0; 7356 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7357 if (i < 8 && j < 2) 7358 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh( 7359 adev, disable_masks[i * 2 + j]); 7360 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev); 7361 7362 /** 7363 * GFX11 could support more than 4 SEs, while the bitmap 7364 * in cu_info struct is 4x4 and ioctl interface struct 7365 * drm_amdgpu_info_device should keep stable. 7366 * So we use last two columns of bitmap to store cu mask for 7367 * SEs 4 to 7, the layout of the bitmap is as below: 7368 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]} 7369 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]} 7370 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]} 7371 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]} 7372 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]} 7373 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]} 7374 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]} 7375 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]} 7376 */ 7377 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap; 7378 7379 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { 7380 if (bitmap & mask) 7381 counter++; 7382 7383 mask <<= 1; 7384 } 7385 active_cu_number += counter; 7386 } 7387 } 7388 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7389 mutex_unlock(&adev->grbm_idx_mutex); 7390 7391 cu_info->number = active_cu_number; 7392 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7393 7394 return 0; 7395 } 7396 7397 const struct amdgpu_ip_block_version gfx_v11_0_ip_block = 7398 { 7399 .type = AMD_IP_BLOCK_TYPE_GFX, 7400 .major = 11, 7401 .minor = 0, 7402 .rev = 0, 7403 .funcs = &gfx_v11_0_ip_funcs, 7404 }; 7405