1 /* 2 * Copyright 2021 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/delay.h> 24 #include <linux/kernel.h> 25 #include <linux/firmware.h> 26 #include <linux/module.h> 27 #include <linux/pci.h> 28 #include "amdgpu.h" 29 #include "amdgpu_gfx.h" 30 #include "amdgpu_psp.h" 31 #include "amdgpu_smu.h" 32 #include "imu_v11_0.h" 33 #include "soc21.h" 34 #include "nvd.h" 35 36 #include "gc/gc_11_0_0_offset.h" 37 #include "gc/gc_11_0_0_sh_mask.h" 38 #include "smuio/smuio_13_0_6_offset.h" 39 #include "smuio/smuio_13_0_6_sh_mask.h" 40 #include "navi10_enum.h" 41 #include "ivsrcid/gfx/irqsrcs_gfx_11_0_0.h" 42 43 #include "soc15.h" 44 #include "clearstate_gfx11.h" 45 #include "v11_structs.h" 46 #include "gfx_v11_0.h" 47 #include "gfx_v11_0_cleaner_shader.h" 48 #include "gfx_v11_0_3.h" 49 #include "nbio_v4_3.h" 50 #include "mes_v11_0.h" 51 #include "mes_userqueue.h" 52 #include "amdgpu_userq_fence.h" 53 54 #define GFX11_NUM_GFX_RINGS 1 55 #define GFX11_MEC_HPD_SIZE 2048 56 57 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 58 #define RLC_PG_DELAY_3_DEFAULT_GC_11_0_1 0x1388 59 60 #define regCGTT_WD_CLK_CTRL 0x5086 61 #define regCGTT_WD_CLK_CTRL_BASE_IDX 1 62 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1 0x4e7e 63 #define regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1_BASE_IDX 1 64 #define regPC_CONFIG_CNTL_1 0x194d 65 #define regPC_CONFIG_CNTL_1_BASE_IDX 1 66 67 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0 0x0030 68 #define regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0_BASE_IDX 1 69 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0 0x0031 70 #define regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0_BASE_IDX 1 71 72 #define regCP_GFX_MQD_CONTROL_DEFAULT 0x00000100 73 #define regCP_GFX_HQD_VMID_DEFAULT 0x00000000 74 #define regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT 0x00000000 75 #define regCP_GFX_HQD_QUANTUM_DEFAULT 0x00000a01 76 #define regCP_GFX_HQD_CNTL_DEFAULT 0x00a00000 77 #define regCP_RB_DOORBELL_CONTROL_DEFAULT 0x00000000 78 #define regCP_GFX_HQD_RPTR_DEFAULT 0x00000000 79 80 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000006 81 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 82 #define regCP_MQD_CONTROL_DEFAULT 0x00000100 83 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509 84 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 85 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000 86 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0be05501 87 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00300000 88 89 MODULE_FIRMWARE("amdgpu/gc_11_0_0_pfp.bin"); 90 MODULE_FIRMWARE("amdgpu/gc_11_0_0_me.bin"); 91 MODULE_FIRMWARE("amdgpu/gc_11_0_0_mec.bin"); 92 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc.bin"); 93 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_kicker.bin"); 94 MODULE_FIRMWARE("amdgpu/gc_11_0_0_rlc_1.bin"); 95 MODULE_FIRMWARE("amdgpu/gc_11_0_0_toc.bin"); 96 MODULE_FIRMWARE("amdgpu/gc_11_0_1_pfp.bin"); 97 MODULE_FIRMWARE("amdgpu/gc_11_0_1_me.bin"); 98 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mec.bin"); 99 MODULE_FIRMWARE("amdgpu/gc_11_0_1_rlc.bin"); 100 MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin"); 101 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin"); 102 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin"); 103 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin"); 104 MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin"); 105 MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin"); 106 MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin"); 107 MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin"); 108 MODULE_FIRMWARE("amdgpu/gc_11_0_4_pfp.bin"); 109 MODULE_FIRMWARE("amdgpu/gc_11_0_4_me.bin"); 110 MODULE_FIRMWARE("amdgpu/gc_11_0_4_mec.bin"); 111 MODULE_FIRMWARE("amdgpu/gc_11_0_4_rlc.bin"); 112 MODULE_FIRMWARE("amdgpu/gc_11_5_0_pfp.bin"); 113 MODULE_FIRMWARE("amdgpu/gc_11_5_0_me.bin"); 114 MODULE_FIRMWARE("amdgpu/gc_11_5_0_mec.bin"); 115 MODULE_FIRMWARE("amdgpu/gc_11_5_0_rlc.bin"); 116 MODULE_FIRMWARE("amdgpu/gc_11_5_1_pfp.bin"); 117 MODULE_FIRMWARE("amdgpu/gc_11_5_1_me.bin"); 118 MODULE_FIRMWARE("amdgpu/gc_11_5_1_mec.bin"); 119 MODULE_FIRMWARE("amdgpu/gc_11_5_1_rlc.bin"); 120 MODULE_FIRMWARE("amdgpu/gc_11_5_2_pfp.bin"); 121 MODULE_FIRMWARE("amdgpu/gc_11_5_2_me.bin"); 122 MODULE_FIRMWARE("amdgpu/gc_11_5_2_mec.bin"); 123 MODULE_FIRMWARE("amdgpu/gc_11_5_2_rlc.bin"); 124 MODULE_FIRMWARE("amdgpu/gc_11_5_3_pfp.bin"); 125 MODULE_FIRMWARE("amdgpu/gc_11_5_3_me.bin"); 126 MODULE_FIRMWARE("amdgpu/gc_11_5_3_mec.bin"); 127 MODULE_FIRMWARE("amdgpu/gc_11_5_3_rlc.bin"); 128 MODULE_FIRMWARE("amdgpu/gc_11_5_4_pfp.bin"); 129 MODULE_FIRMWARE("amdgpu/gc_11_5_4_me.bin"); 130 MODULE_FIRMWARE("amdgpu/gc_11_5_4_mec.bin"); 131 MODULE_FIRMWARE("amdgpu/gc_11_5_4_rlc.bin"); 132 MODULE_FIRMWARE("amdgpu/gc_11_5_6_pfp.bin"); 133 MODULE_FIRMWARE("amdgpu/gc_11_5_6_me.bin"); 134 MODULE_FIRMWARE("amdgpu/gc_11_5_6_mec.bin"); 135 MODULE_FIRMWARE("amdgpu/gc_11_5_6_rlc.bin"); 136 137 static const struct amdgpu_hwip_reg_entry gc_reg_list_11_0[] = { 138 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), 139 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2), 140 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3), 141 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1), 142 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2), 143 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3), 144 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1), 145 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1), 146 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT), 147 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT), 148 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT), 149 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2), 150 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2), 151 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS), 152 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR), 153 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0), 154 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE), 155 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 156 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR), 157 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE), 158 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR), 159 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR), 160 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_BASE), 161 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_RPTR), 162 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB1_WPTR), 163 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 164 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ), 165 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 166 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 167 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 168 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO), 169 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI), 170 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ), 171 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS), 172 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS), 173 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS), 174 SOC15_REG_ENTRY_STR(GC, 0, regGDS_PROTECTION_FAULT), 175 SOC15_REG_ENTRY_STR(GC, 0, regGDS_VM_PROTECTION_FAULT), 176 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS), 177 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2), 178 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS), 179 SOC15_REG_ENTRY_STR(GC, 0, regRLC_UTCL1_STATUS), 180 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS), 181 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES), 182 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS), 183 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS), 184 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL), 185 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS), 186 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG), 187 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL), 188 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL), 189 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC1_INSTR_PNTR), 190 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_DEBUG_INTERRUPT_INSTR_PNTR), 191 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR), 192 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR), 193 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR), 194 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS), 195 /* cp header registers */ 196 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 197 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 198 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 199 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 200 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 201 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 202 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 203 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 204 /* SE status registers */ 205 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0), 206 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1), 207 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE2), 208 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE3), 209 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE4), 210 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE5) 211 }; 212 213 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_11[] = { 214 /* compute registers */ 215 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID), 216 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE), 217 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY), 218 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY), 219 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM), 220 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE), 221 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI), 222 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR), 223 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 224 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 225 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), 226 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL), 227 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR), 228 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI), 229 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR), 230 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL), 231 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST), 232 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR), 233 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI), 234 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL), 235 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR), 236 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR), 237 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS), 238 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO), 239 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI), 240 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL), 241 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET), 242 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE), 243 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET), 244 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE), 245 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE), 246 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR), 247 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM), 248 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO), 249 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI), 250 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET), 251 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT), 252 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET), 253 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS), 254 /* cp header registers */ 255 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 256 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 257 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 258 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 259 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 260 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 261 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 262 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 263 }; 264 265 static const struct amdgpu_hwip_reg_entry gc_gfx_queue_reg_list_11[] = { 266 /* gfx queue registers */ 267 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_ACTIVE), 268 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_VMID), 269 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUEUE_PRIORITY), 270 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUANTUM), 271 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE), 272 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_BASE_HI), 273 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_OFFSET), 274 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CNTL), 275 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_CSMD_RPTR), 276 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR), 277 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_WPTR_HI), 278 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST), 279 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_MAPPED), 280 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_QUE_MGR_CONTROL), 281 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_CONTROL0), 282 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HQD_HQ_STATUS0), 283 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR), 284 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_MQD_BASE_ADDR_HI), 285 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO), 286 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI), 287 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 288 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 289 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 290 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 291 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 292 /* cp header registers */ 293 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 294 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 295 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 296 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 297 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 298 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 299 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 300 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_HEADER_DUMP), 301 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 302 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 303 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 304 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 305 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 306 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 307 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 308 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_HEADER_DUMP), 309 }; 310 311 static const struct soc15_reg_golden golden_settings_gc_11_0[] = { 312 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL, 0x20000000, 0x20000000) 313 }; 314 315 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] = 316 { 317 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_GS_NGG_CLK_CTRL, 0x9fff8fff, 0x00000010), 318 SOC15_REG_GOLDEN_VALUE(GC, 0, regCGTT_WD_CLK_CTRL, 0xffff8fff, 0x00000010), 319 SOC15_REG_GOLDEN_VALUE(GC, 0, regCPF_GCR_CNTL, 0x0007ffff, 0x0000c200), 320 SOC15_REG_GOLDEN_VALUE(GC, 0, regGL2C_CTRL3, 0xffff001b, 0x00f01988), 321 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_CL_ENHANCE, 0xf0ffffff, 0x00880007), 322 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_ENHANCE_3, 0xfffffffd, 0x00000008), 323 SOC15_REG_GOLDEN_VALUE(GC, 0, regPA_SC_VRS_SURFACE_CNTL_1, 0xfff891ff, 0x55480100), 324 SOC15_REG_GOLDEN_VALUE(GC, 0, regTA_CNTL_AUX, 0xf7f7ffff, 0x01030000), 325 SOC15_REG_GOLDEN_VALUE(GC, 0, regTCP_CNTL2, 0xfcffffff, 0x0000000a) 326 }; 327 328 #define DEFAULT_SH_MEM_CONFIG \ 329 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \ 330 (SH_MEM_ALIGNMENT_MODE_UNALIGNED << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \ 331 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT)) 332 333 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev); 334 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev); 335 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev); 336 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev); 337 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev); 338 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev); 339 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev); 340 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 341 struct amdgpu_cu_info *cu_info); 342 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev); 343 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 344 u32 sh_num, u32 instance, int xcc_id); 345 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev); 346 347 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume); 348 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, bool secure); 349 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 350 uint32_t val); 351 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); 352 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 353 uint16_t pasid, uint32_t flush_type, 354 bool all_hub, uint8_t dst_sel); 355 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 356 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 357 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 358 bool enable); 359 360 static void gfx11_kiq_set_resources(struct amdgpu_ring *kiq_ring, uint64_t queue_mask) 361 { 362 struct amdgpu_device *adev = kiq_ring->adev; 363 u64 shader_mc_addr; 364 365 /* Cleaner shader MC address */ 366 shader_mc_addr = adev->gfx.cleaner_shader_gpu_addr >> 8; 367 368 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 369 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | 370 PACKET3_SET_RESOURCES_UNMAP_LATENTY(0xa) | /* unmap_latency: 0xa (~ 1s) */ 371 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ 372 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 373 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 374 amdgpu_ring_write(kiq_ring, lower_32_bits(shader_mc_addr)); /* cleaner shader addr lo */ 375 amdgpu_ring_write(kiq_ring, upper_32_bits(shader_mc_addr)); /* cleaner shader addr hi */ 376 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 377 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */ 378 } 379 380 static void gfx11_kiq_map_queues(struct amdgpu_ring *kiq_ring, 381 struct amdgpu_ring *ring) 382 { 383 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 384 uint64_t wptr_addr = ring->wptr_gpu_addr; 385 uint32_t me = 0, eng_sel = 0; 386 387 switch (ring->funcs->type) { 388 case AMDGPU_RING_TYPE_COMPUTE: 389 me = 1; 390 eng_sel = 0; 391 break; 392 case AMDGPU_RING_TYPE_GFX: 393 me = 0; 394 eng_sel = 4; 395 break; 396 case AMDGPU_RING_TYPE_MES: 397 me = 2; 398 eng_sel = 5; 399 break; 400 default: 401 WARN_ON(1); 402 } 403 404 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 405 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 406 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 407 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 408 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 409 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 410 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 411 PACKET3_MAP_QUEUES_ME((me)) | 412 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 413 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) | /* alloc format: all_on_one_pipe */ 414 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 415 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 416 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 417 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 418 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 419 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 420 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 421 } 422 423 static void gfx11_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 424 struct amdgpu_ring *ring, 425 enum amdgpu_unmap_queues_action action, 426 u64 gpu_addr, u64 seq) 427 { 428 struct amdgpu_device *adev = kiq_ring->adev; 429 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 430 431 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) { 432 amdgpu_mes_unmap_legacy_queue(adev, ring, action, 433 gpu_addr, seq, 0); 434 return; 435 } 436 437 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 438 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 439 PACKET3_UNMAP_QUEUES_ACTION(action) | 440 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 441 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 442 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 443 amdgpu_ring_write(kiq_ring, 444 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 445 446 if (action == PREEMPT_QUEUES_NO_UNMAP) { 447 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 448 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 449 amdgpu_ring_write(kiq_ring, seq); 450 } else { 451 amdgpu_ring_write(kiq_ring, 0); 452 amdgpu_ring_write(kiq_ring, 0); 453 amdgpu_ring_write(kiq_ring, 0); 454 } 455 } 456 457 static void gfx11_kiq_query_status(struct amdgpu_ring *kiq_ring, 458 struct amdgpu_ring *ring, 459 u64 addr, 460 u64 seq) 461 { 462 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 463 464 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 465 amdgpu_ring_write(kiq_ring, 466 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 467 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 468 PACKET3_QUERY_STATUS_COMMAND(2)); 469 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 470 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 471 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 472 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 473 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 474 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 475 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 476 } 477 478 static void gfx11_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 479 uint16_t pasid, uint32_t flush_type, 480 bool all_hub) 481 { 482 gfx_v11_0_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1); 483 } 484 485 static const struct kiq_pm4_funcs gfx_v11_0_kiq_pm4_funcs = { 486 .kiq_set_resources = gfx11_kiq_set_resources, 487 .kiq_map_queues = gfx11_kiq_map_queues, 488 .kiq_unmap_queues = gfx11_kiq_unmap_queues, 489 .kiq_query_status = gfx11_kiq_query_status, 490 .kiq_invalidate_tlbs = gfx11_kiq_invalidate_tlbs, 491 .set_resources_size = 8, 492 .map_queues_size = 7, 493 .unmap_queues_size = 6, 494 .query_status_size = 7, 495 .invalidate_tlbs_size = 2, 496 }; 497 498 static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev) 499 { 500 adev->gfx.kiq[0].pmf = &gfx_v11_0_kiq_pm4_funcs; 501 } 502 503 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev) 504 { 505 if (amdgpu_sriov_vf(adev)) 506 return; 507 508 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 509 case IP_VERSION(11, 0, 1): 510 case IP_VERSION(11, 0, 4): 511 soc15_program_register_sequence(adev, 512 golden_settings_gc_11_0_1, 513 (const u32)ARRAY_SIZE(golden_settings_gc_11_0_1)); 514 break; 515 default: 516 break; 517 } 518 soc15_program_register_sequence(adev, 519 golden_settings_gc_11_0, 520 (const u32)ARRAY_SIZE(golden_settings_gc_11_0)); 521 522 } 523 524 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel, 525 bool wc, uint32_t reg, uint32_t val) 526 { 527 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 528 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) | 529 WRITE_DATA_DST_SEL(0) | (wc ? WR_CONFIRM : 0)); 530 amdgpu_ring_write(ring, reg); 531 amdgpu_ring_write(ring, 0); 532 amdgpu_ring_write(ring, val); 533 } 534 535 static void gfx_v11_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 536 int mem_space, int opt, uint32_t addr0, 537 uint32_t addr1, uint32_t ref, uint32_t mask, 538 uint32_t inv) 539 { 540 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 541 amdgpu_ring_write(ring, 542 /* memory (1) or register (0) */ 543 (WAIT_REG_MEM_MEM_SPACE(mem_space) | 544 WAIT_REG_MEM_OPERATION(opt) | /* wait */ 545 WAIT_REG_MEM_FUNCTION(3) | /* equal */ 546 WAIT_REG_MEM_ENGINE(eng_sel))); 547 548 if (mem_space) 549 WARN_ON(addr0 & 0x3); /* Dword align */ 550 amdgpu_ring_write(ring, addr0); 551 amdgpu_ring_write(ring, addr1); 552 amdgpu_ring_write(ring, ref); 553 amdgpu_ring_write(ring, mask); 554 amdgpu_ring_write(ring, inv); /* poll interval */ 555 } 556 557 static void gfx_v11_ring_insert_nop(struct amdgpu_ring *ring, uint32_t num_nop) 558 { 559 /* Header itself is a NOP packet */ 560 if (num_nop == 1) { 561 amdgpu_ring_write(ring, ring->funcs->nop); 562 return; 563 } 564 565 /* Max HW optimization till 0x3ffe, followed by remaining one NOP at a time*/ 566 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, min(num_nop - 2, 0x3ffe))); 567 568 /* Header is at index 0, followed by num_nops - 1 NOP packet's */ 569 amdgpu_ring_insert_nop(ring, num_nop - 1); 570 } 571 572 static int gfx_v11_0_ring_test_ring(struct amdgpu_ring *ring) 573 { 574 struct amdgpu_device *adev = ring->adev; 575 uint32_t scratch = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 576 uint32_t tmp = 0; 577 unsigned i; 578 int r; 579 580 WREG32(scratch, 0xCAFEDEAD); 581 r = amdgpu_ring_alloc(ring, 5); 582 if (r) { 583 drm_err(adev_to_drm(adev), "cp failed to lock ring %d (%d).\n", 584 ring->idx, r); 585 return r; 586 } 587 588 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) { 589 gfx_v11_0_ring_emit_wreg(ring, scratch, 0xDEADBEEF); 590 } else { 591 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 592 amdgpu_ring_write(ring, scratch - 593 PACKET3_SET_UCONFIG_REG_START); 594 amdgpu_ring_write(ring, 0xDEADBEEF); 595 } 596 amdgpu_ring_commit(ring); 597 598 for (i = 0; i < adev->usec_timeout; i++) { 599 tmp = RREG32(scratch); 600 if (tmp == 0xDEADBEEF) 601 break; 602 if (amdgpu_emu_mode == 1) 603 msleep(1); 604 else 605 udelay(1); 606 } 607 608 if (i >= adev->usec_timeout) 609 r = -ETIMEDOUT; 610 return r; 611 } 612 613 static int gfx_v11_0_ring_test_ib(struct amdgpu_ring *ring, long timeout) 614 { 615 struct amdgpu_device *adev = ring->adev; 616 struct amdgpu_ib ib; 617 struct dma_fence *f = NULL; 618 unsigned index; 619 uint64_t gpu_addr; 620 uint32_t *cpu_ptr; 621 long r; 622 623 /* MES KIQ fw hasn't indirect buffer support for now */ 624 if (adev->enable_mes_kiq && 625 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 626 return 0; 627 628 memset(&ib, 0, sizeof(ib)); 629 630 r = amdgpu_device_wb_get(adev, &index); 631 if (r) 632 return r; 633 634 gpu_addr = adev->wb.gpu_addr + (index * 4); 635 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 636 cpu_ptr = &adev->wb.wb[index]; 637 638 r = amdgpu_ib_get(adev, NULL, 20, AMDGPU_IB_POOL_DIRECT, &ib); 639 if (r) { 640 drm_err(adev_to_drm(adev), "failed to get ib (%ld).\n", r); 641 goto err1; 642 } 643 644 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 645 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM; 646 ib.ptr[2] = lower_32_bits(gpu_addr); 647 ib.ptr[3] = upper_32_bits(gpu_addr); 648 ib.ptr[4] = 0xDEADBEEF; 649 ib.length_dw = 5; 650 651 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 652 if (r) 653 goto err2; 654 655 r = dma_fence_wait_timeout(f, false, timeout); 656 if (r == 0) { 657 r = -ETIMEDOUT; 658 goto err2; 659 } else if (r < 0) { 660 goto err2; 661 } 662 663 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF) 664 r = 0; 665 else 666 r = -EINVAL; 667 err2: 668 amdgpu_ib_free(&ib, NULL); 669 dma_fence_put(f); 670 err1: 671 amdgpu_device_wb_free(adev, index); 672 return r; 673 } 674 675 static void gfx_v11_0_free_microcode(struct amdgpu_device *adev) 676 { 677 amdgpu_ucode_release(&adev->gfx.pfp_fw); 678 amdgpu_ucode_release(&adev->gfx.me_fw); 679 amdgpu_ucode_release(&adev->gfx.rlc_fw); 680 amdgpu_ucode_release(&adev->gfx.mec_fw); 681 682 kfree(adev->gfx.rlc.register_list_format); 683 } 684 685 static int gfx_v11_0_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix) 686 { 687 const struct psp_firmware_header_v1_0 *toc_hdr; 688 int err = 0; 689 690 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, 691 AMDGPU_UCODE_REQUIRED, 692 "amdgpu/%s_toc.bin", ucode_prefix); 693 if (err) 694 goto out; 695 696 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data; 697 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version); 698 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version); 699 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes); 700 adev->psp.toc.start_addr = (uint8_t *)toc_hdr + 701 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes); 702 return 0; 703 out: 704 amdgpu_ucode_release(&adev->psp.toc_fw); 705 return err; 706 } 707 708 static void gfx_v11_0_check_fw_cp_gfx_shadow(struct amdgpu_device *adev) 709 { 710 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 711 case IP_VERSION(11, 0, 0): 712 case IP_VERSION(11, 0, 2): 713 case IP_VERSION(11, 0, 3): 714 if ((adev->gfx.me_fw_version >= 1505) && 715 (adev->gfx.pfp_fw_version >= 1600) && 716 (adev->gfx.mec_fw_version >= 512)) { 717 if (amdgpu_sriov_vf(adev)) 718 adev->gfx.cp_gfx_shadow = true; 719 else 720 adev->gfx.cp_gfx_shadow = false; 721 } 722 break; 723 default: 724 adev->gfx.cp_gfx_shadow = false; 725 break; 726 } 727 } 728 729 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev) 730 { 731 char ucode_prefix[25]; 732 int err; 733 const struct rlc_firmware_header_v2_0 *rlc_hdr; 734 uint16_t version_major; 735 uint16_t version_minor; 736 737 DRM_DEBUG("\n"); 738 739 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 740 err = amdgpu_ucode_request(adev, &adev->gfx.pfp_fw, 741 AMDGPU_UCODE_REQUIRED, 742 "amdgpu/%s_pfp.bin", ucode_prefix); 743 if (err) 744 goto out; 745 /* check pfp fw hdr version to decide if enable rs64 for gfx11.*/ 746 adev->gfx.rs64_enable = amdgpu_ucode_hdr_version( 747 (union amdgpu_firmware_header *) 748 adev->gfx.pfp_fw->data, 2, 0); 749 if (adev->gfx.rs64_enable) { 750 dev_info(adev->dev, "CP RS64 enable\n"); 751 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP); 752 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK); 753 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK); 754 } else { 755 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP); 756 } 757 758 err = amdgpu_ucode_request(adev, &adev->gfx.me_fw, 759 AMDGPU_UCODE_REQUIRED, 760 "amdgpu/%s_me.bin", ucode_prefix); 761 if (err) 762 goto out; 763 if (adev->gfx.rs64_enable) { 764 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME); 765 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK); 766 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK); 767 } else { 768 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME); 769 } 770 771 if (!amdgpu_sriov_vf(adev)) { 772 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 0) && 773 adev->pdev->revision == 0xCE) 774 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 775 AMDGPU_UCODE_REQUIRED, 776 "amdgpu/gc_11_0_0_rlc_1.bin"); 777 else if (amdgpu_is_kicker_fw(adev)) 778 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 779 AMDGPU_UCODE_REQUIRED, 780 "amdgpu/%s_rlc_kicker.bin", ucode_prefix); 781 else 782 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 783 AMDGPU_UCODE_REQUIRED, 784 "amdgpu/%s_rlc.bin", ucode_prefix); 785 if (err) 786 goto out; 787 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 788 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 789 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 790 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 791 if (err) 792 goto out; 793 } 794 795 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 796 AMDGPU_UCODE_REQUIRED, 797 "amdgpu/%s_mec.bin", ucode_prefix); 798 if (err) 799 goto out; 800 if (adev->gfx.rs64_enable) { 801 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC); 802 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK); 803 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK); 804 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK); 805 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK); 806 } else { 807 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1); 808 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT); 809 } 810 811 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 812 err = gfx_v11_0_init_toc_microcode(adev, ucode_prefix); 813 814 /* only one MEC for gfx 11.0.0. */ 815 adev->gfx.mec2_fw = NULL; 816 817 gfx_v11_0_check_fw_cp_gfx_shadow(adev); 818 819 if (adev->gfx.imu.funcs && adev->gfx.imu.funcs->init_microcode) { 820 err = adev->gfx.imu.funcs->init_microcode(adev); 821 if (err) 822 DRM_ERROR("Failed to init imu firmware!\n"); 823 return err; 824 } 825 826 out: 827 if (err) { 828 amdgpu_ucode_release(&adev->gfx.pfp_fw); 829 amdgpu_ucode_release(&adev->gfx.me_fw); 830 amdgpu_ucode_release(&adev->gfx.rlc_fw); 831 amdgpu_ucode_release(&adev->gfx.mec_fw); 832 } 833 834 return err; 835 } 836 837 static u32 gfx_v11_0_get_csb_size(struct amdgpu_device *adev) 838 { 839 u32 count = 0; 840 const struct cs_section_def *sect = NULL; 841 const struct cs_extent_def *ext = NULL; 842 843 /* begin clear state */ 844 count += 2; 845 /* context control state */ 846 count += 3; 847 848 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 849 for (ext = sect->section; ext->extent != NULL; ++ext) { 850 if (sect->id == SECT_CONTEXT) 851 count += 2 + ext->reg_count; 852 else 853 return 0; 854 } 855 } 856 857 /* set PA_SC_TILE_STEERING_OVERRIDE */ 858 count += 3; 859 /* end clear state */ 860 count += 2; 861 /* clear state */ 862 count += 2; 863 864 return count; 865 } 866 867 static void gfx_v11_0_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 868 { 869 u32 count = 0; 870 int ctx_reg_offset; 871 872 if (adev->gfx.rlc.cs_data == NULL) 873 return; 874 if (buffer == NULL) 875 return; 876 877 count = amdgpu_gfx_csb_preamble_start(buffer); 878 count = amdgpu_gfx_csb_data_parser(adev, buffer, count); 879 880 ctx_reg_offset = SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 881 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 882 buffer[count++] = cpu_to_le32(ctx_reg_offset); 883 buffer[count++] = cpu_to_le32(adev->gfx.config.pa_sc_tile_steering_override); 884 885 amdgpu_gfx_csb_preamble_end(buffer, count); 886 } 887 888 static void gfx_v11_0_rlc_fini(struct amdgpu_device *adev) 889 { 890 /* clear state block */ 891 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 892 &adev->gfx.rlc.clear_state_gpu_addr, 893 (void **)&adev->gfx.rlc.cs_ptr); 894 895 /* jump table block */ 896 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 897 &adev->gfx.rlc.cp_table_gpu_addr, 898 (void **)&adev->gfx.rlc.cp_table_ptr); 899 } 900 901 static void gfx_v11_0_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 902 { 903 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 904 905 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[0]; 906 reg_access_ctrl->scratch_reg0 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 907 reg_access_ctrl->scratch_reg1 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG1); 908 reg_access_ctrl->scratch_reg2 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG2); 909 reg_access_ctrl->scratch_reg3 = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG3); 910 reg_access_ctrl->grbm_cntl = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL); 911 reg_access_ctrl->grbm_idx = SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_INDEX); 912 reg_access_ctrl->spare_int = SOC15_REG_OFFSET(GC, 0, regRLC_SPARE_INT_0); 913 adev->gfx.rlc.rlcg_reg_access_supported = true; 914 } 915 916 static int gfx_v11_0_rlc_init(struct amdgpu_device *adev) 917 { 918 const struct cs_section_def *cs_data; 919 int r; 920 921 adev->gfx.rlc.cs_data = gfx11_cs_data; 922 923 cs_data = adev->gfx.rlc.cs_data; 924 925 if (cs_data) { 926 /* init clear state block */ 927 r = amdgpu_gfx_rlc_init_csb(adev); 928 if (r) 929 return r; 930 } 931 932 /* init spm vmid with 0xf */ 933 if (adev->gfx.rlc.funcs->update_spm_vmid) 934 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0, NULL, 0xf); 935 936 return 0; 937 } 938 939 static void gfx_v11_0_mec_fini(struct amdgpu_device *adev) 940 { 941 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 942 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 943 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL); 944 } 945 946 static void gfx_v11_0_me_init(struct amdgpu_device *adev) 947 { 948 bitmap_zero(adev->gfx.me.queue_bitmap, AMDGPU_MAX_GFX_QUEUES); 949 950 amdgpu_gfx_graphics_queue_acquire(adev); 951 } 952 953 static int gfx_v11_0_mec_init(struct amdgpu_device *adev) 954 { 955 int r; 956 u32 *hpd; 957 size_t mec_hpd_size; 958 959 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 960 961 /* take ownership of the relevant compute queues */ 962 amdgpu_gfx_compute_queue_acquire(adev); 963 mec_hpd_size = adev->gfx.num_compute_rings * GFX11_MEC_HPD_SIZE; 964 965 if (mec_hpd_size) { 966 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 967 AMDGPU_GEM_DOMAIN_GTT, 968 &adev->gfx.mec.hpd_eop_obj, 969 &adev->gfx.mec.hpd_eop_gpu_addr, 970 (void **)&hpd); 971 if (r) { 972 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 973 gfx_v11_0_mec_fini(adev); 974 return r; 975 } 976 977 memset(hpd, 0, mec_hpd_size); 978 979 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 980 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 981 } 982 983 return 0; 984 } 985 986 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t wave, uint32_t address) 987 { 988 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 989 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 990 (address << SQ_IND_INDEX__INDEX__SHIFT)); 991 return RREG32_SOC15(GC, 0, regSQ_IND_DATA); 992 } 993 994 static void wave_read_regs(struct amdgpu_device *adev, uint32_t wave, 995 uint32_t thread, uint32_t regno, 996 uint32_t num, uint32_t *out) 997 { 998 WREG32_SOC15(GC, 0, regSQ_IND_INDEX, 999 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 1000 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 1001 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) | 1002 (SQ_IND_INDEX__AUTO_INCR_MASK)); 1003 while (num--) 1004 *(out++) = RREG32_SOC15(GC, 0, regSQ_IND_DATA); 1005 } 1006 1007 static void gfx_v11_0_read_wave_data(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields) 1008 { 1009 /* in gfx11 the SIMD_ID is specified as part of the INSTANCE 1010 * field when performing a select_se_sh so it should be 1011 * zero here */ 1012 WARN_ON(simd != 0); 1013 1014 /* type 3 wave data */ 1015 dst[(*no_fields)++] = 3; 1016 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_STATUS); 1017 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_LO); 1018 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_PC_HI); 1019 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_LO); 1020 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_EXEC_HI); 1021 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID1); 1022 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_HW_ID2); 1023 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_GPR_ALLOC); 1024 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_LDS_ALLOC); 1025 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_TRAPSTS); 1026 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS); 1027 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_STS2); 1028 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_IB_DBG1); 1029 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_M0); 1030 dst[(*no_fields)++] = wave_read_ind(adev, wave, ixSQ_WAVE_MODE); 1031 } 1032 1033 static void gfx_v11_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1034 uint32_t wave, uint32_t start, 1035 uint32_t size, uint32_t *dst) 1036 { 1037 WARN_ON(simd != 0); 1038 1039 wave_read_regs( 1040 adev, wave, 0, start + SQIND_WAVE_SGPRS_OFFSET, size, 1041 dst); 1042 } 1043 1044 static void gfx_v11_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t xcc_id, uint32_t simd, 1045 uint32_t wave, uint32_t thread, 1046 uint32_t start, uint32_t size, 1047 uint32_t *dst) 1048 { 1049 wave_read_regs( 1050 adev, wave, thread, 1051 start + SQIND_WAVE_VGPRS_OFFSET, size, dst); 1052 } 1053 1054 static void gfx_v11_0_select_me_pipe_q(struct amdgpu_device *adev, 1055 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 1056 { 1057 soc21_grbm_select(adev, me, pipe, q, vm); 1058 } 1059 1060 /* all sizes are in bytes */ 1061 #define MQD_SHADOW_BASE_SIZE 73728 1062 #define MQD_SHADOW_BASE_ALIGNMENT 256 1063 #define MQD_FWWORKAREA_SIZE 484 1064 #define MQD_FWWORKAREA_ALIGNMENT 256 1065 1066 static void gfx_v11_0_get_gfx_shadow_info_nocheck(struct amdgpu_device *adev, 1067 struct amdgpu_gfx_shadow_info *shadow_info) 1068 { 1069 /* for gfx */ 1070 shadow_info->shadow_size = MQD_SHADOW_BASE_SIZE; 1071 shadow_info->shadow_alignment = MQD_SHADOW_BASE_ALIGNMENT; 1072 shadow_info->csa_size = MQD_FWWORKAREA_SIZE; 1073 shadow_info->csa_alignment = MQD_FWWORKAREA_ALIGNMENT; 1074 /* for compute */ 1075 shadow_info->eop_size = GFX11_MEC_HPD_SIZE; 1076 shadow_info->eop_alignment = 256; 1077 } 1078 1079 static int gfx_v11_0_get_gfx_shadow_info(struct amdgpu_device *adev, 1080 struct amdgpu_gfx_shadow_info *shadow_info, 1081 bool skip_check) 1082 { 1083 if (adev->gfx.cp_gfx_shadow || skip_check) { 1084 gfx_v11_0_get_gfx_shadow_info_nocheck(adev, shadow_info); 1085 return 0; 1086 } else { 1087 memset(shadow_info, 0, sizeof(struct amdgpu_gfx_shadow_info)); 1088 return -ENOTSUPP; 1089 } 1090 } 1091 1092 static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = { 1093 .get_gpu_clock_counter = &gfx_v11_0_get_gpu_clock_counter, 1094 .select_se_sh = &gfx_v11_0_select_se_sh, 1095 .read_wave_data = &gfx_v11_0_read_wave_data, 1096 .read_wave_sgprs = &gfx_v11_0_read_wave_sgprs, 1097 .read_wave_vgprs = &gfx_v11_0_read_wave_vgprs, 1098 .select_me_pipe_q = &gfx_v11_0_select_me_pipe_q, 1099 .update_perfmon_mgcg = &gfx_v11_0_update_perf_clk, 1100 .get_gfx_shadow_info = &gfx_v11_0_get_gfx_shadow_info, 1101 .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask, 1102 }; 1103 1104 static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev) 1105 { 1106 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1107 case IP_VERSION(11, 0, 0): 1108 case IP_VERSION(11, 0, 2): 1109 adev->gfx.config.max_hw_contexts = 8; 1110 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1111 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1112 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1113 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1114 break; 1115 case IP_VERSION(11, 0, 3): 1116 adev->gfx.ras = &gfx_v11_0_3_ras; 1117 adev->gfx.config.max_hw_contexts = 8; 1118 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1119 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1120 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 1121 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 1122 break; 1123 case IP_VERSION(11, 0, 1): 1124 case IP_VERSION(11, 0, 4): 1125 case IP_VERSION(11, 5, 0): 1126 case IP_VERSION(11, 5, 1): 1127 case IP_VERSION(11, 5, 2): 1128 case IP_VERSION(11, 5, 3): 1129 case IP_VERSION(11, 5, 4): 1130 case IP_VERSION(11, 5, 6): 1131 adev->gfx.config.max_hw_contexts = 8; 1132 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 1133 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 1134 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80; 1135 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x300; 1136 break; 1137 default: 1138 BUG(); 1139 break; 1140 } 1141 1142 return 0; 1143 } 1144 1145 static int gfx_v11_0_gfx_ring_init(struct amdgpu_device *adev, int ring_id, 1146 int me, int pipe, int queue) 1147 { 1148 struct amdgpu_ring *ring; 1149 unsigned int irq_type; 1150 unsigned int hw_prio; 1151 1152 ring = &adev->gfx.gfx_ring[ring_id]; 1153 1154 ring->me = me; 1155 ring->pipe = pipe; 1156 ring->queue = queue; 1157 1158 ring->ring_obj = NULL; 1159 ring->use_doorbell = true; 1160 if (adev->gfx.disable_kq) { 1161 ring->no_scheduler = true; 1162 ring->no_user_submission = true; 1163 } 1164 1165 if (!ring_id) 1166 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1; 1167 else 1168 ring->doorbell_index = adev->doorbell_index.gfx_ring1 << 1; 1169 ring->vm_hub = AMDGPU_GFXHUB(0); 1170 sprintf(ring->name, "gfx_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1171 1172 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + ring->pipe; 1173 hw_prio = amdgpu_gfx_is_high_priority_graphics_queue(adev, ring) ? 1174 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1175 return amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1176 hw_prio, NULL); 1177 } 1178 1179 static int gfx_v11_0_compute_ring_init(struct amdgpu_device *adev, int ring_id, 1180 int mec, int pipe, int queue) 1181 { 1182 int r; 1183 unsigned irq_type; 1184 struct amdgpu_ring *ring; 1185 unsigned int hw_prio; 1186 1187 ring = &adev->gfx.compute_ring[ring_id]; 1188 1189 /* mec0 is me1 */ 1190 ring->me = mec + 1; 1191 ring->pipe = pipe; 1192 ring->queue = queue; 1193 1194 ring->ring_obj = NULL; 1195 ring->use_doorbell = true; 1196 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1; 1197 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr 1198 + (ring_id * GFX11_MEC_HPD_SIZE); 1199 ring->vm_hub = AMDGPU_GFXHUB(0); 1200 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue); 1201 1202 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 1203 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 1204 + ring->pipe; 1205 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 1206 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 1207 /* type-2 packets are deprecated on MEC, use type-3 instead */ 1208 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 1209 hw_prio, NULL); 1210 if (r) 1211 return r; 1212 1213 return 0; 1214 } 1215 1216 static struct { 1217 SOC21_FIRMWARE_ID id; 1218 unsigned int offset; 1219 unsigned int size; 1220 } rlc_autoload_info[SOC21_FIRMWARE_ID_MAX]; 1221 1222 static void gfx_v11_0_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc) 1223 { 1224 RLC_TABLE_OF_CONTENT *ucode = rlc_toc; 1225 1226 while (ucode && (ucode->id > SOC21_FIRMWARE_ID_INVALID) && 1227 (ucode->id < SOC21_FIRMWARE_ID_MAX)) { 1228 rlc_autoload_info[ucode->id].id = ucode->id; 1229 rlc_autoload_info[ucode->id].offset = ucode->offset * 4; 1230 rlc_autoload_info[ucode->id].size = ucode->size * 4; 1231 1232 ucode++; 1233 } 1234 } 1235 1236 static uint32_t gfx_v11_0_calc_toc_total_size(struct amdgpu_device *adev) 1237 { 1238 uint32_t total_size = 0; 1239 SOC21_FIRMWARE_ID id; 1240 1241 gfx_v11_0_parse_rlc_toc(adev, adev->psp.toc.start_addr); 1242 1243 for (id = SOC21_FIRMWARE_ID_RLC_G_UCODE; id < SOC21_FIRMWARE_ID_MAX; id++) 1244 total_size += rlc_autoload_info[id].size; 1245 1246 /* In case the offset in rlc toc ucode is aligned */ 1247 if (total_size < rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset) 1248 total_size = rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].offset + 1249 rlc_autoload_info[SOC21_FIRMWARE_ID_MAX-1].size; 1250 1251 return total_size; 1252 } 1253 1254 static int gfx_v11_0_rlc_autoload_buffer_init(struct amdgpu_device *adev) 1255 { 1256 int r; 1257 uint32_t total_size; 1258 1259 total_size = gfx_v11_0_calc_toc_total_size(adev); 1260 1261 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024, 1262 AMDGPU_GEM_DOMAIN_VRAM | 1263 AMDGPU_GEM_DOMAIN_GTT, 1264 &adev->gfx.rlc.rlc_autoload_bo, 1265 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1266 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1267 1268 if (r) { 1269 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r); 1270 return r; 1271 } 1272 1273 return 0; 1274 } 1275 1276 static void gfx_v11_0_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev, 1277 SOC21_FIRMWARE_ID id, 1278 const void *fw_data, 1279 uint32_t fw_size, 1280 uint32_t *fw_autoload_mask) 1281 { 1282 uint32_t toc_offset; 1283 uint32_t toc_fw_size; 1284 char *ptr = adev->gfx.rlc.rlc_autoload_ptr; 1285 1286 if (id <= SOC21_FIRMWARE_ID_INVALID || id >= SOC21_FIRMWARE_ID_MAX) 1287 return; 1288 1289 toc_offset = rlc_autoload_info[id].offset; 1290 toc_fw_size = rlc_autoload_info[id].size; 1291 1292 if (fw_size == 0) 1293 fw_size = toc_fw_size; 1294 1295 if (fw_size > toc_fw_size) 1296 fw_size = toc_fw_size; 1297 1298 memcpy(ptr + toc_offset, fw_data, fw_size); 1299 1300 if (fw_size < toc_fw_size) 1301 memset(ptr + toc_offset + fw_size, 0, toc_fw_size - fw_size); 1302 1303 if ((id != SOC21_FIRMWARE_ID_RS64_PFP) && (id != SOC21_FIRMWARE_ID_RS64_ME)) 1304 *(uint64_t *)fw_autoload_mask |= 1ULL << id; 1305 } 1306 1307 static void gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev, 1308 uint32_t *fw_autoload_mask) 1309 { 1310 void *data; 1311 uint32_t size; 1312 uint64_t *toc_ptr; 1313 1314 *(uint64_t *)fw_autoload_mask |= 0x1; 1315 1316 DRM_DEBUG("rlc autoload enabled fw: 0x%llx\n", *(uint64_t *)fw_autoload_mask); 1317 1318 data = adev->psp.toc.start_addr; 1319 size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_TOC].size; 1320 1321 toc_ptr = (uint64_t *)data + size / 8 - 1; 1322 *toc_ptr = *(uint64_t *)fw_autoload_mask; 1323 1324 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_TOC, 1325 data, size, fw_autoload_mask); 1326 } 1327 1328 static void gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev, 1329 uint32_t *fw_autoload_mask) 1330 { 1331 const __le32 *fw_data; 1332 uint32_t fw_size; 1333 const struct gfx_firmware_header_v1_0 *cp_hdr; 1334 const struct gfx_firmware_header_v2_0 *cpv2_hdr; 1335 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1336 const struct rlc_firmware_header_v2_2 *rlcv22_hdr; 1337 uint16_t version_major, version_minor; 1338 1339 if (adev->gfx.rs64_enable) { 1340 /* pfp ucode */ 1341 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1342 adev->gfx.pfp_fw->data; 1343 /* instruction */ 1344 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1345 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1346 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1347 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP, 1348 fw_data, fw_size, fw_autoload_mask); 1349 /* data */ 1350 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1351 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1352 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1353 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK, 1354 fw_data, fw_size, fw_autoload_mask); 1355 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_PFP_P1_STACK, 1356 fw_data, fw_size, fw_autoload_mask); 1357 /* me ucode */ 1358 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1359 adev->gfx.me_fw->data; 1360 /* instruction */ 1361 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1362 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1363 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1364 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME, 1365 fw_data, fw_size, fw_autoload_mask); 1366 /* data */ 1367 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1368 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1369 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1370 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P0_STACK, 1371 fw_data, fw_size, fw_autoload_mask); 1372 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_ME_P1_STACK, 1373 fw_data, fw_size, fw_autoload_mask); 1374 /* mec ucode */ 1375 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1376 adev->gfx.mec_fw->data; 1377 /* instruction */ 1378 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1379 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1380 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1381 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC, 1382 fw_data, fw_size, fw_autoload_mask); 1383 /* data */ 1384 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1385 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1386 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1387 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK, 1388 fw_data, fw_size, fw_autoload_mask); 1389 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P1_STACK, 1390 fw_data, fw_size, fw_autoload_mask); 1391 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P2_STACK, 1392 fw_data, fw_size, fw_autoload_mask); 1393 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RS64_MEC_P3_STACK, 1394 fw_data, fw_size, fw_autoload_mask); 1395 } else { 1396 /* pfp ucode */ 1397 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1398 adev->gfx.pfp_fw->data; 1399 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 1400 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1401 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1402 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_PFP, 1403 fw_data, fw_size, fw_autoload_mask); 1404 1405 /* me ucode */ 1406 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1407 adev->gfx.me_fw->data; 1408 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 1409 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1410 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes); 1411 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_ME, 1412 fw_data, fw_size, fw_autoload_mask); 1413 1414 /* mec ucode */ 1415 cp_hdr = (const struct gfx_firmware_header_v1_0 *) 1416 adev->gfx.mec_fw->data; 1417 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1418 le32_to_cpu(cp_hdr->header.ucode_array_offset_bytes)); 1419 fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) - 1420 cp_hdr->jt_size * 4; 1421 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_CP_MEC, 1422 fw_data, fw_size, fw_autoload_mask); 1423 } 1424 1425 /* rlc ucode */ 1426 rlc_hdr = (const struct rlc_firmware_header_v2_0 *) 1427 adev->gfx.rlc_fw->data; 1428 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1429 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes)); 1430 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes); 1431 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLC_G_UCODE, 1432 fw_data, fw_size, fw_autoload_mask); 1433 1434 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1435 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1436 if (version_major == 2) { 1437 if (version_minor >= 2) { 1438 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 1439 1440 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1441 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes)); 1442 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes); 1443 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_UCODE, 1444 fw_data, fw_size, fw_autoload_mask); 1445 1446 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1447 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes)); 1448 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes); 1449 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, SOC21_FIRMWARE_ID_RLX6_DRAM_BOOT, 1450 fw_data, fw_size, fw_autoload_mask); 1451 } 1452 } 1453 } 1454 1455 static void gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev, 1456 uint32_t *fw_autoload_mask) 1457 { 1458 const __le32 *fw_data; 1459 uint32_t fw_size; 1460 const struct sdma_firmware_header_v2_0 *sdma_hdr; 1461 1462 sdma_hdr = (const struct sdma_firmware_header_v2_0 *) 1463 adev->sdma.instance[0].fw->data; 1464 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1465 le32_to_cpu(sdma_hdr->header.ucode_array_offset_bytes)); 1466 fw_size = le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes); 1467 1468 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1469 SOC21_FIRMWARE_ID_SDMA_UCODE_TH0, fw_data, fw_size, fw_autoload_mask); 1470 1471 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1472 le32_to_cpu(sdma_hdr->ctl_ucode_offset)); 1473 fw_size = le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes); 1474 1475 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1476 SOC21_FIRMWARE_ID_SDMA_UCODE_TH1, fw_data, fw_size, fw_autoload_mask); 1477 } 1478 1479 static void gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev, 1480 uint32_t *fw_autoload_mask) 1481 { 1482 const __le32 *fw_data; 1483 unsigned fw_size; 1484 const struct mes_firmware_header_v1_0 *mes_hdr; 1485 int pipe, ucode_id, data_id; 1486 1487 for (pipe = 0; pipe < 2; pipe++) { 1488 if (pipe==0) { 1489 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P0; 1490 data_id = SOC21_FIRMWARE_ID_RS64_MES_P0_STACK; 1491 } else { 1492 ucode_id = SOC21_FIRMWARE_ID_RS64_MES_P1; 1493 data_id = SOC21_FIRMWARE_ID_RS64_MES_P1_STACK; 1494 } 1495 1496 mes_hdr = (const struct mes_firmware_header_v1_0 *) 1497 adev->mes.fw[pipe]->data; 1498 1499 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1500 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); 1501 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); 1502 1503 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1504 ucode_id, fw_data, fw_size, fw_autoload_mask); 1505 1506 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1507 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); 1508 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); 1509 1510 gfx_v11_0_rlc_backdoor_autoload_copy_ucode(adev, 1511 data_id, fw_data, fw_size, fw_autoload_mask); 1512 } 1513 } 1514 1515 static int gfx_v11_0_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) 1516 { 1517 uint32_t rlc_g_offset, rlc_g_size; 1518 uint64_t gpu_addr; 1519 uint32_t autoload_fw_id[2]; 1520 1521 memset(autoload_fw_id, 0, sizeof(uint32_t) * 2); 1522 1523 /* RLC autoload sequence 2: copy ucode */ 1524 gfx_v11_0_rlc_backdoor_autoload_copy_sdma_ucode(adev, autoload_fw_id); 1525 gfx_v11_0_rlc_backdoor_autoload_copy_gfx_ucode(adev, autoload_fw_id); 1526 gfx_v11_0_rlc_backdoor_autoload_copy_mes_ucode(adev, autoload_fw_id); 1527 gfx_v11_0_rlc_backdoor_autoload_copy_toc_ucode(adev, autoload_fw_id); 1528 1529 rlc_g_offset = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].offset; 1530 rlc_g_size = rlc_autoload_info[SOC21_FIRMWARE_ID_RLC_G_UCODE].size; 1531 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset; 1532 1533 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, upper_32_bits(gpu_addr)); 1534 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, lower_32_bits(gpu_addr)); 1535 1536 WREG32_SOC15(GC, 0, regGFX_IMU_RLC_BOOTLOADER_SIZE, rlc_g_size); 1537 1538 /* RLC autoload sequence 3: load IMU fw */ 1539 if (adev->gfx.imu.funcs->load_microcode) 1540 adev->gfx.imu.funcs->load_microcode(adev); 1541 /* RLC autoload sequence 4 init IMU fw */ 1542 if (adev->gfx.imu.funcs->setup_imu) 1543 adev->gfx.imu.funcs->setup_imu(adev); 1544 if (adev->gfx.imu.funcs->start_imu) 1545 adev->gfx.imu.funcs->start_imu(adev); 1546 1547 /* RLC autoload sequence 5 disable gpa mode */ 1548 gfx_v11_0_disable_gpa_mode(adev); 1549 1550 return 0; 1551 } 1552 1553 static void gfx_v11_0_alloc_ip_dump(struct amdgpu_device *adev) 1554 { 1555 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 1556 uint32_t *ptr; 1557 uint32_t inst; 1558 1559 ptr = kcalloc(reg_count, sizeof(uint32_t), GFP_KERNEL); 1560 if (!ptr) { 1561 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 1562 adev->gfx.ip_dump_core = NULL; 1563 } else { 1564 adev->gfx.ip_dump_core = ptr; 1565 } 1566 1567 /* Allocate memory for compute queue registers for all the instances */ 1568 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 1569 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1570 adev->gfx.mec.num_queue_per_pipe; 1571 1572 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1573 if (!ptr) { 1574 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 1575 adev->gfx.ip_dump_compute_queues = NULL; 1576 } else { 1577 adev->gfx.ip_dump_compute_queues = ptr; 1578 } 1579 1580 /* Allocate memory for gfx queue registers for all the instances */ 1581 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 1582 inst = adev->gfx.me.num_me * adev->gfx.me.num_pipe_per_me * 1583 adev->gfx.me.num_queue_per_pipe; 1584 1585 ptr = kcalloc(reg_count * inst, sizeof(uint32_t), GFP_KERNEL); 1586 if (!ptr) { 1587 DRM_ERROR("Failed to allocate memory for GFX Queues IP Dump\n"); 1588 adev->gfx.ip_dump_gfx_queues = NULL; 1589 } else { 1590 adev->gfx.ip_dump_gfx_queues = ptr; 1591 } 1592 } 1593 1594 static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) 1595 { 1596 int i, j, k, r, ring_id; 1597 int xcc_id = 0; 1598 struct amdgpu_device *adev = ip_block->adev; 1599 int num_queue_per_pipe = 1; /* we only enable 1 KGQ per pipe */ 1600 1601 INIT_DELAYED_WORK(&adev->gfx.idle_work, amdgpu_gfx_profile_idle_work_handler); 1602 1603 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1604 case IP_VERSION(11, 0, 0): 1605 case IP_VERSION(11, 0, 1): 1606 case IP_VERSION(11, 0, 2): 1607 case IP_VERSION(11, 0, 3): 1608 case IP_VERSION(11, 0, 4): 1609 case IP_VERSION(11, 5, 0): 1610 case IP_VERSION(11, 5, 1): 1611 case IP_VERSION(11, 5, 2): 1612 case IP_VERSION(11, 5, 3): 1613 case IP_VERSION(11, 5, 4): 1614 case IP_VERSION(11, 5, 6): 1615 adev->gfx.me.num_me = 1; 1616 adev->gfx.me.num_pipe_per_me = 1; 1617 adev->gfx.me.num_queue_per_pipe = 2; 1618 adev->gfx.mec.num_mec = 1; 1619 adev->gfx.mec.num_pipe_per_mec = 4; 1620 adev->gfx.mec.num_queue_per_pipe = 4; 1621 break; 1622 default: 1623 adev->gfx.me.num_me = 1; 1624 adev->gfx.me.num_pipe_per_me = 1; 1625 adev->gfx.me.num_queue_per_pipe = 1; 1626 adev->gfx.mec.num_mec = 1; 1627 adev->gfx.mec.num_pipe_per_mec = 4; 1628 adev->gfx.mec.num_queue_per_pipe = 8; 1629 break; 1630 } 1631 1632 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1633 case IP_VERSION(11, 0, 0): 1634 case IP_VERSION(11, 0, 2): 1635 case IP_VERSION(11, 0, 3): 1636 if (!adev->gfx.disable_uq && 1637 adev->gfx.me_fw_version >= 2420 && 1638 adev->gfx.pfp_fw_version >= 2580 && 1639 adev->gfx.mec_fw_version >= 2650 && 1640 adev->mes.fw_version[0] >= 120) { 1641 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1642 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1643 } 1644 break; 1645 case IP_VERSION(11, 0, 1): 1646 case IP_VERSION(11, 0, 4): 1647 case IP_VERSION(11, 5, 0): 1648 case IP_VERSION(11, 5, 1): 1649 case IP_VERSION(11, 5, 2): 1650 case IP_VERSION(11, 5, 3): 1651 /* add firmware version checks here */ 1652 if (0 && !adev->gfx.disable_uq) { 1653 adev->userq_funcs[AMDGPU_HW_IP_GFX] = &userq_mes_funcs; 1654 adev->userq_funcs[AMDGPU_HW_IP_COMPUTE] = &userq_mes_funcs; 1655 } 1656 break; 1657 default: 1658 break; 1659 } 1660 1661 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1662 case IP_VERSION(11, 0, 0): 1663 case IP_VERSION(11, 0, 2): 1664 case IP_VERSION(11, 0, 3): 1665 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1666 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1667 if (adev->gfx.me_fw_version >= 2280 && 1668 adev->gfx.pfp_fw_version >= 2370 && 1669 adev->gfx.mec_fw_version >= 2450 && 1670 adev->mes.fw_version[0] >= 99) { 1671 adev->gfx.enable_cleaner_shader = true; 1672 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1673 if (r) { 1674 adev->gfx.enable_cleaner_shader = false; 1675 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1676 } 1677 } 1678 break; 1679 case IP_VERSION(11, 0, 1): 1680 case IP_VERSION(11, 0, 4): 1681 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1682 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1683 if (adev->gfx.pfp_fw_version >= 102 && 1684 adev->gfx.mec_fw_version >= 66 && 1685 adev->mes.fw_version[0] >= 128) { 1686 adev->gfx.enable_cleaner_shader = true; 1687 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1688 if (r) { 1689 adev->gfx.enable_cleaner_shader = false; 1690 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1691 } 1692 } 1693 break; 1694 case IP_VERSION(11, 5, 0): 1695 case IP_VERSION(11, 5, 1): 1696 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1697 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1698 if (adev->gfx.mec_fw_version >= 26 && 1699 adev->mes.fw_version[0] >= 114) { 1700 adev->gfx.enable_cleaner_shader = true; 1701 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1702 if (r) { 1703 adev->gfx.enable_cleaner_shader = false; 1704 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1705 } 1706 } 1707 break; 1708 case IP_VERSION(11, 5, 2): 1709 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1710 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1711 if (adev->gfx.me_fw_version >= 12 && 1712 adev->gfx.pfp_fw_version >= 15 && 1713 adev->gfx.mec_fw_version >= 15) { 1714 adev->gfx.enable_cleaner_shader = true; 1715 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1716 if (r) { 1717 adev->gfx.enable_cleaner_shader = false; 1718 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1719 } 1720 } 1721 break; 1722 case IP_VERSION(11, 5, 3): 1723 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1724 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1725 if (adev->gfx.me_fw_version >= 7 && 1726 adev->gfx.pfp_fw_version >= 8 && 1727 adev->gfx.mec_fw_version >= 8) { 1728 adev->gfx.enable_cleaner_shader = true; 1729 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1730 if (r) { 1731 adev->gfx.enable_cleaner_shader = false; 1732 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1733 } 1734 } 1735 break; 1736 case IP_VERSION(11, 5, 4): 1737 adev->gfx.cleaner_shader_ptr = gfx_11_0_3_cleaner_shader_hex; 1738 adev->gfx.cleaner_shader_size = sizeof(gfx_11_0_3_cleaner_shader_hex); 1739 if (adev->gfx.me_fw_version >= 4 && 1740 adev->gfx.pfp_fw_version >= 7 && 1741 adev->gfx.mec_fw_version >= 5) { 1742 adev->gfx.enable_cleaner_shader = true; 1743 r = amdgpu_gfx_cleaner_shader_sw_init(adev, adev->gfx.cleaner_shader_size); 1744 if (r) { 1745 adev->gfx.enable_cleaner_shader = false; 1746 dev_err(adev->dev, "Failed to initialize cleaner shader\n"); 1747 } 1748 } 1749 break; 1750 default: 1751 adev->gfx.enable_cleaner_shader = false; 1752 break; 1753 } 1754 1755 /* Enable CG flag in one VF mode for enabling RLC safe mode enter/exit */ 1756 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3) && 1757 amdgpu_sriov_is_pp_one_vf(adev)) 1758 adev->cg_flags = AMD_CG_SUPPORT_GFX_CGCG; 1759 1760 /* EOP Event */ 1761 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1762 GFX_11_0_0__SRCID__CP_EOP_INTERRUPT, 1763 &adev->gfx.eop_irq); 1764 if (r) 1765 return r; 1766 1767 /* Bad opcode Event */ 1768 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1769 GFX_11_0_0__SRCID__CP_BAD_OPCODE_ERROR, 1770 &adev->gfx.bad_op_irq); 1771 if (r) 1772 return r; 1773 1774 /* Privileged reg */ 1775 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1776 GFX_11_0_0__SRCID__CP_PRIV_REG_FAULT, 1777 &adev->gfx.priv_reg_irq); 1778 if (r) 1779 return r; 1780 1781 /* Privileged inst */ 1782 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GRBM_CP, 1783 GFX_11_0_0__SRCID__CP_PRIV_INSTR_FAULT, 1784 &adev->gfx.priv_inst_irq); 1785 if (r) 1786 return r; 1787 1788 /* FED error */ 1789 r = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_GFX, 1790 GFX_11_0_0__SRCID__RLC_GC_FED_INTERRUPT, 1791 &adev->gfx.rlc_gc_fed_irq); 1792 if (r) 1793 return r; 1794 1795 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1796 1797 gfx_v11_0_me_init(adev); 1798 1799 r = gfx_v11_0_rlc_init(adev); 1800 if (r) { 1801 DRM_ERROR("Failed to init rlc BOs!\n"); 1802 return r; 1803 } 1804 1805 r = gfx_v11_0_mec_init(adev); 1806 if (r) { 1807 DRM_ERROR("Failed to init MEC BOs!\n"); 1808 return r; 1809 } 1810 1811 if (adev->gfx.num_gfx_rings) { 1812 ring_id = 0; 1813 /* set up the gfx ring */ 1814 for (i = 0; i < adev->gfx.me.num_me; i++) { 1815 for (j = 0; j < num_queue_per_pipe; j++) { 1816 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 1817 if (!amdgpu_gfx_is_me_queue_enabled(adev, i, k, j)) 1818 continue; 1819 1820 r = gfx_v11_0_gfx_ring_init(adev, ring_id, 1821 i, k, j); 1822 if (r) 1823 return r; 1824 ring_id++; 1825 } 1826 } 1827 } 1828 } 1829 1830 if (adev->gfx.num_compute_rings) { 1831 ring_id = 0; 1832 /* set up the compute queues - allocate horizontally across pipes */ 1833 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1834 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1835 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1836 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 0, i, 1837 k, j)) 1838 continue; 1839 1840 r = gfx_v11_0_compute_ring_init(adev, ring_id, 1841 i, k, j); 1842 if (r) 1843 return r; 1844 1845 ring_id++; 1846 } 1847 } 1848 } 1849 } 1850 1851 adev->gfx.gfx_supported_reset = 1852 amdgpu_get_soft_full_reset_mask(&adev->gfx.gfx_ring[0]); 1853 adev->gfx.compute_supported_reset = 1854 amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); 1855 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1856 case IP_VERSION(11, 0, 0): 1857 case IP_VERSION(11, 0, 2): 1858 case IP_VERSION(11, 0, 3): 1859 if ((adev->gfx.me_fw_version >= 2280) && 1860 (adev->gfx.mec_fw_version >= 2410) && 1861 !amdgpu_sriov_vf(adev) && 1862 !adev->debug_disable_gpu_ring_reset) { 1863 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1864 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1865 } 1866 break; 1867 default: 1868 if (!amdgpu_sriov_vf(adev) && 1869 !adev->debug_disable_gpu_ring_reset) { 1870 adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1871 adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 1872 } 1873 break; 1874 } 1875 1876 if (!adev->enable_mes_kiq) { 1877 r = amdgpu_gfx_kiq_init(adev, GFX11_MEC_HPD_SIZE, 0); 1878 if (r) { 1879 DRM_ERROR("Failed to init KIQ BOs!\n"); 1880 return r; 1881 } 1882 1883 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 1884 if (r) 1885 return r; 1886 } 1887 1888 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v11_compute_mqd), 0); 1889 if (r) 1890 return r; 1891 1892 /* allocate visible FB for rlc auto-loading fw */ 1893 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 1894 r = gfx_v11_0_rlc_autoload_buffer_init(adev); 1895 if (r) 1896 return r; 1897 } 1898 1899 r = gfx_v11_0_gpu_early_init(adev); 1900 if (r) 1901 return r; 1902 1903 if (amdgpu_gfx_ras_sw_init(adev)) { 1904 dev_err(adev->dev, "Failed to initialize gfx ras block!\n"); 1905 return -EINVAL; 1906 } 1907 1908 gfx_v11_0_alloc_ip_dump(adev); 1909 1910 r = amdgpu_gfx_sysfs_init(adev); 1911 if (r) 1912 return r; 1913 1914 adev->gfx.me.use_mmio_for_reset = false; 1915 adev->gfx.mec.use_mmio_for_reset = true; 1916 1917 mutex_init(&adev->gfx.mec.reset_mutex); 1918 1919 return 0; 1920 } 1921 1922 static void gfx_v11_0_pfp_fini(struct amdgpu_device *adev) 1923 { 1924 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_obj, 1925 &adev->gfx.pfp.pfp_fw_gpu_addr, 1926 (void **)&adev->gfx.pfp.pfp_fw_ptr); 1927 1928 amdgpu_bo_free_kernel(&adev->gfx.pfp.pfp_fw_data_obj, 1929 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 1930 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 1931 } 1932 1933 static void gfx_v11_0_me_fini(struct amdgpu_device *adev) 1934 { 1935 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_obj, 1936 &adev->gfx.me.me_fw_gpu_addr, 1937 (void **)&adev->gfx.me.me_fw_ptr); 1938 1939 amdgpu_bo_free_kernel(&adev->gfx.me.me_fw_data_obj, 1940 &adev->gfx.me.me_fw_data_gpu_addr, 1941 (void **)&adev->gfx.me.me_fw_data_ptr); 1942 } 1943 1944 static void gfx_v11_0_rlc_autoload_buffer_fini(struct amdgpu_device *adev) 1945 { 1946 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo, 1947 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1948 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1949 } 1950 1951 static int gfx_v11_0_sw_fini(struct amdgpu_ip_block *ip_block) 1952 { 1953 int i; 1954 struct amdgpu_device *adev = ip_block->adev; 1955 1956 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 1957 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]); 1958 for (i = 0; i < adev->gfx.num_compute_rings; i++) 1959 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 1960 1961 amdgpu_gfx_mqd_sw_fini(adev, 0); 1962 1963 if (!adev->enable_mes_kiq) { 1964 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[0].ring); 1965 amdgpu_gfx_kiq_fini(adev, 0); 1966 } 1967 1968 amdgpu_gfx_cleaner_shader_sw_fini(adev); 1969 1970 gfx_v11_0_pfp_fini(adev); 1971 gfx_v11_0_me_fini(adev); 1972 gfx_v11_0_rlc_fini(adev); 1973 gfx_v11_0_mec_fini(adev); 1974 1975 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 1976 gfx_v11_0_rlc_autoload_buffer_fini(adev); 1977 1978 gfx_v11_0_free_microcode(adev); 1979 1980 amdgpu_gfx_sysfs_fini(adev); 1981 1982 kfree(adev->gfx.ip_dump_core); 1983 kfree(adev->gfx.ip_dump_compute_queues); 1984 kfree(adev->gfx.ip_dump_gfx_queues); 1985 1986 return 0; 1987 } 1988 1989 static void gfx_v11_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, 1990 u32 sh_num, u32 instance, int xcc_id) 1991 { 1992 u32 data; 1993 1994 if (instance == 0xffffffff) 1995 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, 1996 INSTANCE_BROADCAST_WRITES, 1); 1997 else 1998 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, 1999 instance); 2000 2001 if (se_num == 0xffffffff) 2002 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 2003 1); 2004 else 2005 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 2006 2007 if (sh_num == 0xffffffff) 2008 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES, 2009 1); 2010 else 2011 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num); 2012 2013 WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, data); 2014 } 2015 2016 static u32 gfx_v11_0_get_sa_active_bitmap(struct amdgpu_device *adev) 2017 { 2018 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask; 2019 2020 gc_disabled_sa_mask = RREG32_SOC15(GC, 0, regCC_GC_SA_UNIT_DISABLE); 2021 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask, 2022 CC_GC_SA_UNIT_DISABLE, 2023 SA_DISABLE); 2024 gc_user_disabled_sa_mask = RREG32_SOC15(GC, 0, regGC_USER_SA_UNIT_DISABLE); 2025 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask, 2026 GC_USER_SA_UNIT_DISABLE, 2027 SA_DISABLE); 2028 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * 2029 adev->gfx.config.max_shader_engines); 2030 2031 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask)); 2032 } 2033 2034 static u32 gfx_v11_0_get_rb_active_bitmap(struct amdgpu_device *adev) 2035 { 2036 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask; 2037 u32 rb_mask; 2038 2039 gc_disabled_rb_mask = RREG32_SOC15(GC, 0, regCC_RB_BACKEND_DISABLE); 2040 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask, 2041 CC_RB_BACKEND_DISABLE, 2042 BACKEND_DISABLE); 2043 gc_user_disabled_rb_mask = RREG32_SOC15(GC, 0, regGC_USER_RB_BACKEND_DISABLE); 2044 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask, 2045 GC_USER_RB_BACKEND_DISABLE, 2046 BACKEND_DISABLE); 2047 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se * 2048 adev->gfx.config.max_shader_engines); 2049 2050 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask)); 2051 } 2052 2053 static void gfx_v11_0_setup_rb(struct amdgpu_device *adev) 2054 { 2055 u32 rb_bitmap_per_sa; 2056 u32 rb_bitmap_width_per_sa; 2057 u32 max_sa; 2058 u32 active_sa_bitmap; 2059 u32 global_active_rb_bitmap; 2060 u32 active_rb_bitmap = 0; 2061 u32 i; 2062 2063 /* query sa bitmap from SA_UNIT_DISABLE registers */ 2064 active_sa_bitmap = gfx_v11_0_get_sa_active_bitmap(adev); 2065 /* query rb bitmap from RB_BACKEND_DISABLE registers */ 2066 global_active_rb_bitmap = gfx_v11_0_get_rb_active_bitmap(adev); 2067 2068 /* generate active rb bitmap according to active sa bitmap */ 2069 max_sa = adev->gfx.config.max_shader_engines * 2070 adev->gfx.config.max_sh_per_se; 2071 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se / 2072 adev->gfx.config.max_sh_per_se; 2073 rb_bitmap_per_sa = amdgpu_gfx_create_bitmask(rb_bitmap_width_per_sa); 2074 2075 for (i = 0; i < max_sa; i++) { 2076 if (active_sa_bitmap & (1 << i)) 2077 active_rb_bitmap |= (rb_bitmap_per_sa << (i * rb_bitmap_width_per_sa)); 2078 } 2079 2080 active_rb_bitmap &= global_active_rb_bitmap; 2081 adev->gfx.config.backend_enable_mask = active_rb_bitmap; 2082 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap); 2083 } 2084 2085 #define DEFAULT_SH_MEM_BASES (0x6000) 2086 #define LDS_APP_BASE 0x1 2087 #define SCRATCH_APP_BASE 0x2 2088 2089 static void gfx_v11_0_init_compute_vmid(struct amdgpu_device *adev) 2090 { 2091 int i; 2092 uint32_t sh_mem_bases; 2093 uint32_t data; 2094 2095 /* 2096 * Configure apertures: 2097 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB) 2098 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB) 2099 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB) 2100 */ 2101 sh_mem_bases = (LDS_APP_BASE << SH_MEM_BASES__SHARED_BASE__SHIFT) | 2102 SCRATCH_APP_BASE; 2103 2104 mutex_lock(&adev->srbm_mutex); 2105 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2106 soc21_grbm_select(adev, 0, 0, 0, i); 2107 /* CP and shaders */ 2108 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2109 WREG32_SOC15(GC, 0, regSH_MEM_BASES, sh_mem_bases); 2110 2111 /* Enable trap for each kfd vmid. */ 2112 data = RREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL); 2113 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 2114 WREG32_SOC15(GC, 0, regSPI_GDBG_PER_VMID_CNTL, data); 2115 } 2116 soc21_grbm_select(adev, 0, 0, 0, 0); 2117 mutex_unlock(&adev->srbm_mutex); 2118 2119 /* 2120 * Initialize all compute VMIDs to have no GDS, GWS, or OA 2121 * access. These should be enabled by FW for target VMIDs. 2122 */ 2123 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 2124 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * i, 0); 2125 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * i, 0); 2126 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, i, 0); 2127 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, i, 0); 2128 } 2129 } 2130 2131 static void gfx_v11_0_init_gds_vmid(struct amdgpu_device *adev) 2132 { 2133 int vmid; 2134 2135 /* 2136 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA 2137 * access. Compute VMIDs should be enabled by FW for target VMIDs, 2138 * the driver can enable them for graphics. VMID0 should maintain 2139 * access so that HWS firmware can save/restore entries. 2140 */ 2141 for (vmid = 1; vmid < 16; vmid++) { 2142 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_BASE, 2 * vmid, 0); 2143 WREG32_SOC15_OFFSET(GC, 0, regGDS_VMID0_SIZE, 2 * vmid, 0); 2144 WREG32_SOC15_OFFSET(GC, 0, regGDS_GWS_VMID0, vmid, 0); 2145 WREG32_SOC15_OFFSET(GC, 0, regGDS_OA_VMID0, vmid, 0); 2146 } 2147 } 2148 2149 static void gfx_v11_0_tcp_harvest(struct amdgpu_device *adev) 2150 { 2151 /* TODO: harvest feature to be added later. */ 2152 } 2153 2154 static void gfx_v11_0_get_tcc_info(struct amdgpu_device *adev) 2155 { 2156 /* TCCs are global (not instanced). */ 2157 uint32_t tcc_disable = RREG32_SOC15(GC, 0, regCGTS_TCC_DISABLE) | 2158 RREG32_SOC15(GC, 0, regCGTS_USER_TCC_DISABLE); 2159 2160 adev->gfx.config.tcc_disabled_mask = 2161 REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, TCC_DISABLE) | 2162 (REG_GET_FIELD(tcc_disable, CGTS_TCC_DISABLE, HI_TCC_DISABLE) << 16); 2163 } 2164 2165 static void gfx_v11_0_constants_init(struct amdgpu_device *adev) 2166 { 2167 u32 tmp; 2168 int i; 2169 2170 if (!amdgpu_sriov_vf(adev)) 2171 WREG32_FIELD15_PREREG(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff); 2172 2173 gfx_v11_0_setup_rb(adev); 2174 gfx_v11_0_get_cu_info(adev, &adev->gfx.cu_info); 2175 gfx_v11_0_get_tcc_info(adev); 2176 adev->gfx.config.pa_sc_tile_steering_override = 0; 2177 2178 /* Set whether texture coordinate truncation is conformant. */ 2179 tmp = RREG32_SOC15(GC, 0, regTA_CNTL2); 2180 adev->gfx.config.ta_cntl2_truncate_coord_mode = 2181 REG_GET_FIELD(tmp, TA_CNTL2, TRUNCATE_COORD_MODE); 2182 2183 /* XXX SH_MEM regs */ 2184 /* where to put LDS, scratch, GPUVM in FSA64 space */ 2185 mutex_lock(&adev->srbm_mutex); 2186 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 2187 soc21_grbm_select(adev, 0, 0, 0, i); 2188 /* CP and shaders */ 2189 WREG32_SOC15(GC, 0, regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 2190 if (i != 0) { 2191 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 2192 (adev->gmc.private_aperture_start >> 48)); 2193 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 2194 (adev->gmc.shared_aperture_start >> 48)); 2195 WREG32_SOC15(GC, 0, regSH_MEM_BASES, tmp); 2196 } 2197 } 2198 soc21_grbm_select(adev, 0, 0, 0, 0); 2199 2200 mutex_unlock(&adev->srbm_mutex); 2201 2202 gfx_v11_0_init_compute_vmid(adev); 2203 gfx_v11_0_init_gds_vmid(adev); 2204 } 2205 2206 static u32 gfx_v11_0_get_cpg_int_cntl(struct amdgpu_device *adev, 2207 int me, int pipe) 2208 { 2209 if (me != 0) 2210 return 0; 2211 2212 switch (pipe) { 2213 case 0: 2214 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 2215 case 1: 2216 return SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 2217 default: 2218 return 0; 2219 } 2220 } 2221 2222 static u32 gfx_v11_0_get_cpc_int_cntl(struct amdgpu_device *adev, 2223 int me, int pipe) 2224 { 2225 /* 2226 * amdgpu controls only the first MEC. That's why this function only 2227 * handles the setting of interrupts for this specific MEC. All other 2228 * pipes' interrupts are set by amdkfd. 2229 */ 2230 if (me != 1) 2231 return 0; 2232 2233 switch (pipe) { 2234 case 0: 2235 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 2236 case 1: 2237 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 2238 case 2: 2239 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 2240 case 3: 2241 return SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 2242 default: 2243 return 0; 2244 } 2245 } 2246 2247 static void gfx_v11_0_enable_gui_idle_interrupt(struct amdgpu_device *adev, 2248 bool enable) 2249 { 2250 u32 tmp, cp_int_cntl_reg; 2251 int i, j; 2252 2253 if (amdgpu_sriov_vf(adev)) 2254 return; 2255 2256 for (i = 0; i < adev->gfx.me.num_me; i++) { 2257 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 2258 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 2259 2260 if (cp_int_cntl_reg) { 2261 tmp = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 2262 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 2263 enable ? 1 : 0); 2264 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 2265 enable ? 1 : 0); 2266 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 2267 enable ? 1 : 0); 2268 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 2269 enable ? 1 : 0); 2270 WREG32_SOC15_IP(GC, cp_int_cntl_reg, tmp); 2271 } 2272 } 2273 } 2274 } 2275 2276 static int gfx_v11_0_init_csb(struct amdgpu_device *adev) 2277 { 2278 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 2279 2280 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_HI, 2281 adev->gfx.rlc.clear_state_gpu_addr >> 32); 2282 WREG32_SOC15(GC, 0, regRLC_CSIB_ADDR_LO, 2283 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 2284 WREG32_SOC15(GC, 0, regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); 2285 2286 return 0; 2287 } 2288 2289 static void gfx_v11_0_rlc_stop(struct amdgpu_device *adev) 2290 { 2291 u32 tmp = RREG32_SOC15(GC, 0, regRLC_CNTL); 2292 2293 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); 2294 WREG32_SOC15(GC, 0, regRLC_CNTL, tmp); 2295 } 2296 2297 static void gfx_v11_0_rlc_reset(struct amdgpu_device *adev) 2298 { 2299 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 2300 udelay(50); 2301 WREG32_FIELD15_PREREG(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 2302 udelay(50); 2303 } 2304 2305 static void gfx_v11_0_rlc_smu_handshake_cntl(struct amdgpu_device *adev, 2306 bool enable) 2307 { 2308 uint32_t rlc_pg_cntl; 2309 2310 rlc_pg_cntl = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 2311 2312 if (!enable) { 2313 /* RLC_PG_CNTL[23] = 0 (default) 2314 * RLC will wait for handshake acks with SMU 2315 * GFXOFF will be enabled 2316 * RLC_PG_CNTL[23] = 1 2317 * RLC will not issue any message to SMU 2318 * hence no handshake between SMU & RLC 2319 * GFXOFF will be disabled 2320 */ 2321 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2322 } else 2323 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 2324 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, rlc_pg_cntl); 2325 } 2326 2327 static void gfx_v11_0_rlc_start(struct amdgpu_device *adev) 2328 { 2329 /* TODO: enable rlc & smu handshake until smu 2330 * and gfxoff feature works as expected */ 2331 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK)) 2332 gfx_v11_0_rlc_smu_handshake_cntl(adev, false); 2333 2334 WREG32_FIELD15_PREREG(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1); 2335 udelay(50); 2336 } 2337 2338 static void gfx_v11_0_rlc_enable_srm(struct amdgpu_device *adev) 2339 { 2340 uint32_t tmp; 2341 2342 /* enable Save Restore Machine */ 2343 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL)); 2344 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 2345 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK; 2346 WREG32(SOC15_REG_OFFSET(GC, 0, regRLC_SRM_CNTL), tmp); 2347 } 2348 2349 static void gfx_v11_0_load_rlcg_microcode(struct amdgpu_device *adev) 2350 { 2351 const struct rlc_firmware_header_v2_0 *hdr; 2352 const __le32 *fw_data; 2353 unsigned i, fw_size; 2354 2355 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2356 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2357 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2358 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 2359 2360 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, 2361 RLCG_UCODE_LOADING_START_ADDRESS); 2362 2363 for (i = 0; i < fw_size; i++) 2364 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_DATA, 2365 le32_to_cpup(fw_data++)); 2366 2367 WREG32_SOC15(GC, 0, regRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version); 2368 } 2369 2370 static void gfx_v11_0_load_rlc_iram_dram_microcode(struct amdgpu_device *adev) 2371 { 2372 const struct rlc_firmware_header_v2_2 *hdr; 2373 const __le32 *fw_data; 2374 unsigned i, fw_size; 2375 u32 tmp; 2376 2377 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 2378 2379 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2380 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes)); 2381 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4; 2382 2383 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, 0); 2384 2385 for (i = 0; i < fw_size; i++) { 2386 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2387 msleep(1); 2388 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_DATA, 2389 le32_to_cpup(fw_data++)); 2390 } 2391 2392 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2393 2394 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2395 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes)); 2396 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4; 2397 2398 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_ADDR, 0); 2399 for (i = 0; i < fw_size; i++) { 2400 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2401 msleep(1); 2402 WREG32_SOC15(GC, 0, regRLC_LX6_DRAM_DATA, 2403 le32_to_cpup(fw_data++)); 2404 } 2405 2406 WREG32_SOC15(GC, 0, regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2407 2408 tmp = RREG32_SOC15(GC, 0, regRLC_LX6_CNTL); 2409 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1); 2410 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0); 2411 WREG32_SOC15(GC, 0, regRLC_LX6_CNTL, tmp); 2412 } 2413 2414 static void gfx_v11_0_load_rlcp_rlcv_microcode(struct amdgpu_device *adev) 2415 { 2416 const struct rlc_firmware_header_v2_3 *hdr; 2417 const __le32 *fw_data; 2418 unsigned i, fw_size; 2419 u32 tmp; 2420 2421 hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data; 2422 2423 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2424 le32_to_cpu(hdr->rlcp_ucode_offset_bytes)); 2425 fw_size = le32_to_cpu(hdr->rlcp_ucode_size_bytes) / 4; 2426 2427 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, 0); 2428 2429 for (i = 0; i < fw_size; i++) { 2430 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2431 msleep(1); 2432 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_DATA, 2433 le32_to_cpup(fw_data++)); 2434 } 2435 2436 WREG32_SOC15(GC, 0, regRLC_PACE_UCODE_ADDR, adev->gfx.rlc_fw_version); 2437 2438 tmp = RREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE); 2439 tmp = REG_SET_FIELD(tmp, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1); 2440 WREG32_SOC15(GC, 0, regRLC_GPM_THREAD_ENABLE, tmp); 2441 2442 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 2443 le32_to_cpu(hdr->rlcv_ucode_offset_bytes)); 2444 fw_size = le32_to_cpu(hdr->rlcv_ucode_size_bytes) / 4; 2445 2446 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, 0); 2447 2448 for (i = 0; i < fw_size; i++) { 2449 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2450 msleep(1); 2451 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_DATA, 2452 le32_to_cpup(fw_data++)); 2453 } 2454 2455 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_UCODE_ADDR, adev->gfx.rlc_fw_version); 2456 2457 tmp = RREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL); 2458 tmp = REG_SET_FIELD(tmp, RLC_GPU_IOV_F32_CNTL, ENABLE, 1); 2459 WREG32_SOC15(GC, 0, regRLC_GPU_IOV_F32_CNTL, tmp); 2460 } 2461 2462 static int gfx_v11_0_rlc_load_microcode(struct amdgpu_device *adev) 2463 { 2464 const struct rlc_firmware_header_v2_0 *hdr; 2465 uint16_t version_major; 2466 uint16_t version_minor; 2467 2468 if (!adev->gfx.rlc_fw) 2469 return -EINVAL; 2470 2471 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2472 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2473 2474 version_major = le16_to_cpu(hdr->header.header_version_major); 2475 version_minor = le16_to_cpu(hdr->header.header_version_minor); 2476 2477 if (version_major == 2) { 2478 gfx_v11_0_load_rlcg_microcode(adev); 2479 if (amdgpu_dpm == 1) { 2480 if (version_minor >= 2) 2481 gfx_v11_0_load_rlc_iram_dram_microcode(adev); 2482 if (version_minor == 3) 2483 gfx_v11_0_load_rlcp_rlcv_microcode(adev); 2484 } 2485 2486 return 0; 2487 } 2488 2489 return -EINVAL; 2490 } 2491 2492 static int gfx_v11_0_rlc_resume(struct amdgpu_device *adev) 2493 { 2494 int r; 2495 2496 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 2497 gfx_v11_0_init_csb(adev); 2498 2499 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */ 2500 gfx_v11_0_rlc_enable_srm(adev); 2501 } else { 2502 if (amdgpu_sriov_vf(adev)) { 2503 gfx_v11_0_init_csb(adev); 2504 return 0; 2505 } 2506 2507 adev->gfx.rlc.funcs->stop(adev); 2508 2509 /* disable CG */ 2510 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, 0); 2511 2512 /* disable PG */ 2513 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, 0); 2514 2515 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 2516 /* legacy rlc firmware loading */ 2517 r = gfx_v11_0_rlc_load_microcode(adev); 2518 if (r) 2519 return r; 2520 } 2521 2522 gfx_v11_0_init_csb(adev); 2523 2524 adev->gfx.rlc.funcs->start(adev); 2525 } 2526 return 0; 2527 } 2528 2529 static int gfx_v11_0_config_me_cache(struct amdgpu_device *adev, uint64_t addr) 2530 { 2531 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2532 uint32_t tmp; 2533 int i; 2534 2535 /* Trigger an invalidation of the L1 instruction caches */ 2536 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2537 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2538 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2539 2540 /* Wait for invalidation complete */ 2541 for (i = 0; i < usec_timeout; i++) { 2542 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2543 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2544 INVALIDATE_CACHE_COMPLETE)) 2545 break; 2546 udelay(1); 2547 } 2548 2549 if (i >= usec_timeout) { 2550 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2551 return -EINVAL; 2552 } 2553 2554 if (amdgpu_emu_mode == 1) 2555 amdgpu_device_flush_hdp(adev, NULL); 2556 2557 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2558 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2559 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2560 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2561 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2562 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2563 2564 /* Program me ucode address into intruction cache address register */ 2565 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2566 lower_32_bits(addr) & 0xFFFFF000); 2567 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2568 upper_32_bits(addr)); 2569 2570 return 0; 2571 } 2572 2573 static int gfx_v11_0_config_pfp_cache(struct amdgpu_device *adev, uint64_t addr) 2574 { 2575 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2576 uint32_t tmp; 2577 int i; 2578 2579 /* Trigger an invalidation of the L1 instruction caches */ 2580 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2581 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2582 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2583 2584 /* Wait for invalidation complete */ 2585 for (i = 0; i < usec_timeout; i++) { 2586 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2587 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2588 INVALIDATE_CACHE_COMPLETE)) 2589 break; 2590 udelay(1); 2591 } 2592 2593 if (i >= usec_timeout) { 2594 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2595 return -EINVAL; 2596 } 2597 2598 if (amdgpu_emu_mode == 1) 2599 amdgpu_device_flush_hdp(adev, NULL); 2600 2601 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2602 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2603 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2604 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2605 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2606 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2607 2608 /* Program pfp ucode address into intruction cache address register */ 2609 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2610 lower_32_bits(addr) & 0xFFFFF000); 2611 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2612 upper_32_bits(addr)); 2613 2614 return 0; 2615 } 2616 2617 static int gfx_v11_0_config_mec_cache(struct amdgpu_device *adev, uint64_t addr) 2618 { 2619 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2620 uint32_t tmp; 2621 int i; 2622 2623 /* Trigger an invalidation of the L1 instruction caches */ 2624 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2625 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2626 2627 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2628 2629 /* Wait for invalidation complete */ 2630 for (i = 0; i < usec_timeout; i++) { 2631 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2632 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2633 INVALIDATE_CACHE_COMPLETE)) 2634 break; 2635 udelay(1); 2636 } 2637 2638 if (i >= usec_timeout) { 2639 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2640 return -EINVAL; 2641 } 2642 2643 if (amdgpu_emu_mode == 1) 2644 amdgpu_device_flush_hdp(adev, NULL); 2645 2646 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2647 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2648 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2649 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, ADDRESS_CLAMP, 1); 2650 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2651 2652 /* Program mec1 ucode address into intruction cache address register */ 2653 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, 2654 lower_32_bits(addr) & 0xFFFFF000); 2655 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2656 upper_32_bits(addr)); 2657 2658 return 0; 2659 } 2660 2661 static int gfx_v11_0_config_pfp_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2662 { 2663 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2664 uint32_t tmp; 2665 unsigned i, pipe_id; 2666 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2667 2668 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 2669 adev->gfx.pfp_fw->data; 2670 2671 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 2672 lower_32_bits(addr)); 2673 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 2674 upper_32_bits(addr)); 2675 2676 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 2677 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 2678 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 2679 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 2680 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 2681 2682 /* 2683 * Programming any of the CP_PFP_IC_BASE registers 2684 * forces invalidation of the ME L1 I$. Wait for the 2685 * invalidation complete 2686 */ 2687 for (i = 0; i < usec_timeout; i++) { 2688 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2689 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2690 INVALIDATE_CACHE_COMPLETE)) 2691 break; 2692 udelay(1); 2693 } 2694 2695 if (i >= usec_timeout) { 2696 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2697 return -EINVAL; 2698 } 2699 2700 /* Prime the L1 instruction caches */ 2701 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2702 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 2703 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 2704 /* Waiting for cache primed*/ 2705 for (i = 0; i < usec_timeout; i++) { 2706 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 2707 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 2708 ICACHE_PRIMED)) 2709 break; 2710 udelay(1); 2711 } 2712 2713 if (i >= usec_timeout) { 2714 dev_err(adev->dev, "failed to prime instruction cache\n"); 2715 return -EINVAL; 2716 } 2717 2718 mutex_lock(&adev->srbm_mutex); 2719 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2720 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2721 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 2722 (pfp_hdr->ucode_start_addr_hi << 30) | 2723 (pfp_hdr->ucode_start_addr_lo >> 2)); 2724 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 2725 pfp_hdr->ucode_start_addr_hi >> 2); 2726 2727 /* 2728 * Program CP_ME_CNTL to reset given PIPE to take 2729 * effect of CP_PFP_PRGRM_CNTR_START. 2730 */ 2731 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2732 if (pipe_id == 0) 2733 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2734 PFP_PIPE0_RESET, 1); 2735 else 2736 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2737 PFP_PIPE1_RESET, 1); 2738 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2739 2740 /* Clear pfp pipe0 reset bit. */ 2741 if (pipe_id == 0) 2742 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2743 PFP_PIPE0_RESET, 0); 2744 else 2745 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2746 PFP_PIPE1_RESET, 0); 2747 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2748 2749 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 2750 lower_32_bits(addr2)); 2751 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 2752 upper_32_bits(addr2)); 2753 } 2754 soc21_grbm_select(adev, 0, 0, 0, 0); 2755 mutex_unlock(&adev->srbm_mutex); 2756 2757 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2758 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2759 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2760 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2761 2762 /* Invalidate the data caches */ 2763 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2764 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2765 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2766 2767 for (i = 0; i < usec_timeout; i++) { 2768 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2769 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2770 INVALIDATE_DCACHE_COMPLETE)) 2771 break; 2772 udelay(1); 2773 } 2774 2775 if (i >= usec_timeout) { 2776 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2777 return -EINVAL; 2778 } 2779 2780 return 0; 2781 } 2782 2783 static int gfx_v11_0_config_me_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2784 { 2785 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2786 uint32_t tmp; 2787 unsigned i, pipe_id; 2788 const struct gfx_firmware_header_v2_0 *me_hdr; 2789 2790 me_hdr = (const struct gfx_firmware_header_v2_0 *) 2791 adev->gfx.me_fw->data; 2792 2793 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 2794 lower_32_bits(addr)); 2795 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 2796 upper_32_bits(addr)); 2797 2798 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 2799 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 2800 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 2801 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 2802 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 2803 2804 /* 2805 * Programming any of the CP_ME_IC_BASE registers 2806 * forces invalidation of the ME L1 I$. Wait for the 2807 * invalidation complete 2808 */ 2809 for (i = 0; i < usec_timeout; i++) { 2810 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2811 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2812 INVALIDATE_CACHE_COMPLETE)) 2813 break; 2814 udelay(1); 2815 } 2816 2817 if (i >= usec_timeout) { 2818 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2819 return -EINVAL; 2820 } 2821 2822 /* Prime the instruction caches */ 2823 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2824 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 2825 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 2826 2827 /* Waiting for instruction cache primed*/ 2828 for (i = 0; i < usec_timeout; i++) { 2829 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 2830 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 2831 ICACHE_PRIMED)) 2832 break; 2833 udelay(1); 2834 } 2835 2836 if (i >= usec_timeout) { 2837 dev_err(adev->dev, "failed to prime instruction cache\n"); 2838 return -EINVAL; 2839 } 2840 2841 mutex_lock(&adev->srbm_mutex); 2842 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 2843 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 2844 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 2845 (me_hdr->ucode_start_addr_hi << 30) | 2846 (me_hdr->ucode_start_addr_lo >> 2) ); 2847 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 2848 me_hdr->ucode_start_addr_hi>>2); 2849 2850 /* 2851 * Program CP_ME_CNTL to reset given PIPE to take 2852 * effect of CP_PFP_PRGRM_CNTR_START. 2853 */ 2854 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 2855 if (pipe_id == 0) 2856 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2857 ME_PIPE0_RESET, 1); 2858 else 2859 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2860 ME_PIPE1_RESET, 1); 2861 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2862 2863 /* Clear pfp pipe0 reset bit. */ 2864 if (pipe_id == 0) 2865 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2866 ME_PIPE0_RESET, 0); 2867 else 2868 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 2869 ME_PIPE1_RESET, 0); 2870 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 2871 2872 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 2873 lower_32_bits(addr2)); 2874 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 2875 upper_32_bits(addr2)); 2876 } 2877 soc21_grbm_select(adev, 0, 0, 0, 0); 2878 mutex_unlock(&adev->srbm_mutex); 2879 2880 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 2881 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 2882 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 2883 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 2884 2885 /* Invalidate the data caches */ 2886 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2887 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2888 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 2889 2890 for (i = 0; i < usec_timeout; i++) { 2891 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 2892 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 2893 INVALIDATE_DCACHE_COMPLETE)) 2894 break; 2895 udelay(1); 2896 } 2897 2898 if (i >= usec_timeout) { 2899 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 2900 return -EINVAL; 2901 } 2902 2903 return 0; 2904 } 2905 2906 static int gfx_v11_0_config_mec_cache_rs64(struct amdgpu_device *adev, uint64_t addr, uint64_t addr2) 2907 { 2908 uint32_t usec_timeout = 50000; /* wait for 50ms */ 2909 uint32_t tmp; 2910 unsigned i; 2911 const struct gfx_firmware_header_v2_0 *mec_hdr; 2912 2913 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2914 adev->gfx.mec_fw->data; 2915 2916 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 2917 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 2918 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2919 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2920 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 2921 2922 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 2923 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 2924 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 2925 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 2926 2927 mutex_lock(&adev->srbm_mutex); 2928 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 2929 soc21_grbm_select(adev, 1, i, 0, 0); 2930 2931 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, addr2); 2932 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 2933 upper_32_bits(addr2)); 2934 2935 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 2936 mec_hdr->ucode_start_addr_lo >> 2 | 2937 mec_hdr->ucode_start_addr_hi << 30); 2938 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2939 mec_hdr->ucode_start_addr_hi >> 2); 2940 2941 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, addr); 2942 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 2943 upper_32_bits(addr)); 2944 } 2945 mutex_unlock(&adev->srbm_mutex); 2946 soc21_grbm_select(adev, 0, 0, 0, 0); 2947 2948 /* Trigger an invalidation of the L1 instruction caches */ 2949 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2950 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2951 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 2952 2953 /* Wait for invalidation complete */ 2954 for (i = 0; i < usec_timeout; i++) { 2955 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 2956 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 2957 INVALIDATE_DCACHE_COMPLETE)) 2958 break; 2959 udelay(1); 2960 } 2961 2962 if (i >= usec_timeout) { 2963 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2964 return -EINVAL; 2965 } 2966 2967 /* Trigger an invalidation of the L1 instruction caches */ 2968 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2969 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2970 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 2971 2972 /* Wait for invalidation complete */ 2973 for (i = 0; i < usec_timeout; i++) { 2974 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 2975 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2976 INVALIDATE_CACHE_COMPLETE)) 2977 break; 2978 udelay(1); 2979 } 2980 2981 if (i >= usec_timeout) { 2982 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2983 return -EINVAL; 2984 } 2985 2986 return 0; 2987 } 2988 2989 static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev) 2990 { 2991 const struct gfx_firmware_header_v2_0 *pfp_hdr; 2992 const struct gfx_firmware_header_v2_0 *me_hdr; 2993 const struct gfx_firmware_header_v2_0 *mec_hdr; 2994 uint32_t pipe_id, tmp; 2995 2996 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2997 adev->gfx.mec_fw->data; 2998 me_hdr = (const struct gfx_firmware_header_v2_0 *) 2999 adev->gfx.me_fw->data; 3000 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3001 adev->gfx.pfp_fw->data; 3002 3003 /* config pfp program start addr */ 3004 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3005 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3006 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3007 (pfp_hdr->ucode_start_addr_hi << 30) | 3008 (pfp_hdr->ucode_start_addr_lo >> 2)); 3009 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3010 pfp_hdr->ucode_start_addr_hi >> 2); 3011 } 3012 soc21_grbm_select(adev, 0, 0, 0, 0); 3013 3014 /* reset pfp pipe */ 3015 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3016 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 1); 3017 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 1); 3018 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3019 3020 /* clear pfp pipe reset */ 3021 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE0_RESET, 0); 3022 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_PIPE1_RESET, 0); 3023 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3024 3025 /* config me program start addr */ 3026 for (pipe_id = 0; pipe_id < 2; pipe_id++) { 3027 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3028 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3029 (me_hdr->ucode_start_addr_hi << 30) | 3030 (me_hdr->ucode_start_addr_lo >> 2) ); 3031 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3032 me_hdr->ucode_start_addr_hi>>2); 3033 } 3034 soc21_grbm_select(adev, 0, 0, 0, 0); 3035 3036 /* reset me pipe */ 3037 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3038 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 1); 3039 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 1); 3040 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3041 3042 /* clear me pipe reset */ 3043 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE0_RESET, 0); 3044 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_PIPE1_RESET, 0); 3045 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3046 3047 /* config mec program start addr */ 3048 for (pipe_id = 0; pipe_id < 4; pipe_id++) { 3049 soc21_grbm_select(adev, 1, pipe_id, 0, 0); 3050 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 3051 mec_hdr->ucode_start_addr_lo >> 2 | 3052 mec_hdr->ucode_start_addr_hi << 30); 3053 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 3054 mec_hdr->ucode_start_addr_hi >> 2); 3055 } 3056 soc21_grbm_select(adev, 0, 0, 0, 0); 3057 3058 /* reset mec pipe */ 3059 tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3060 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1); 3061 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1); 3062 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1); 3063 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1); 3064 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3065 3066 /* clear mec pipe reset */ 3067 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0); 3068 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0); 3069 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0); 3070 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0); 3071 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp); 3072 } 3073 3074 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) 3075 { 3076 uint32_t cp_status; 3077 uint32_t bootload_status; 3078 int i, r; 3079 uint64_t addr, addr2; 3080 3081 for (i = 0; i < adev->usec_timeout; i++) { 3082 cp_status = RREG32_SOC15(GC, 0, regCP_STAT); 3083 3084 if (amdgpu_ip_version(adev, GC_HWIP, 0) == 3085 IP_VERSION(11, 0, 1) || 3086 amdgpu_ip_version(adev, GC_HWIP, 0) == 3087 IP_VERSION(11, 0, 4) || 3088 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 0) || 3089 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 1) || 3090 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 2) || 3091 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 3) || 3092 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4) || 3093 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 6)) 3094 bootload_status = RREG32_SOC15(GC, 0, 3095 regRLC_RLCS_BOOTLOAD_STATUS_gc_11_0_1); 3096 else 3097 bootload_status = RREG32_SOC15(GC, 0, regRLC_RLCS_BOOTLOAD_STATUS); 3098 3099 if ((cp_status == 0) && 3100 (REG_GET_FIELD(bootload_status, 3101 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) { 3102 break; 3103 } 3104 udelay(1); 3105 } 3106 3107 if (i >= adev->usec_timeout) { 3108 dev_err(adev->dev, "rlc autoload: gc ucode autoload timeout\n"); 3109 return -ETIMEDOUT; 3110 } 3111 3112 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 3113 if (adev->gfx.rs64_enable) { 3114 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3115 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME].offset; 3116 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3117 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_ME_P0_STACK].offset; 3118 r = gfx_v11_0_config_me_cache_rs64(adev, addr, addr2); 3119 if (r) 3120 return r; 3121 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3122 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP].offset; 3123 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3124 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_PFP_P0_STACK].offset; 3125 r = gfx_v11_0_config_pfp_cache_rs64(adev, addr, addr2); 3126 if (r) 3127 return r; 3128 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3129 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC].offset; 3130 addr2 = adev->gfx.rlc.rlc_autoload_gpu_addr + 3131 rlc_autoload_info[SOC21_FIRMWARE_ID_RS64_MEC_P0_STACK].offset; 3132 r = gfx_v11_0_config_mec_cache_rs64(adev, addr, addr2); 3133 if (r) 3134 return r; 3135 } else { 3136 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3137 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_ME].offset; 3138 r = gfx_v11_0_config_me_cache(adev, addr); 3139 if (r) 3140 return r; 3141 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3142 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_PFP].offset; 3143 r = gfx_v11_0_config_pfp_cache(adev, addr); 3144 if (r) 3145 return r; 3146 addr = adev->gfx.rlc.rlc_autoload_gpu_addr + 3147 rlc_autoload_info[SOC21_FIRMWARE_ID_CP_MEC].offset; 3148 r = gfx_v11_0_config_mec_cache(adev, addr); 3149 if (r) 3150 return r; 3151 } 3152 } 3153 3154 return 0; 3155 } 3156 3157 static int gfx_v11_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable) 3158 { 3159 int i; 3160 u32 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3161 3162 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1); 3163 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1); 3164 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3165 3166 for (i = 0; i < adev->usec_timeout; i++) { 3167 if (RREG32_SOC15(GC, 0, regCP_STAT) == 0) 3168 break; 3169 udelay(1); 3170 } 3171 3172 if (i >= adev->usec_timeout) 3173 DRM_ERROR("failed to %s cp gfx\n", enable ? "unhalt" : "halt"); 3174 3175 return 0; 3176 } 3177 3178 static int gfx_v11_0_cp_gfx_load_pfp_microcode(struct amdgpu_device *adev) 3179 { 3180 int r; 3181 const struct gfx_firmware_header_v1_0 *pfp_hdr; 3182 const __le32 *fw_data; 3183 unsigned i, fw_size; 3184 3185 pfp_hdr = (const struct gfx_firmware_header_v1_0 *) 3186 adev->gfx.pfp_fw->data; 3187 3188 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3189 3190 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3191 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes)); 3192 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes); 3193 3194 r = amdgpu_bo_create_reserved(adev, pfp_hdr->header.ucode_size_bytes, 3195 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3196 &adev->gfx.pfp.pfp_fw_obj, 3197 &adev->gfx.pfp.pfp_fw_gpu_addr, 3198 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3199 if (r) { 3200 dev_err(adev->dev, "(%d) failed to create pfp fw bo\n", r); 3201 gfx_v11_0_pfp_fini(adev); 3202 return r; 3203 } 3204 3205 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_data, fw_size); 3206 3207 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3208 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3209 3210 gfx_v11_0_config_pfp_cache(adev, adev->gfx.pfp.pfp_fw_gpu_addr); 3211 3212 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, 0); 3213 3214 for (i = 0; i < pfp_hdr->jt_size; i++) 3215 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_DATA, 3216 le32_to_cpup(fw_data + pfp_hdr->jt_offset + i)); 3217 3218 WREG32_SOC15(GC, 0, regCP_HYP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version); 3219 3220 return 0; 3221 } 3222 3223 static int gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(struct amdgpu_device *adev) 3224 { 3225 int r; 3226 const struct gfx_firmware_header_v2_0 *pfp_hdr; 3227 const __le32 *fw_ucode, *fw_data; 3228 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3229 uint32_t tmp; 3230 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3231 3232 pfp_hdr = (const struct gfx_firmware_header_v2_0 *) 3233 adev->gfx.pfp_fw->data; 3234 3235 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header); 3236 3237 /* instruction */ 3238 fw_ucode = (const __le32 *)(adev->gfx.pfp_fw->data + 3239 le32_to_cpu(pfp_hdr->ucode_offset_bytes)); 3240 fw_ucode_size = le32_to_cpu(pfp_hdr->ucode_size_bytes); 3241 /* data */ 3242 fw_data = (const __le32 *)(adev->gfx.pfp_fw->data + 3243 le32_to_cpu(pfp_hdr->data_offset_bytes)); 3244 fw_data_size = le32_to_cpu(pfp_hdr->data_size_bytes); 3245 3246 /* 64kb align */ 3247 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3248 64 * 1024, 3249 AMDGPU_GEM_DOMAIN_VRAM | 3250 AMDGPU_GEM_DOMAIN_GTT, 3251 &adev->gfx.pfp.pfp_fw_obj, 3252 &adev->gfx.pfp.pfp_fw_gpu_addr, 3253 (void **)&adev->gfx.pfp.pfp_fw_ptr); 3254 if (r) { 3255 dev_err(adev->dev, "(%d) failed to create pfp ucode fw bo\n", r); 3256 gfx_v11_0_pfp_fini(adev); 3257 return r; 3258 } 3259 3260 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3261 64 * 1024, 3262 AMDGPU_GEM_DOMAIN_VRAM | 3263 AMDGPU_GEM_DOMAIN_GTT, 3264 &adev->gfx.pfp.pfp_fw_data_obj, 3265 &adev->gfx.pfp.pfp_fw_data_gpu_addr, 3266 (void **)&adev->gfx.pfp.pfp_fw_data_ptr); 3267 if (r) { 3268 dev_err(adev->dev, "(%d) failed to create pfp data fw bo\n", r); 3269 gfx_v11_0_pfp_fini(adev); 3270 return r; 3271 } 3272 3273 memcpy(adev->gfx.pfp.pfp_fw_ptr, fw_ucode, fw_ucode_size); 3274 memcpy(adev->gfx.pfp.pfp_fw_data_ptr, fw_data, fw_data_size); 3275 3276 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_obj); 3277 amdgpu_bo_kunmap(adev->gfx.pfp.pfp_fw_data_obj); 3278 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_obj); 3279 amdgpu_bo_unreserve(adev->gfx.pfp.pfp_fw_data_obj); 3280 3281 if (amdgpu_emu_mode == 1) 3282 amdgpu_device_flush_hdp(adev, NULL); 3283 3284 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_LO, 3285 lower_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3286 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_HI, 3287 upper_32_bits(adev->gfx.pfp.pfp_fw_gpu_addr)); 3288 3289 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL); 3290 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, VMID, 0); 3291 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, CACHE_POLICY, 0); 3292 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_BASE_CNTL, EXE_DISABLE, 0); 3293 WREG32_SOC15(GC, 0, regCP_PFP_IC_BASE_CNTL, tmp); 3294 3295 /* 3296 * Programming any of the CP_PFP_IC_BASE registers 3297 * forces invalidation of the ME L1 I$. Wait for the 3298 * invalidation complete 3299 */ 3300 for (i = 0; i < usec_timeout; i++) { 3301 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3302 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3303 INVALIDATE_CACHE_COMPLETE)) 3304 break; 3305 udelay(1); 3306 } 3307 3308 if (i >= usec_timeout) { 3309 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3310 return -EINVAL; 3311 } 3312 3313 /* Prime the L1 instruction caches */ 3314 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3315 tmp = REG_SET_FIELD(tmp, CP_PFP_IC_OP_CNTL, PRIME_ICACHE, 1); 3316 WREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL, tmp); 3317 /* Waiting for cache primed*/ 3318 for (i = 0; i < usec_timeout; i++) { 3319 tmp = RREG32_SOC15(GC, 0, regCP_PFP_IC_OP_CNTL); 3320 if (1 == REG_GET_FIELD(tmp, CP_PFP_IC_OP_CNTL, 3321 ICACHE_PRIMED)) 3322 break; 3323 udelay(1); 3324 } 3325 3326 if (i >= usec_timeout) { 3327 dev_err(adev->dev, "failed to prime instruction cache\n"); 3328 return -EINVAL; 3329 } 3330 3331 mutex_lock(&adev->srbm_mutex); 3332 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3333 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3334 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START, 3335 (pfp_hdr->ucode_start_addr_hi << 30) | 3336 (pfp_hdr->ucode_start_addr_lo >> 2) ); 3337 WREG32_SOC15(GC, 0, regCP_PFP_PRGRM_CNTR_START_HI, 3338 pfp_hdr->ucode_start_addr_hi>>2); 3339 3340 /* 3341 * Program CP_ME_CNTL to reset given PIPE to take 3342 * effect of CP_PFP_PRGRM_CNTR_START. 3343 */ 3344 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3345 if (pipe_id == 0) 3346 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3347 PFP_PIPE0_RESET, 1); 3348 else 3349 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3350 PFP_PIPE1_RESET, 1); 3351 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3352 3353 /* Clear pfp pipe0 reset bit. */ 3354 if (pipe_id == 0) 3355 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3356 PFP_PIPE0_RESET, 0); 3357 else 3358 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3359 PFP_PIPE1_RESET, 0); 3360 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3361 3362 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_LO, 3363 lower_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3364 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE0_HI, 3365 upper_32_bits(adev->gfx.pfp.pfp_fw_data_gpu_addr)); 3366 } 3367 soc21_grbm_select(adev, 0, 0, 0, 0); 3368 mutex_unlock(&adev->srbm_mutex); 3369 3370 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3371 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3372 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3373 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3374 3375 /* Invalidate the data caches */ 3376 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3377 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3378 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3379 3380 for (i = 0; i < usec_timeout; i++) { 3381 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3382 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3383 INVALIDATE_DCACHE_COMPLETE)) 3384 break; 3385 udelay(1); 3386 } 3387 3388 if (i >= usec_timeout) { 3389 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3390 return -EINVAL; 3391 } 3392 3393 return 0; 3394 } 3395 3396 static int gfx_v11_0_cp_gfx_load_me_microcode(struct amdgpu_device *adev) 3397 { 3398 int r; 3399 const struct gfx_firmware_header_v1_0 *me_hdr; 3400 const __le32 *fw_data; 3401 unsigned i, fw_size; 3402 3403 me_hdr = (const struct gfx_firmware_header_v1_0 *) 3404 adev->gfx.me_fw->data; 3405 3406 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3407 3408 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3409 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes)); 3410 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes); 3411 3412 r = amdgpu_bo_create_reserved(adev, me_hdr->header.ucode_size_bytes, 3413 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3414 &adev->gfx.me.me_fw_obj, 3415 &adev->gfx.me.me_fw_gpu_addr, 3416 (void **)&adev->gfx.me.me_fw_ptr); 3417 if (r) { 3418 dev_err(adev->dev, "(%d) failed to create me fw bo\n", r); 3419 gfx_v11_0_me_fini(adev); 3420 return r; 3421 } 3422 3423 memcpy(adev->gfx.me.me_fw_ptr, fw_data, fw_size); 3424 3425 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3426 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3427 3428 gfx_v11_0_config_me_cache(adev, adev->gfx.me.me_fw_gpu_addr); 3429 3430 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, 0); 3431 3432 for (i = 0; i < me_hdr->jt_size; i++) 3433 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_DATA, 3434 le32_to_cpup(fw_data + me_hdr->jt_offset + i)); 3435 3436 WREG32_SOC15(GC, 0, regCP_HYP_ME_UCODE_ADDR, adev->gfx.me_fw_version); 3437 3438 return 0; 3439 } 3440 3441 static int gfx_v11_0_cp_gfx_load_me_microcode_rs64(struct amdgpu_device *adev) 3442 { 3443 int r; 3444 const struct gfx_firmware_header_v2_0 *me_hdr; 3445 const __le32 *fw_ucode, *fw_data; 3446 unsigned i, pipe_id, fw_ucode_size, fw_data_size; 3447 uint32_t tmp; 3448 uint32_t usec_timeout = 50000; /* wait for 50ms */ 3449 3450 me_hdr = (const struct gfx_firmware_header_v2_0 *) 3451 adev->gfx.me_fw->data; 3452 3453 amdgpu_ucode_print_gfx_hdr(&me_hdr->header); 3454 3455 /* instruction */ 3456 fw_ucode = (const __le32 *)(adev->gfx.me_fw->data + 3457 le32_to_cpu(me_hdr->ucode_offset_bytes)); 3458 fw_ucode_size = le32_to_cpu(me_hdr->ucode_size_bytes); 3459 /* data */ 3460 fw_data = (const __le32 *)(adev->gfx.me_fw->data + 3461 le32_to_cpu(me_hdr->data_offset_bytes)); 3462 fw_data_size = le32_to_cpu(me_hdr->data_size_bytes); 3463 3464 /* 64kb align*/ 3465 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3466 64 * 1024, 3467 AMDGPU_GEM_DOMAIN_VRAM | 3468 AMDGPU_GEM_DOMAIN_GTT, 3469 &adev->gfx.me.me_fw_obj, 3470 &adev->gfx.me.me_fw_gpu_addr, 3471 (void **)&adev->gfx.me.me_fw_ptr); 3472 if (r) { 3473 dev_err(adev->dev, "(%d) failed to create me ucode bo\n", r); 3474 gfx_v11_0_me_fini(adev); 3475 return r; 3476 } 3477 3478 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3479 64 * 1024, 3480 AMDGPU_GEM_DOMAIN_VRAM | 3481 AMDGPU_GEM_DOMAIN_GTT, 3482 &adev->gfx.me.me_fw_data_obj, 3483 &adev->gfx.me.me_fw_data_gpu_addr, 3484 (void **)&adev->gfx.me.me_fw_data_ptr); 3485 if (r) { 3486 dev_err(adev->dev, "(%d) failed to create me data bo\n", r); 3487 gfx_v11_0_pfp_fini(adev); 3488 return r; 3489 } 3490 3491 memcpy(adev->gfx.me.me_fw_ptr, fw_ucode, fw_ucode_size); 3492 memcpy(adev->gfx.me.me_fw_data_ptr, fw_data, fw_data_size); 3493 3494 amdgpu_bo_kunmap(adev->gfx.me.me_fw_obj); 3495 amdgpu_bo_kunmap(adev->gfx.me.me_fw_data_obj); 3496 amdgpu_bo_unreserve(adev->gfx.me.me_fw_obj); 3497 amdgpu_bo_unreserve(adev->gfx.me.me_fw_data_obj); 3498 3499 if (amdgpu_emu_mode == 1) 3500 amdgpu_device_flush_hdp(adev, NULL); 3501 3502 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_LO, 3503 lower_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3504 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_HI, 3505 upper_32_bits(adev->gfx.me.me_fw_gpu_addr)); 3506 3507 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL); 3508 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, VMID, 0); 3509 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, CACHE_POLICY, 0); 3510 tmp = REG_SET_FIELD(tmp, CP_ME_IC_BASE_CNTL, EXE_DISABLE, 0); 3511 WREG32_SOC15(GC, 0, regCP_ME_IC_BASE_CNTL, tmp); 3512 3513 /* 3514 * Programming any of the CP_ME_IC_BASE registers 3515 * forces invalidation of the ME L1 I$. Wait for the 3516 * invalidation complete 3517 */ 3518 for (i = 0; i < usec_timeout; i++) { 3519 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3520 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3521 INVALIDATE_CACHE_COMPLETE)) 3522 break; 3523 udelay(1); 3524 } 3525 3526 if (i >= usec_timeout) { 3527 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 3528 return -EINVAL; 3529 } 3530 3531 /* Prime the instruction caches */ 3532 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3533 tmp = REG_SET_FIELD(tmp, CP_ME_IC_OP_CNTL, PRIME_ICACHE, 1); 3534 WREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL, tmp); 3535 3536 /* Waiting for instruction cache primed*/ 3537 for (i = 0; i < usec_timeout; i++) { 3538 tmp = RREG32_SOC15(GC, 0, regCP_ME_IC_OP_CNTL); 3539 if (1 == REG_GET_FIELD(tmp, CP_ME_IC_OP_CNTL, 3540 ICACHE_PRIMED)) 3541 break; 3542 udelay(1); 3543 } 3544 3545 if (i >= usec_timeout) { 3546 dev_err(adev->dev, "failed to prime instruction cache\n"); 3547 return -EINVAL; 3548 } 3549 3550 mutex_lock(&adev->srbm_mutex); 3551 for (pipe_id = 0; pipe_id < adev->gfx.me.num_pipe_per_me; pipe_id++) { 3552 soc21_grbm_select(adev, 0, pipe_id, 0, 0); 3553 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START, 3554 (me_hdr->ucode_start_addr_hi << 30) | 3555 (me_hdr->ucode_start_addr_lo >> 2) ); 3556 WREG32_SOC15(GC, 0, regCP_ME_PRGRM_CNTR_START_HI, 3557 me_hdr->ucode_start_addr_hi>>2); 3558 3559 /* 3560 * Program CP_ME_CNTL to reset given PIPE to take 3561 * effect of CP_PFP_PRGRM_CNTR_START. 3562 */ 3563 tmp = RREG32_SOC15(GC, 0, regCP_ME_CNTL); 3564 if (pipe_id == 0) 3565 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3566 ME_PIPE0_RESET, 1); 3567 else 3568 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3569 ME_PIPE1_RESET, 1); 3570 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3571 3572 /* Clear pfp pipe0 reset bit. */ 3573 if (pipe_id == 0) 3574 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3575 ME_PIPE0_RESET, 0); 3576 else 3577 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, 3578 ME_PIPE1_RESET, 0); 3579 WREG32_SOC15(GC, 0, regCP_ME_CNTL, tmp); 3580 3581 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_LO, 3582 lower_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3583 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE1_HI, 3584 upper_32_bits(adev->gfx.me.me_fw_data_gpu_addr)); 3585 } 3586 soc21_grbm_select(adev, 0, 0, 0, 0); 3587 mutex_unlock(&adev->srbm_mutex); 3588 3589 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL); 3590 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, VMID, 0); 3591 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_BASE_CNTL, CACHE_POLICY, 0); 3592 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_BASE_CNTL, tmp); 3593 3594 /* Invalidate the data caches */ 3595 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3596 tmp = REG_SET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 3597 WREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL, tmp); 3598 3599 for (i = 0; i < usec_timeout; i++) { 3600 tmp = RREG32_SOC15(GC, 0, regCP_GFX_RS64_DC_OP_CNTL); 3601 if (1 == REG_GET_FIELD(tmp, CP_GFX_RS64_DC_OP_CNTL, 3602 INVALIDATE_DCACHE_COMPLETE)) 3603 break; 3604 udelay(1); 3605 } 3606 3607 if (i >= usec_timeout) { 3608 dev_err(adev->dev, "failed to invalidate RS64 data cache\n"); 3609 return -EINVAL; 3610 } 3611 3612 return 0; 3613 } 3614 3615 static int gfx_v11_0_cp_gfx_load_microcode(struct amdgpu_device *adev) 3616 { 3617 int r; 3618 3619 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw) 3620 return -EINVAL; 3621 3622 gfx_v11_0_cp_gfx_enable(adev, false); 3623 3624 if (adev->gfx.rs64_enable) 3625 r = gfx_v11_0_cp_gfx_load_pfp_microcode_rs64(adev); 3626 else 3627 r = gfx_v11_0_cp_gfx_load_pfp_microcode(adev); 3628 if (r) { 3629 dev_err(adev->dev, "(%d) failed to load pfp fw\n", r); 3630 return r; 3631 } 3632 3633 if (adev->gfx.rs64_enable) 3634 r = gfx_v11_0_cp_gfx_load_me_microcode_rs64(adev); 3635 else 3636 r = gfx_v11_0_cp_gfx_load_me_microcode(adev); 3637 if (r) { 3638 dev_err(adev->dev, "(%d) failed to load me fw\n", r); 3639 return r; 3640 } 3641 3642 return 0; 3643 } 3644 3645 static int gfx_v11_0_cp_gfx_start(struct amdgpu_device *adev) 3646 { 3647 struct amdgpu_ring *ring; 3648 const struct cs_section_def *sect = NULL; 3649 const struct cs_extent_def *ext = NULL; 3650 int r, i; 3651 int ctx_reg_offset; 3652 3653 /* init the CP */ 3654 WREG32_SOC15(GC, 0, regCP_MAX_CONTEXT, 3655 adev->gfx.config.max_hw_contexts - 1); 3656 WREG32_SOC15(GC, 0, regCP_DEVICE_ID, 1); 3657 3658 if (!amdgpu_async_gfx_ring) 3659 gfx_v11_0_cp_gfx_enable(adev, true); 3660 3661 ring = &adev->gfx.gfx_ring[0]; 3662 r = amdgpu_ring_alloc(ring, gfx_v11_0_get_csb_size(adev)); 3663 if (r) { 3664 drm_err(&adev->ddev, "cp failed to lock ring (%d).\n", r); 3665 return r; 3666 } 3667 3668 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3669 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE); 3670 3671 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 3672 amdgpu_ring_write(ring, 0x80000000); 3673 amdgpu_ring_write(ring, 0x80000000); 3674 3675 for (sect = gfx11_cs_data; sect->section != NULL; ++sect) { 3676 for (ext = sect->section; ext->extent != NULL; ++ext) { 3677 if (sect->id == SECT_CONTEXT) { 3678 amdgpu_ring_write(ring, 3679 PACKET3(PACKET3_SET_CONTEXT_REG, 3680 ext->reg_count)); 3681 amdgpu_ring_write(ring, ext->reg_index - 3682 PACKET3_SET_CONTEXT_REG_START); 3683 for (i = 0; i < ext->reg_count; i++) 3684 amdgpu_ring_write(ring, ext->extent[i]); 3685 } 3686 } 3687 } 3688 3689 ctx_reg_offset = 3690 SOC15_REG_OFFSET(GC, 0, regPA_SC_TILE_STEERING_OVERRIDE) - PACKET3_SET_CONTEXT_REG_START; 3691 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 1)); 3692 amdgpu_ring_write(ring, ctx_reg_offset); 3693 amdgpu_ring_write(ring, adev->gfx.config.pa_sc_tile_steering_override); 3694 3695 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0)); 3696 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE); 3697 3698 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3699 amdgpu_ring_write(ring, 0); 3700 3701 amdgpu_ring_commit(ring); 3702 3703 /* submit cs packet to copy state 0 to next available state */ 3704 if (adev->gfx.num_gfx_rings > 1) { 3705 /* maximum supported gfx ring is 2 */ 3706 ring = &adev->gfx.gfx_ring[1]; 3707 r = amdgpu_ring_alloc(ring, 2); 3708 if (r) { 3709 drm_err(adev_to_drm(adev), "cp failed to lock ring (%d).\n", r); 3710 return r; 3711 } 3712 3713 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0)); 3714 amdgpu_ring_write(ring, 0); 3715 3716 amdgpu_ring_commit(ring); 3717 } 3718 return 0; 3719 } 3720 3721 static void gfx_v11_0_cp_gfx_switch_pipe(struct amdgpu_device *adev, 3722 CP_PIPE_ID pipe) 3723 { 3724 u32 tmp; 3725 3726 tmp = RREG32_SOC15(GC, 0, regGRBM_GFX_CNTL); 3727 tmp = REG_SET_FIELD(tmp, GRBM_GFX_CNTL, PIPEID, pipe); 3728 3729 WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, tmp); 3730 } 3731 3732 static void gfx_v11_0_cp_gfx_set_doorbell(struct amdgpu_device *adev, 3733 struct amdgpu_ring *ring) 3734 { 3735 u32 tmp; 3736 3737 tmp = RREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL); 3738 if (ring->use_doorbell) { 3739 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3740 DOORBELL_OFFSET, ring->doorbell_index); 3741 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3742 DOORBELL_EN, 1); 3743 } else { 3744 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 3745 DOORBELL_EN, 0); 3746 } 3747 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_CONTROL, tmp); 3748 3749 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER, 3750 DOORBELL_RANGE_LOWER, ring->doorbell_index); 3751 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, tmp); 3752 3753 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 3754 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK); 3755 } 3756 3757 static int gfx_v11_0_cp_gfx_resume(struct amdgpu_device *adev) 3758 { 3759 struct amdgpu_ring *ring; 3760 u32 tmp; 3761 u32 rb_bufsz; 3762 u64 rb_addr, rptr_addr, wptr_gpu_addr; 3763 3764 /* Set the write pointer delay */ 3765 WREG32_SOC15(GC, 0, regCP_RB_WPTR_DELAY, 0); 3766 3767 /* set the RB to use vmid 0 */ 3768 WREG32_SOC15(GC, 0, regCP_RB_VMID, 0); 3769 3770 /* Init gfx ring 0 for pipe 0 */ 3771 mutex_lock(&adev->srbm_mutex); 3772 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3773 3774 /* Set ring buffer size */ 3775 ring = &adev->gfx.gfx_ring[0]; 3776 rb_bufsz = order_base_2(ring->ring_size / 8); 3777 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz); 3778 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2); 3779 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3780 3781 /* Initialize the ring buffer's write pointers */ 3782 ring->wptr = 0; 3783 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, lower_32_bits(ring->wptr)); 3784 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, upper_32_bits(ring->wptr)); 3785 3786 /* set the wb address whether it's enabled or not */ 3787 rptr_addr = ring->rptr_gpu_addr; 3788 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr)); 3789 WREG32_SOC15(GC, 0, regCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3790 CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3791 3792 wptr_gpu_addr = ring->wptr_gpu_addr; 3793 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3794 lower_32_bits(wptr_gpu_addr)); 3795 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3796 upper_32_bits(wptr_gpu_addr)); 3797 3798 mdelay(1); 3799 WREG32_SOC15(GC, 0, regCP_RB0_CNTL, tmp); 3800 3801 rb_addr = ring->gpu_addr >> 8; 3802 WREG32_SOC15(GC, 0, regCP_RB0_BASE, rb_addr); 3803 WREG32_SOC15(GC, 0, regCP_RB0_BASE_HI, upper_32_bits(rb_addr)); 3804 3805 WREG32_SOC15(GC, 0, regCP_RB_ACTIVE, 1); 3806 3807 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3808 mutex_unlock(&adev->srbm_mutex); 3809 3810 /* Init gfx ring 1 for pipe 1 */ 3811 if (adev->gfx.num_gfx_rings > 1) { 3812 mutex_lock(&adev->srbm_mutex); 3813 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID1); 3814 /* maximum supported gfx ring is 2 */ 3815 ring = &adev->gfx.gfx_ring[1]; 3816 rb_bufsz = order_base_2(ring->ring_size / 8); 3817 tmp = REG_SET_FIELD(0, CP_RB1_CNTL, RB_BUFSZ, rb_bufsz); 3818 tmp = REG_SET_FIELD(tmp, CP_RB1_CNTL, RB_BLKSZ, rb_bufsz - 2); 3819 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3820 /* Initialize the ring buffer's write pointers */ 3821 ring->wptr = 0; 3822 WREG32_SOC15(GC, 0, regCP_RB1_WPTR, lower_32_bits(ring->wptr)); 3823 WREG32_SOC15(GC, 0, regCP_RB1_WPTR_HI, upper_32_bits(ring->wptr)); 3824 /* Set the wb address whether it's enabled or not */ 3825 rptr_addr = ring->rptr_gpu_addr; 3826 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR, lower_32_bits(rptr_addr)); 3827 WREG32_SOC15(GC, 0, regCP_RB1_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & 3828 CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK); 3829 wptr_gpu_addr = ring->wptr_gpu_addr; 3830 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_LO, 3831 lower_32_bits(wptr_gpu_addr)); 3832 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_ADDR_HI, 3833 upper_32_bits(wptr_gpu_addr)); 3834 3835 mdelay(1); 3836 WREG32_SOC15(GC, 0, regCP_RB1_CNTL, tmp); 3837 3838 rb_addr = ring->gpu_addr >> 8; 3839 WREG32_SOC15(GC, 0, regCP_RB1_BASE, rb_addr); 3840 WREG32_SOC15(GC, 0, regCP_RB1_BASE_HI, upper_32_bits(rb_addr)); 3841 WREG32_SOC15(GC, 0, regCP_RB1_ACTIVE, 1); 3842 3843 gfx_v11_0_cp_gfx_set_doorbell(adev, ring); 3844 mutex_unlock(&adev->srbm_mutex); 3845 } 3846 /* Switch to pipe 0 */ 3847 mutex_lock(&adev->srbm_mutex); 3848 gfx_v11_0_cp_gfx_switch_pipe(adev, PIPE_ID0); 3849 mutex_unlock(&adev->srbm_mutex); 3850 3851 /* start the ring */ 3852 gfx_v11_0_cp_gfx_start(adev); 3853 3854 return 0; 3855 } 3856 3857 static void gfx_v11_0_cp_compute_enable(struct amdgpu_device *adev, bool enable) 3858 { 3859 u32 data; 3860 3861 if (adev->gfx.rs64_enable) { 3862 data = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); 3863 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE, 3864 enable ? 0 : 1); 3865 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 3866 enable ? 0 : 1); 3867 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 3868 enable ? 0 : 1); 3869 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 3870 enable ? 0 : 1); 3871 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 3872 enable ? 0 : 1); 3873 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE, 3874 enable ? 1 : 0); 3875 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE, 3876 enable ? 1 : 0); 3877 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE, 3878 enable ? 1 : 0); 3879 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE, 3880 enable ? 1 : 0); 3881 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT, 3882 enable ? 0 : 1); 3883 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, data); 3884 } else { 3885 data = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); 3886 3887 if (enable) { 3888 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 0); 3889 if (!adev->enable_mes_kiq) 3890 data = REG_SET_FIELD(data, CP_MEC_CNTL, 3891 MEC_ME2_HALT, 0); 3892 } else { 3893 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME1_HALT, 1); 3894 data = REG_SET_FIELD(data, CP_MEC_CNTL, MEC_ME2_HALT, 1); 3895 } 3896 WREG32_SOC15(GC, 0, regCP_MEC_CNTL, data); 3897 } 3898 3899 udelay(50); 3900 } 3901 3902 static int gfx_v11_0_cp_compute_load_microcode(struct amdgpu_device *adev) 3903 { 3904 const struct gfx_firmware_header_v1_0 *mec_hdr; 3905 const __le32 *fw_data; 3906 unsigned i, fw_size; 3907 u32 *fw = NULL; 3908 int r; 3909 3910 if (!adev->gfx.mec_fw) 3911 return -EINVAL; 3912 3913 gfx_v11_0_cp_compute_enable(adev, false); 3914 3915 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data; 3916 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3917 3918 fw_data = (const __le32 *) 3919 (adev->gfx.mec_fw->data + 3920 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes)); 3921 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes); 3922 3923 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes, 3924 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 3925 &adev->gfx.mec.mec_fw_obj, 3926 &adev->gfx.mec.mec_fw_gpu_addr, 3927 (void **)&fw); 3928 if (r) { 3929 dev_err(adev->dev, "(%d) failed to create mec fw bo\n", r); 3930 gfx_v11_0_mec_fini(adev); 3931 return r; 3932 } 3933 3934 memcpy(fw, fw_data, fw_size); 3935 3936 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 3937 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 3938 3939 gfx_v11_0_config_mec_cache(adev, adev->gfx.mec.mec_fw_gpu_addr); 3940 3941 /* MEC1 */ 3942 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, 0); 3943 3944 for (i = 0; i < mec_hdr->jt_size; i++) 3945 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_DATA, 3946 le32_to_cpup(fw_data + mec_hdr->jt_offset + i)); 3947 3948 WREG32_SOC15(GC, 0, regCP_MEC_ME1_UCODE_ADDR, adev->gfx.mec_fw_version); 3949 3950 return 0; 3951 } 3952 3953 static int gfx_v11_0_cp_compute_load_microcode_rs64(struct amdgpu_device *adev) 3954 { 3955 const struct gfx_firmware_header_v2_0 *mec_hdr; 3956 const __le32 *fw_ucode, *fw_data; 3957 u32 tmp, fw_ucode_size, fw_data_size; 3958 u32 i, usec_timeout = 50000; /* Wait for 50 ms */ 3959 u32 *fw_ucode_ptr, *fw_data_ptr; 3960 int r; 3961 3962 if (!adev->gfx.mec_fw) 3963 return -EINVAL; 3964 3965 gfx_v11_0_cp_compute_enable(adev, false); 3966 3967 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; 3968 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 3969 3970 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data + 3971 le32_to_cpu(mec_hdr->ucode_offset_bytes)); 3972 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes); 3973 3974 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 3975 le32_to_cpu(mec_hdr->data_offset_bytes)); 3976 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes); 3977 3978 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 3979 64 * 1024, 3980 AMDGPU_GEM_DOMAIN_VRAM | 3981 AMDGPU_GEM_DOMAIN_GTT, 3982 &adev->gfx.mec.mec_fw_obj, 3983 &adev->gfx.mec.mec_fw_gpu_addr, 3984 (void **)&fw_ucode_ptr); 3985 if (r) { 3986 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 3987 gfx_v11_0_mec_fini(adev); 3988 return r; 3989 } 3990 3991 r = amdgpu_bo_create_reserved(adev, fw_data_size, 3992 64 * 1024, 3993 AMDGPU_GEM_DOMAIN_VRAM | 3994 AMDGPU_GEM_DOMAIN_GTT, 3995 &adev->gfx.mec.mec_fw_data_obj, 3996 &adev->gfx.mec.mec_fw_data_gpu_addr, 3997 (void **)&fw_data_ptr); 3998 if (r) { 3999 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 4000 gfx_v11_0_mec_fini(adev); 4001 return r; 4002 } 4003 4004 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); 4005 memcpy(fw_data_ptr, fw_data, fw_data_size); 4006 4007 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 4008 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj); 4009 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 4010 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj); 4011 4012 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL); 4013 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 4014 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 4015 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 4016 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_CNTL, tmp); 4017 4018 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL); 4019 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 4020 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 4021 WREG32_SOC15(GC, 0, regCP_MEC_DC_BASE_CNTL, tmp); 4022 4023 mutex_lock(&adev->srbm_mutex); 4024 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 4025 soc21_grbm_select(adev, 1, i, 0, 0); 4026 4027 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_LO, adev->gfx.mec.mec_fw_data_gpu_addr); 4028 WREG32_SOC15(GC, 0, regCP_MEC_MDBASE_HI, 4029 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr)); 4030 4031 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START, 4032 mec_hdr->ucode_start_addr_lo >> 2 | 4033 mec_hdr->ucode_start_addr_hi << 30); 4034 WREG32_SOC15(GC, 0, regCP_MEC_RS64_PRGRM_CNTR_START_HI, 4035 mec_hdr->ucode_start_addr_hi >> 2); 4036 4037 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_LO, adev->gfx.mec.mec_fw_gpu_addr); 4038 WREG32_SOC15(GC, 0, regCP_CPC_IC_BASE_HI, 4039 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 4040 } 4041 mutex_unlock(&adev->srbm_mutex); 4042 soc21_grbm_select(adev, 0, 0, 0, 0); 4043 4044 /* Trigger an invalidation of the L1 instruction caches */ 4045 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4046 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 4047 WREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL, tmp); 4048 4049 /* Wait for invalidation complete */ 4050 for (i = 0; i < usec_timeout; i++) { 4051 tmp = RREG32_SOC15(GC, 0, regCP_MEC_DC_OP_CNTL); 4052 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 4053 INVALIDATE_DCACHE_COMPLETE)) 4054 break; 4055 udelay(1); 4056 } 4057 4058 if (i >= usec_timeout) { 4059 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4060 return -EINVAL; 4061 } 4062 4063 /* Trigger an invalidation of the L1 instruction caches */ 4064 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4065 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 4066 WREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL, tmp); 4067 4068 /* Wait for invalidation complete */ 4069 for (i = 0; i < usec_timeout; i++) { 4070 tmp = RREG32_SOC15(GC, 0, regCP_CPC_IC_OP_CNTL); 4071 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 4072 INVALIDATE_CACHE_COMPLETE)) 4073 break; 4074 udelay(1); 4075 } 4076 4077 if (i >= usec_timeout) { 4078 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 4079 return -EINVAL; 4080 } 4081 4082 return 0; 4083 } 4084 4085 static void gfx_v11_0_kiq_setting(struct amdgpu_ring *ring) 4086 { 4087 uint32_t tmp; 4088 struct amdgpu_device *adev = ring->adev; 4089 4090 /* tell RLC which is KIQ queue */ 4091 tmp = RREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS); 4092 tmp &= 0xffffff00; 4093 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 4094 WREG32_SOC15(GC, 0, regRLC_CP_SCHEDULERS, tmp | 0x80); 4095 } 4096 4097 static void gfx_v11_0_cp_set_doorbell_range(struct amdgpu_device *adev) 4098 { 4099 /* set graphics engine doorbell range */ 4100 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_LOWER, 4101 (adev->doorbell_index.gfx_ring0 * 2) << 2); 4102 WREG32_SOC15(GC, 0, regCP_RB_DOORBELL_RANGE_UPPER, 4103 (adev->doorbell_index.gfx_userqueue_end * 2) << 2); 4104 4105 /* set compute engine doorbell range */ 4106 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4107 (adev->doorbell_index.kiq * 2) << 2); 4108 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4109 (adev->doorbell_index.userqueue_end * 2) << 2); 4110 } 4111 4112 static void gfx_v11_0_gfx_mqd_set_priority(struct amdgpu_device *adev, 4113 struct v11_gfx_mqd *mqd, 4114 struct amdgpu_mqd_prop *prop) 4115 { 4116 bool priority = 0; 4117 u32 tmp; 4118 4119 /* set up default queue priority level 4120 * 0x0 = low priority, 0x1 = high priority 4121 */ 4122 if (prop->hqd_queue_priority == AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM) 4123 priority = 1; 4124 4125 tmp = regCP_GFX_HQD_QUEUE_PRIORITY_DEFAULT; 4126 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUEUE_PRIORITY, PRIORITY_LEVEL, priority); 4127 mqd->cp_gfx_hqd_queue_priority = tmp; 4128 } 4129 4130 static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, 4131 struct amdgpu_mqd_prop *prop) 4132 { 4133 struct v11_gfx_mqd *mqd = m; 4134 uint64_t hqd_gpu_addr, wb_gpu_addr; 4135 uint32_t tmp; 4136 uint32_t rb_bufsz; 4137 4138 /* set up gfx hqd wptr */ 4139 mqd->cp_gfx_hqd_wptr = 0; 4140 mqd->cp_gfx_hqd_wptr_hi = 0; 4141 4142 /* set the pointer to the MQD */ 4143 mqd->cp_mqd_base_addr = prop->mqd_gpu_addr & 0xfffffffc; 4144 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4145 4146 /* set up mqd control */ 4147 tmp = regCP_GFX_MQD_CONTROL_DEFAULT; 4148 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, VMID, 0); 4149 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, PRIV_STATE, 1); 4150 tmp = REG_SET_FIELD(tmp, CP_GFX_MQD_CONTROL, CACHE_POLICY, 0); 4151 mqd->cp_gfx_mqd_control = tmp; 4152 4153 /* set up gfx_hqd_vimd with 0x0 to indicate the ring buffer's vmid */ 4154 tmp = regCP_GFX_HQD_VMID_DEFAULT; 4155 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_VMID, VMID, 0); 4156 mqd->cp_gfx_hqd_vmid = 0; 4157 4158 /* set up gfx queue priority */ 4159 gfx_v11_0_gfx_mqd_set_priority(adev, mqd, prop); 4160 4161 /* set up time quantum */ 4162 tmp = regCP_GFX_HQD_QUANTUM_DEFAULT; 4163 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_QUANTUM, QUANTUM_EN, 1); 4164 mqd->cp_gfx_hqd_quantum = tmp; 4165 4166 /* set up gfx hqd base. this is similar as CP_RB_BASE */ 4167 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4168 mqd->cp_gfx_hqd_base = hqd_gpu_addr; 4169 mqd->cp_gfx_hqd_base_hi = upper_32_bits(hqd_gpu_addr); 4170 4171 /* set up hqd_rptr_addr/_hi, similar as CP_RB_RPTR */ 4172 wb_gpu_addr = prop->rptr_gpu_addr; 4173 mqd->cp_gfx_hqd_rptr_addr = wb_gpu_addr & 0xfffffffc; 4174 mqd->cp_gfx_hqd_rptr_addr_hi = 4175 upper_32_bits(wb_gpu_addr) & 0xffff; 4176 4177 /* set up rb_wptr_poll addr */ 4178 wb_gpu_addr = prop->wptr_gpu_addr; 4179 mqd->cp_rb_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4180 mqd->cp_rb_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4181 4182 /* set up the gfx_hqd_control, similar as CP_RB0_CNTL */ 4183 rb_bufsz = order_base_2(prop->queue_size / 4) - 1; 4184 tmp = regCP_GFX_HQD_CNTL_DEFAULT; 4185 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BUFSZ, rb_bufsz); 4186 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_BLKSZ, rb_bufsz - 2); 4187 #ifdef __BIG_ENDIAN 4188 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, BUF_SWAP, 1); 4189 #endif 4190 if (prop->tmz_queue) 4191 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, TMZ_MATCH, 1); 4192 if (!prop->kernel_queue) 4193 tmp = REG_SET_FIELD(tmp, CP_GFX_HQD_CNTL, RB_NON_PRIV, 1); 4194 mqd->cp_gfx_hqd_cntl = tmp; 4195 4196 /* set up cp_doorbell_control */ 4197 tmp = regCP_RB_DOORBELL_CONTROL_DEFAULT; 4198 if (prop->use_doorbell) { 4199 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4200 DOORBELL_OFFSET, prop->doorbell_index); 4201 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4202 DOORBELL_EN, 1); 4203 } else 4204 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, 4205 DOORBELL_EN, 0); 4206 mqd->cp_rb_doorbell_control = tmp; 4207 4208 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4209 mqd->cp_gfx_hqd_rptr = regCP_GFX_HQD_RPTR_DEFAULT; 4210 4211 /* active the queue */ 4212 mqd->cp_gfx_hqd_active = 1; 4213 4214 /* set gfx UQ items */ 4215 mqd->shadow_base_lo = lower_32_bits(prop->shadow_addr); 4216 mqd->shadow_base_hi = upper_32_bits(prop->shadow_addr); 4217 mqd->gds_bkup_base_lo = lower_32_bits(prop->gds_bkup_addr); 4218 mqd->gds_bkup_base_hi = upper_32_bits(prop->gds_bkup_addr); 4219 mqd->fw_work_area_base_lo = lower_32_bits(prop->csa_addr); 4220 mqd->fw_work_area_base_hi = upper_32_bits(prop->csa_addr); 4221 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4222 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4223 4224 return 0; 4225 } 4226 4227 static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring) 4228 { 4229 struct amdgpu_device *adev = ring->adev; 4230 struct v11_gfx_mqd *mqd = ring->mqd_ptr; 4231 int mqd_idx = ring - &adev->gfx.gfx_ring[0]; 4232 4233 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4234 memset((void *)mqd, 0, sizeof(*mqd)); 4235 mutex_lock(&adev->srbm_mutex); 4236 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4237 amdgpu_ring_init_mqd(ring); 4238 soc21_grbm_select(adev, 0, 0, 0, 0); 4239 mutex_unlock(&adev->srbm_mutex); 4240 if (adev->gfx.me.mqd_backup[mqd_idx]) 4241 memcpy_fromio(adev->gfx.me.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4242 } else { 4243 /* restore mqd with the backup copy */ 4244 if (adev->gfx.me.mqd_backup[mqd_idx]) 4245 memcpy_toio(mqd, adev->gfx.me.mqd_backup[mqd_idx], sizeof(*mqd)); 4246 /* reset the ring */ 4247 ring->wptr = 0; 4248 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4249 amdgpu_ring_clear_ring(ring); 4250 } 4251 4252 return 0; 4253 } 4254 4255 static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) 4256 { 4257 int r, i; 4258 4259 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4260 r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i]); 4261 if (r) 4262 return r; 4263 } 4264 4265 r = amdgpu_gfx_enable_kgq(adev, 0); 4266 if (r) 4267 return r; 4268 4269 return gfx_v11_0_cp_gfx_start(adev); 4270 } 4271 4272 static void gfx_v11_0_compute_mqd_set_cu_mask(struct amdgpu_device *adev, 4273 struct v11_compute_mqd *mqd, 4274 struct amdgpu_mqd_prop *prop) 4275 { 4276 uint32_t se_mask[8] = {0}; 4277 uint32_t wa_mask; 4278 bool has_wa_flag = prop->cu_flags & (AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE | 4279 AMDGPU_UPDATE_FLAG_DBG_WA_DISABLE); 4280 4281 if (!has_wa_flag && (!prop->cu_mask || !prop->cu_mask_count)) 4282 return; 4283 4284 if (has_wa_flag) { 4285 wa_mask = (prop->cu_flags & AMDGPU_UPDATE_FLAG_DBG_WA_ENABLE) ? 4286 0xffff : 0xffffffff; 4287 mqd->compute_static_thread_mgmt_se0 = wa_mask; 4288 mqd->compute_static_thread_mgmt_se1 = wa_mask; 4289 mqd->compute_static_thread_mgmt_se2 = wa_mask; 4290 mqd->compute_static_thread_mgmt_se3 = wa_mask; 4291 return; 4292 } 4293 4294 amdgpu_gfx_mqd_symmetrically_map_cu_mask(adev, prop->cu_mask, 4295 prop->cu_mask_count, se_mask); 4296 4297 mqd->compute_static_thread_mgmt_se0 = se_mask[0]; 4298 mqd->compute_static_thread_mgmt_se1 = se_mask[1]; 4299 mqd->compute_static_thread_mgmt_se2 = se_mask[2]; 4300 mqd->compute_static_thread_mgmt_se3 = se_mask[3]; 4301 } 4302 4303 static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m, 4304 struct amdgpu_mqd_prop *prop) 4305 { 4306 struct v11_compute_mqd *mqd = m; 4307 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 4308 uint32_t tmp; 4309 4310 mqd->header = 0xC0310800; 4311 mqd->compute_pipelinestat_enable = 0x00000001; 4312 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 4313 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 4314 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 4315 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 4316 mqd->compute_misc_reserved = 0x00000007; 4317 4318 eop_base_addr = prop->eop_gpu_addr >> 8; 4319 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 4320 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 4321 4322 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4323 tmp = regCP_HQD_EOP_CONTROL_DEFAULT; 4324 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 4325 (order_base_2(GFX11_MEC_HPD_SIZE / 4) - 1)); 4326 4327 mqd->cp_hqd_eop_control = tmp; 4328 4329 /* enable doorbell? */ 4330 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4331 4332 if (prop->use_doorbell) { 4333 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4334 DOORBELL_OFFSET, prop->doorbell_index); 4335 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4336 DOORBELL_EN, 1); 4337 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4338 DOORBELL_SOURCE, 0); 4339 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4340 DOORBELL_HIT, 0); 4341 } else { 4342 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4343 DOORBELL_EN, 0); 4344 } 4345 4346 mqd->cp_hqd_pq_doorbell_control = tmp; 4347 4348 /* disable the queue if it's active */ 4349 mqd->cp_hqd_dequeue_request = 0; 4350 mqd->cp_hqd_pq_rptr = 0; 4351 mqd->cp_hqd_pq_wptr_lo = 0; 4352 mqd->cp_hqd_pq_wptr_hi = 0; 4353 4354 /* set the pointer to the MQD */ 4355 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc; 4356 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 4357 4358 /* set MQD vmid to 0 */ 4359 tmp = regCP_MQD_CONTROL_DEFAULT; 4360 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 4361 mqd->cp_mqd_control = tmp; 4362 4363 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4364 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 4365 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 4366 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 4367 4368 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4369 tmp = regCP_HQD_PQ_CONTROL_DEFAULT; 4370 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 4371 (order_base_2(prop->queue_size / 4) - 1)); 4372 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 4373 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 4374 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 1); 4375 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 4376 prop->allow_tunneling); 4377 if (prop->kernel_queue) { 4378 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 4379 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 4380 } 4381 if (prop->tmz_queue) 4382 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TMZ, 1); 4383 mqd->cp_hqd_pq_control = tmp; 4384 4385 /* set the wb address whether it's enabled or not */ 4386 wb_gpu_addr = prop->rptr_gpu_addr; 4387 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 4388 mqd->cp_hqd_pq_rptr_report_addr_hi = 4389 upper_32_bits(wb_gpu_addr) & 0xffff; 4390 4391 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4392 wb_gpu_addr = prop->wptr_gpu_addr; 4393 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 4394 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 4395 4396 tmp = 0; 4397 /* enable the doorbell if requested */ 4398 if (prop->use_doorbell) { 4399 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 4400 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4401 DOORBELL_OFFSET, prop->doorbell_index); 4402 4403 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4404 DOORBELL_EN, 1); 4405 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4406 DOORBELL_SOURCE, 0); 4407 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 4408 DOORBELL_HIT, 0); 4409 } 4410 4411 mqd->cp_hqd_pq_doorbell_control = tmp; 4412 4413 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4414 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT; 4415 4416 /* set the vmid for the queue */ 4417 mqd->cp_hqd_vmid = 0; 4418 4419 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT; 4420 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x55); 4421 mqd->cp_hqd_persistent_state = tmp; 4422 4423 /* set MIN_IB_AVAIL_SIZE */ 4424 tmp = regCP_HQD_IB_CONTROL_DEFAULT; 4425 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3); 4426 mqd->cp_hqd_ib_control = tmp; 4427 4428 /* set static priority for a compute queue/ring */ 4429 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; 4430 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; 4431 4432 tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); 4433 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 4434 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); 4435 mqd->cp_hqd_quantum = tmp; 4436 4437 mqd->cp_hqd_active = prop->hqd_active; 4438 4439 /* set UQ fenceaddress */ 4440 mqd->fence_address_lo = lower_32_bits(prop->fence_address); 4441 mqd->fence_address_hi = upper_32_bits(prop->fence_address); 4442 /* set CU mask */ 4443 gfx_v11_0_compute_mqd_set_cu_mask(adev, mqd, prop); 4444 4445 return 0; 4446 } 4447 4448 static int gfx_v11_0_kiq_init_register(struct amdgpu_ring *ring) 4449 { 4450 struct amdgpu_device *adev = ring->adev; 4451 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4452 int j; 4453 4454 /* inactivate the queue */ 4455 if (amdgpu_sriov_vf(adev)) 4456 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); 4457 4458 /* disable wptr polling */ 4459 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0); 4460 4461 /* write the EOP addr */ 4462 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR, 4463 mqd->cp_hqd_eop_base_addr_lo); 4464 WREG32_SOC15(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI, 4465 mqd->cp_hqd_eop_base_addr_hi); 4466 4467 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 4468 WREG32_SOC15(GC, 0, regCP_HQD_EOP_CONTROL, 4469 mqd->cp_hqd_eop_control); 4470 4471 /* enable doorbell? */ 4472 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4473 mqd->cp_hqd_pq_doorbell_control); 4474 4475 /* disable the queue if it's active */ 4476 if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { 4477 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 1); 4478 for (j = 0; j < adev->usec_timeout; j++) { 4479 if (!(RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1)) 4480 break; 4481 udelay(1); 4482 } 4483 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 4484 mqd->cp_hqd_dequeue_request); 4485 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 4486 mqd->cp_hqd_pq_rptr); 4487 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4488 mqd->cp_hqd_pq_wptr_lo); 4489 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4490 mqd->cp_hqd_pq_wptr_hi); 4491 } 4492 4493 /* set the pointer to the MQD */ 4494 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR, 4495 mqd->cp_mqd_base_addr_lo); 4496 WREG32_SOC15(GC, 0, regCP_MQD_BASE_ADDR_HI, 4497 mqd->cp_mqd_base_addr_hi); 4498 4499 /* set MQD vmid to 0 */ 4500 WREG32_SOC15(GC, 0, regCP_MQD_CONTROL, 4501 mqd->cp_mqd_control); 4502 4503 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 4504 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE, 4505 mqd->cp_hqd_pq_base_lo); 4506 WREG32_SOC15(GC, 0, regCP_HQD_PQ_BASE_HI, 4507 mqd->cp_hqd_pq_base_hi); 4508 4509 /* set up the HQD, this is similar to CP_RB0_CNTL */ 4510 WREG32_SOC15(GC, 0, regCP_HQD_PQ_CONTROL, 4511 mqd->cp_hqd_pq_control); 4512 4513 /* set the wb address whether it's enabled or not */ 4514 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR, 4515 mqd->cp_hqd_pq_rptr_report_addr_lo); 4516 WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 4517 mqd->cp_hqd_pq_rptr_report_addr_hi); 4518 4519 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 4520 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR, 4521 mqd->cp_hqd_pq_wptr_poll_addr_lo); 4522 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI, 4523 mqd->cp_hqd_pq_wptr_poll_addr_hi); 4524 4525 /* enable the doorbell if requested */ 4526 if (ring->use_doorbell) { 4527 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_LOWER, 4528 (adev->doorbell_index.kiq * 2) << 2); 4529 WREG32_SOC15(GC, 0, regCP_MEC_DOORBELL_RANGE_UPPER, 4530 (adev->doorbell_index.userqueue_end * 2) << 2); 4531 } 4532 4533 WREG32_SOC15(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL, 4534 mqd->cp_hqd_pq_doorbell_control); 4535 4536 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 4537 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_LO, 4538 mqd->cp_hqd_pq_wptr_lo); 4539 WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 4540 mqd->cp_hqd_pq_wptr_hi); 4541 4542 /* set the vmid for the queue */ 4543 WREG32_SOC15(GC, 0, regCP_HQD_VMID, mqd->cp_hqd_vmid); 4544 4545 WREG32_SOC15(GC, 0, regCP_HQD_PERSISTENT_STATE, 4546 mqd->cp_hqd_persistent_state); 4547 4548 /* activate the queue */ 4549 WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 4550 mqd->cp_hqd_active); 4551 4552 if (ring->use_doorbell) 4553 WREG32_FIELD15_PREREG(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1); 4554 4555 return 0; 4556 } 4557 4558 static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) 4559 { 4560 struct amdgpu_device *adev = ring->adev; 4561 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4562 4563 gfx_v11_0_kiq_setting(ring); 4564 4565 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 4566 /* reset MQD to a clean status */ 4567 if (adev->gfx.kiq[0].mqd_backup) 4568 memcpy_toio(mqd, adev->gfx.kiq[0].mqd_backup, sizeof(*mqd)); 4569 4570 /* reset ring buffer */ 4571 ring->wptr = 0; 4572 amdgpu_ring_clear_ring(ring); 4573 4574 mutex_lock(&adev->srbm_mutex); 4575 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4576 gfx_v11_0_kiq_init_register(ring); 4577 soc21_grbm_select(adev, 0, 0, 0, 0); 4578 mutex_unlock(&adev->srbm_mutex); 4579 } else { 4580 memset((void *)mqd, 0, sizeof(*mqd)); 4581 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 4582 amdgpu_ring_clear_ring(ring); 4583 mutex_lock(&adev->srbm_mutex); 4584 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4585 amdgpu_ring_init_mqd(ring); 4586 gfx_v11_0_kiq_init_register(ring); 4587 soc21_grbm_select(adev, 0, 0, 0, 0); 4588 mutex_unlock(&adev->srbm_mutex); 4589 4590 if (adev->gfx.kiq[0].mqd_backup) 4591 memcpy_fromio(adev->gfx.kiq[0].mqd_backup, mqd, sizeof(*mqd)); 4592 } 4593 4594 return 0; 4595 } 4596 4597 static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring) 4598 { 4599 struct amdgpu_device *adev = ring->adev; 4600 struct v11_compute_mqd *mqd = ring->mqd_ptr; 4601 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 4602 4603 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 4604 memset((void *)mqd, 0, sizeof(*mqd)); 4605 mutex_lock(&adev->srbm_mutex); 4606 soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); 4607 amdgpu_ring_init_mqd(ring); 4608 soc21_grbm_select(adev, 0, 0, 0, 0); 4609 mutex_unlock(&adev->srbm_mutex); 4610 4611 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4612 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 4613 } else { 4614 /* restore MQD to a clean status */ 4615 if (adev->gfx.mec.mqd_backup[mqd_idx]) 4616 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); 4617 /* reset ring buffer */ 4618 ring->wptr = 0; 4619 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 4620 amdgpu_ring_clear_ring(ring); 4621 } 4622 4623 return 0; 4624 } 4625 4626 static int gfx_v11_0_kiq_resume(struct amdgpu_device *adev) 4627 { 4628 gfx_v11_0_kiq_init_queue(&adev->gfx.kiq[0].ring); 4629 return 0; 4630 } 4631 4632 static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev) 4633 { 4634 int i, r; 4635 4636 if (!amdgpu_async_gfx_ring) 4637 gfx_v11_0_cp_compute_enable(adev, true); 4638 4639 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4640 r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i]); 4641 if (r) 4642 return r; 4643 } 4644 4645 return amdgpu_gfx_enable_kcq(adev, 0); 4646 } 4647 4648 static int gfx_v11_0_cp_resume(struct amdgpu_device *adev) 4649 { 4650 int r, i; 4651 struct amdgpu_ring *ring; 4652 4653 if (!(adev->flags & AMD_IS_APU)) 4654 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 4655 4656 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4657 /* legacy firmware loading */ 4658 r = gfx_v11_0_cp_gfx_load_microcode(adev); 4659 if (r) 4660 return r; 4661 4662 if (adev->gfx.rs64_enable) 4663 r = gfx_v11_0_cp_compute_load_microcode_rs64(adev); 4664 else 4665 r = gfx_v11_0_cp_compute_load_microcode(adev); 4666 if (r) 4667 return r; 4668 } 4669 4670 gfx_v11_0_cp_set_doorbell_range(adev); 4671 4672 if (amdgpu_async_gfx_ring) { 4673 gfx_v11_0_cp_compute_enable(adev, true); 4674 gfx_v11_0_cp_gfx_enable(adev, true); 4675 } 4676 4677 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) 4678 r = amdgpu_mes_kiq_hw_init(adev, 0); 4679 else 4680 r = gfx_v11_0_kiq_resume(adev); 4681 if (r) 4682 return r; 4683 4684 r = gfx_v11_0_kcq_resume(adev); 4685 if (r) 4686 return r; 4687 4688 if (!amdgpu_async_gfx_ring) { 4689 r = gfx_v11_0_cp_gfx_resume(adev); 4690 if (r) 4691 return r; 4692 } else { 4693 r = gfx_v11_0_cp_async_gfx_ring_resume(adev); 4694 if (r) 4695 return r; 4696 } 4697 4698 if (adev->gfx.disable_kq) { 4699 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4700 ring = &adev->gfx.gfx_ring[i]; 4701 /* we don't want to set ring->ready */ 4702 r = amdgpu_ring_test_ring(ring); 4703 if (r) 4704 return r; 4705 } 4706 if (amdgpu_async_gfx_ring) 4707 amdgpu_gfx_disable_kgq(adev, 0); 4708 } else { 4709 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 4710 ring = &adev->gfx.gfx_ring[i]; 4711 r = amdgpu_ring_test_helper(ring); 4712 if (r) 4713 return r; 4714 } 4715 } 4716 4717 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4718 ring = &adev->gfx.compute_ring[i]; 4719 r = amdgpu_ring_test_helper(ring); 4720 if (r) 4721 return r; 4722 } 4723 4724 return 0; 4725 } 4726 4727 static void gfx_v11_0_cp_enable(struct amdgpu_device *adev, bool enable) 4728 { 4729 gfx_v11_0_cp_gfx_enable(adev, enable); 4730 gfx_v11_0_cp_compute_enable(adev, enable); 4731 } 4732 4733 static int gfx_v11_0_gfxhub_enable(struct amdgpu_device *adev) 4734 { 4735 int r; 4736 bool value; 4737 4738 r = adev->gfxhub.funcs->gart_enable(adev); 4739 if (r) 4740 return r; 4741 4742 amdgpu_device_flush_hdp(adev, NULL); 4743 4744 value = amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS; 4745 4746 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 4747 /* TODO investigate why this and the hdp flush above is needed, 4748 * are we missing a flush somewhere else? */ 4749 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0); 4750 4751 return 0; 4752 } 4753 4754 static void gfx_v11_0_select_cp_fw_arch(struct amdgpu_device *adev) 4755 { 4756 u32 tmp; 4757 4758 /* select RS64 */ 4759 if (adev->gfx.rs64_enable) { 4760 tmp = RREG32_SOC15(GC, 0, regCP_GFX_CNTL); 4761 tmp = REG_SET_FIELD(tmp, CP_GFX_CNTL, ENGINE_SEL, 1); 4762 WREG32_SOC15(GC, 0, regCP_GFX_CNTL, tmp); 4763 4764 tmp = RREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL); 4765 tmp = REG_SET_FIELD(tmp, CP_MEC_ISA_CNTL, ISA_MODE, 1); 4766 WREG32_SOC15(GC, 0, regCP_MEC_ISA_CNTL, tmp); 4767 } 4768 4769 if (amdgpu_emu_mode == 1) 4770 msleep(100); 4771 } 4772 4773 static int get_gb_addr_config(struct amdgpu_device * adev) 4774 { 4775 u32 gb_addr_config; 4776 4777 gb_addr_config = RREG32_SOC15(GC, 0, regGB_ADDR_CONFIG); 4778 if (gb_addr_config == 0) 4779 return -EINVAL; 4780 4781 adev->gfx.config.gb_addr_config_fields.num_pkrs = 4782 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG, NUM_PKRS); 4783 4784 adev->gfx.config.gb_addr_config = gb_addr_config; 4785 4786 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 4787 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4788 GB_ADDR_CONFIG, NUM_PIPES); 4789 4790 adev->gfx.config.max_tile_pipes = 4791 adev->gfx.config.gb_addr_config_fields.num_pipes; 4792 4793 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 4794 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4795 GB_ADDR_CONFIG, MAX_COMPRESSED_FRAGS); 4796 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 4797 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4798 GB_ADDR_CONFIG, NUM_RB_PER_SE); 4799 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 4800 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4801 GB_ADDR_CONFIG, NUM_SHADER_ENGINES); 4802 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 4803 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 4804 GB_ADDR_CONFIG, PIPE_INTERLEAVE_SIZE)); 4805 4806 return 0; 4807 } 4808 4809 static void gfx_v11_0_disable_gpa_mode(struct amdgpu_device *adev) 4810 { 4811 uint32_t data; 4812 4813 data = RREG32_SOC15(GC, 0, regCPC_PSP_DEBUG); 4814 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK; 4815 WREG32_SOC15(GC, 0, regCPC_PSP_DEBUG, data); 4816 4817 data = RREG32_SOC15(GC, 0, regCPG_PSP_DEBUG); 4818 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK; 4819 WREG32_SOC15(GC, 0, regCPG_PSP_DEBUG, data); 4820 } 4821 4822 static int gfx_v11_0_set_userq_eop_interrupts(struct amdgpu_device *adev, 4823 bool enable) 4824 { 4825 unsigned int irq_type; 4826 int m, p, r; 4827 4828 if (adev->userq_funcs[AMDGPU_HW_IP_GFX]) { 4829 for (m = 0; m < adev->gfx.me.num_me; m++) { 4830 for (p = 0; p < adev->gfx.me.num_pipe_per_me; p++) { 4831 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4832 if (enable) 4833 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4834 else 4835 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4836 if (r) { 4837 if (!enable) 4838 return r; 4839 goto err_gfx; 4840 } 4841 } 4842 } 4843 } 4844 4845 if (adev->userq_funcs[AMDGPU_HW_IP_COMPUTE]) { 4846 for (m = 0; m < adev->gfx.mec.num_mec; ++m) { 4847 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { 4848 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4849 + (m * adev->gfx.mec.num_pipe_per_mec) 4850 + p; 4851 if (enable) 4852 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 4853 else 4854 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4855 if (r) { 4856 if (!enable) 4857 return r; 4858 goto err_compute; 4859 } 4860 } 4861 } 4862 } 4863 4864 return 0; 4865 4866 err_compute: 4867 for (p--; p >= 0; p--) { 4868 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4869 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4870 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4871 } 4872 for (m--; m >= 0; m--) { 4873 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { 4874 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 4875 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 4876 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4877 } 4878 } 4879 m = adev->gfx.me.num_me; 4880 err_gfx: 4881 for (p--; p >= 0; p--) { 4882 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4883 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4884 } 4885 for (m--; m >= 0; m--) { 4886 for (p = adev->gfx.me.num_pipe_per_me - 1; p >= 0; p--) { 4887 irq_type = AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP + p; 4888 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 4889 } 4890 } 4891 return r; 4892 } 4893 4894 static int gfx_v11_0_hw_init(struct amdgpu_ip_block *ip_block) 4895 { 4896 int r; 4897 struct amdgpu_device *adev = ip_block->adev; 4898 4899 amdgpu_gfx_cleaner_shader_init(adev, adev->gfx.cleaner_shader_size, 4900 adev->gfx.cleaner_shader_ptr); 4901 4902 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 4903 if (adev->gfx.imu.funcs) { 4904 /* RLC autoload sequence 1: Program rlc ram */ 4905 if (adev->gfx.imu.funcs->program_rlc_ram) 4906 adev->gfx.imu.funcs->program_rlc_ram(adev); 4907 /* rlc autoload firmware */ 4908 r = gfx_v11_0_rlc_backdoor_autoload_enable(adev); 4909 if (r) 4910 return r; 4911 } 4912 } else { 4913 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 4914 if (adev->gfx.imu.funcs && (amdgpu_dpm > 0)) { 4915 if (adev->gfx.imu.funcs->load_microcode) 4916 adev->gfx.imu.funcs->load_microcode(adev); 4917 if (adev->gfx.imu.funcs->setup_imu) 4918 adev->gfx.imu.funcs->setup_imu(adev); 4919 if (adev->gfx.imu.funcs->start_imu) 4920 adev->gfx.imu.funcs->start_imu(adev); 4921 } 4922 4923 /* disable gpa mode in backdoor loading */ 4924 gfx_v11_0_disable_gpa_mode(adev); 4925 } 4926 } 4927 4928 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) || 4929 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 4930 r = gfx_v11_0_wait_for_rlc_autoload_complete(adev); 4931 if (r) { 4932 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r); 4933 return r; 4934 } 4935 } 4936 4937 adev->gfx.is_poweron = true; 4938 4939 if(get_gb_addr_config(adev)) 4940 drm_warn(adev_to_drm(adev), "Invalid gb_addr_config !\n"); 4941 4942 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP && 4943 adev->gfx.rs64_enable) 4944 gfx_v11_0_config_gfx_rs64(adev); 4945 4946 r = gfx_v11_0_gfxhub_enable(adev); 4947 if (r) 4948 return r; 4949 4950 if (!amdgpu_emu_mode) 4951 gfx_v11_0_init_golden_registers(adev); 4952 4953 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) || 4954 (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO && amdgpu_dpm == 1)) { 4955 /** 4956 * For gfx 11, rlc firmware loading relies on smu firmware is 4957 * loaded firstly, so in direct type, it has to load smc ucode 4958 * here before rlc. 4959 */ 4960 r = amdgpu_pm_load_smu_firmware(adev, NULL); 4961 if (r) 4962 return r; 4963 } 4964 4965 gfx_v11_0_constants_init(adev); 4966 4967 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) 4968 gfx_v11_0_select_cp_fw_arch(adev); 4969 4970 if (adev->nbio.funcs->gc_doorbell_init) 4971 adev->nbio.funcs->gc_doorbell_init(adev); 4972 4973 r = gfx_v11_0_rlc_resume(adev); 4974 if (r) 4975 return r; 4976 4977 /* 4978 * init golden registers and rlc resume may override some registers, 4979 * reconfig them here 4980 */ 4981 gfx_v11_0_tcp_harvest(adev); 4982 4983 r = gfx_v11_0_cp_resume(adev); 4984 if (r) 4985 return r; 4986 4987 /* get IMU version from HW if it's not set */ 4988 if (!adev->gfx.imu_fw_version) 4989 adev->gfx.imu_fw_version = RREG32_SOC15(GC, 0, regGFX_IMU_SCRATCH_0); 4990 4991 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 4992 if (r) 4993 return r; 4994 4995 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 4996 if (r) 4997 goto err_priv_inst; 4998 4999 r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); 5000 if (r) 5001 goto err_bad_op; 5002 5003 r = gfx_v11_0_set_userq_eop_interrupts(adev, true); 5004 if (r) 5005 goto err_userq_eop; 5006 5007 return 0; 5008 5009 err_userq_eop: 5010 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5011 err_bad_op: 5012 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5013 err_priv_inst: 5014 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5015 return r; 5016 } 5017 5018 static int gfx_v11_0_hw_fini(struct amdgpu_ip_block *ip_block) 5019 { 5020 struct amdgpu_device *adev = ip_block->adev; 5021 5022 cancel_delayed_work_sync(&adev->gfx.idle_work); 5023 5024 gfx_v11_0_set_userq_eop_interrupts(adev, false); 5025 amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); 5026 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 5027 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 5028 5029 if (!adev->no_hw_access) { 5030 if (amdgpu_async_gfx_ring && 5031 !adev->gfx.disable_kq) { 5032 if (amdgpu_gfx_disable_kgq(adev, 0)) 5033 DRM_ERROR("KGQ disable failed\n"); 5034 } 5035 5036 if (amdgpu_gfx_disable_kcq(adev, 0)) 5037 DRM_ERROR("KCQ disable failed\n"); 5038 5039 amdgpu_mes_kiq_hw_fini(adev, 0); 5040 } 5041 5042 if (amdgpu_sriov_vf(adev)) 5043 /* Remove the steps disabling CPG and clearing KIQ position, 5044 * so that CP could perform IDLE-SAVE during switch. Those 5045 * steps are necessary to avoid a DMAR error in gfx9 but it is 5046 * not reproduced on gfx11. 5047 */ 5048 return 0; 5049 5050 gfx_v11_0_cp_enable(adev, false); 5051 gfx_v11_0_enable_gui_idle_interrupt(adev, false); 5052 5053 adev->gfxhub.funcs->gart_disable(adev); 5054 5055 adev->gfx.is_poweron = false; 5056 5057 return 0; 5058 } 5059 5060 static int gfx_v11_0_suspend(struct amdgpu_ip_block *ip_block) 5061 { 5062 return gfx_v11_0_hw_fini(ip_block); 5063 } 5064 5065 static int gfx_v11_0_resume(struct amdgpu_ip_block *ip_block) 5066 { 5067 return gfx_v11_0_hw_init(ip_block); 5068 } 5069 5070 static bool gfx_v11_0_is_idle(struct amdgpu_ip_block *ip_block) 5071 { 5072 struct amdgpu_device *adev = ip_block->adev; 5073 5074 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, regGRBM_STATUS), 5075 GRBM_STATUS, GUI_ACTIVE)) 5076 return false; 5077 else 5078 return true; 5079 } 5080 5081 static int gfx_v11_0_wait_for_idle(struct amdgpu_ip_block *ip_block) 5082 { 5083 unsigned i; 5084 u32 tmp; 5085 struct amdgpu_device *adev = ip_block->adev; 5086 5087 for (i = 0; i < adev->usec_timeout; i++) { 5088 /* read MC_STATUS */ 5089 tmp = RREG32_SOC15(GC, 0, regGRBM_STATUS) & 5090 GRBM_STATUS__GUI_ACTIVE_MASK; 5091 5092 if (!REG_GET_FIELD(tmp, GRBM_STATUS, GUI_ACTIVE)) 5093 return 0; 5094 udelay(1); 5095 } 5096 return -ETIMEDOUT; 5097 } 5098 5099 int gfx_v11_0_request_gfx_index_mutex(struct amdgpu_device *adev, 5100 bool req) 5101 { 5102 u32 i, tmp, val; 5103 5104 for (i = 0; i < adev->usec_timeout; i++) { 5105 /* Request with MeId=2, PipeId=0 */ 5106 tmp = REG_SET_FIELD(0, CP_GFX_INDEX_MUTEX, REQUEST, req); 5107 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, CLIENTID, 4); 5108 WREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX, tmp); 5109 5110 val = RREG32_SOC15(GC, 0, regCP_GFX_INDEX_MUTEX); 5111 if (req) { 5112 if (val == tmp) 5113 break; 5114 } else { 5115 tmp = REG_SET_FIELD(tmp, CP_GFX_INDEX_MUTEX, 5116 REQUEST, 1); 5117 5118 /* unlocked or locked by firmware */ 5119 if (val != tmp) 5120 break; 5121 } 5122 udelay(1); 5123 } 5124 5125 if (i >= adev->usec_timeout) 5126 return -EINVAL; 5127 5128 return 0; 5129 } 5130 5131 static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block) 5132 { 5133 u32 grbm_soft_reset = 0; 5134 u32 tmp; 5135 int r, i, j, k; 5136 struct amdgpu_device *adev = ip_block->adev; 5137 5138 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5139 5140 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5141 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 0); 5142 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 0); 5143 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 0); 5144 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 0); 5145 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5146 5147 mutex_lock(&adev->srbm_mutex); 5148 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 5149 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 5150 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 5151 soc21_grbm_select(adev, i, k, j, 0); 5152 5153 WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2); 5154 WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1); 5155 } 5156 } 5157 } 5158 for (i = 0; i < adev->gfx.me.num_me; ++i) { 5159 for (j = 0; j < adev->gfx.me.num_queue_per_pipe; j++) { 5160 for (k = 0; k < adev->gfx.me.num_pipe_per_me; k++) { 5161 soc21_grbm_select(adev, i, k, j, 0); 5162 5163 WREG32_SOC15(GC, 0, regCP_GFX_HQD_DEQUEUE_REQUEST, 0x1); 5164 } 5165 } 5166 } 5167 soc21_grbm_select(adev, 0, 0, 0, 0); 5168 mutex_unlock(&adev->srbm_mutex); 5169 5170 /* Try to acquire the gfx mutex before access to CP_VMID_RESET */ 5171 mutex_lock(&adev->gfx.reset_sem_mutex); 5172 r = gfx_v11_0_request_gfx_index_mutex(adev, true); 5173 if (r) { 5174 mutex_unlock(&adev->gfx.reset_sem_mutex); 5175 DRM_ERROR("Failed to acquire the gfx mutex during soft reset\n"); 5176 return r; 5177 } 5178 5179 WREG32_SOC15(GC, 0, regCP_VMID_RESET, 0xfffffffe); 5180 5181 // Read CP_VMID_RESET register three times. 5182 // to get sufficient time for GFX_HQD_ACTIVE reach 0 5183 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5184 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5185 RREG32_SOC15(GC, 0, regCP_VMID_RESET); 5186 5187 /* release the gfx mutex */ 5188 r = gfx_v11_0_request_gfx_index_mutex(adev, false); 5189 mutex_unlock(&adev->gfx.reset_sem_mutex); 5190 if (r) { 5191 DRM_ERROR("Failed to release the gfx mutex during soft reset\n"); 5192 return r; 5193 } 5194 5195 for (i = 0; i < adev->usec_timeout; i++) { 5196 if (!RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) && 5197 !RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE)) 5198 break; 5199 udelay(1); 5200 } 5201 if (i >= adev->usec_timeout) { 5202 printk("Failed to wait all pipes clean\n"); 5203 return -EINVAL; 5204 } 5205 5206 /********** trigger soft reset ***********/ 5207 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5208 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5209 SOFT_RESET_CP, 1); 5210 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5211 SOFT_RESET_GFX, 1); 5212 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5213 SOFT_RESET_CPF, 1); 5214 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5215 SOFT_RESET_CPC, 1); 5216 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5217 SOFT_RESET_CPG, 1); 5218 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5219 /********** exit soft reset ***********/ 5220 grbm_soft_reset = RREG32_SOC15(GC, 0, regGRBM_SOFT_RESET); 5221 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5222 SOFT_RESET_CP, 0); 5223 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5224 SOFT_RESET_GFX, 0); 5225 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5226 SOFT_RESET_CPF, 0); 5227 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5228 SOFT_RESET_CPC, 0); 5229 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, 5230 SOFT_RESET_CPG, 0); 5231 WREG32_SOC15(GC, 0, regGRBM_SOFT_RESET, grbm_soft_reset); 5232 5233 tmp = RREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL); 5234 tmp = REG_SET_FIELD(tmp, CP_SOFT_RESET_CNTL, CMP_HQD_REG_RESET, 0x1); 5235 WREG32_SOC15(GC, 0, regCP_SOFT_RESET_CNTL, tmp); 5236 5237 WREG32_SOC15(GC, 0, regCP_ME_CNTL, 0x0); 5238 WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, 0x0); 5239 5240 for (i = 0; i < adev->usec_timeout; i++) { 5241 if (!RREG32_SOC15(GC, 0, regCP_VMID_RESET)) 5242 break; 5243 udelay(1); 5244 } 5245 if (i >= adev->usec_timeout) { 5246 printk("Failed to wait CP_VMID_RESET to 0\n"); 5247 return -EINVAL; 5248 } 5249 5250 tmp = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5251 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5252 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5253 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5254 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5255 WREG32_SOC15(GC, 0, regCP_INT_CNTL, tmp); 5256 5257 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5258 5259 r = gfx_v11_0_cp_resume(adev); 5260 if (r) 5261 return r; 5262 5263 /** 5264 * GFX soft reset impacts MES, resume MES after GFX soft reset is finished 5265 */ 5266 return amdgpu_mes_resume(adev, 0); 5267 } 5268 5269 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev) 5270 { 5271 uint64_t clock; 5272 uint64_t clock_counter_lo, clock_counter_hi_pre, clock_counter_hi_after; 5273 5274 if (amdgpu_sriov_vf(adev)) { 5275 amdgpu_gfx_off_ctrl(adev, false); 5276 mutex_lock(&adev->gfx.gpu_clock_mutex); 5277 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5278 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5279 clock_counter_hi_after = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_HI); 5280 if (clock_counter_hi_pre != clock_counter_hi_after) 5281 clock_counter_lo = (uint64_t)RREG32_SOC15(GC, 0, regCP_MES_MTIME_LO); 5282 mutex_unlock(&adev->gfx.gpu_clock_mutex); 5283 amdgpu_gfx_off_ctrl(adev, true); 5284 } else { 5285 preempt_disable(); 5286 if (amdgpu_ip_version(adev, SMUIO_HWIP, 0) < IP_VERSION(15, 0, 0)) { 5287 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5288 regGOLDEN_TSC_COUNT_UPPER); 5289 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5290 regGOLDEN_TSC_COUNT_LOWER); 5291 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5292 regGOLDEN_TSC_COUNT_UPPER); 5293 if (clock_counter_hi_pre != clock_counter_hi_after) 5294 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5295 regGOLDEN_TSC_COUNT_LOWER); 5296 } else { 5297 clock_counter_hi_pre = (uint64_t)RREG32_SOC15(SMUIO, 0, 5298 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5299 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5300 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5301 clock_counter_hi_after = (uint64_t)RREG32_SOC15(SMUIO, 0, 5302 regGOLDEN_TSC_COUNT_UPPER_smu_15_0_0); 5303 if (clock_counter_hi_pre != clock_counter_hi_after) 5304 clock_counter_lo = (uint64_t)RREG32_SOC15(SMUIO, 0, 5305 regGOLDEN_TSC_COUNT_LOWER_smu_15_0_0); 5306 } 5307 preempt_enable(); 5308 } 5309 clock = clock_counter_lo | (clock_counter_hi_after << 32ULL); 5310 5311 return clock; 5312 } 5313 5314 static void gfx_v11_0_ring_emit_gds_switch(struct amdgpu_ring *ring, 5315 uint32_t vmid, 5316 uint32_t gds_base, uint32_t gds_size, 5317 uint32_t gws_base, uint32_t gws_size, 5318 uint32_t oa_base, uint32_t oa_size) 5319 { 5320 struct amdgpu_device *adev = ring->adev; 5321 5322 /* GDS Base */ 5323 gfx_v11_0_write_data_to_reg(ring, 0, false, 5324 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_BASE) + 2 * vmid, 5325 gds_base); 5326 5327 /* GDS Size */ 5328 gfx_v11_0_write_data_to_reg(ring, 0, false, 5329 SOC15_REG_OFFSET(GC, 0, regGDS_VMID0_SIZE) + 2 * vmid, 5330 gds_size); 5331 5332 /* GWS */ 5333 gfx_v11_0_write_data_to_reg(ring, 0, false, 5334 SOC15_REG_OFFSET(GC, 0, regGDS_GWS_VMID0) + vmid, 5335 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base); 5336 5337 /* OA */ 5338 gfx_v11_0_write_data_to_reg(ring, 0, false, 5339 SOC15_REG_OFFSET(GC, 0, regGDS_OA_VMID0) + vmid, 5340 (1 << (oa_size + oa_base)) - (1 << oa_base)); 5341 } 5342 5343 static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) 5344 { 5345 struct amdgpu_device *adev = ip_block->adev; 5346 5347 switch (amdgpu_user_queue) { 5348 case -1: 5349 case 0: 5350 default: 5351 adev->gfx.disable_kq = false; 5352 adev->gfx.disable_uq = true; 5353 break; 5354 case 1: 5355 adev->gfx.disable_kq = false; 5356 adev->gfx.disable_uq = false; 5357 break; 5358 case 2: 5359 adev->gfx.disable_kq = true; 5360 adev->gfx.disable_uq = false; 5361 break; 5362 } 5363 5364 adev->gfx.funcs = &gfx_v11_0_gfx_funcs; 5365 5366 if (adev->gfx.disable_kq) { 5367 /* We need one GFX ring temporarily to set up 5368 * the clear state. 5369 */ 5370 adev->gfx.num_gfx_rings = 1; 5371 adev->gfx.num_compute_rings = 0; 5372 } else { 5373 adev->gfx.num_gfx_rings = GFX11_NUM_GFX_RINGS; 5374 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 5375 AMDGPU_MAX_COMPUTE_RINGS); 5376 } 5377 5378 gfx_v11_0_set_kiq_pm4_funcs(adev); 5379 gfx_v11_0_set_ring_funcs(adev); 5380 gfx_v11_0_set_irq_funcs(adev); 5381 gfx_v11_0_set_gds_init(adev); 5382 gfx_v11_0_set_rlc_funcs(adev); 5383 gfx_v11_0_set_mqd_funcs(adev); 5384 gfx_v11_0_set_imu_funcs(adev); 5385 5386 gfx_v11_0_init_rlcg_reg_access_ctrl(adev); 5387 5388 amdgpu_init_rlc_reg_funcs(adev); 5389 5390 return gfx_v11_0_init_microcode(adev); 5391 } 5392 5393 static bool gfx_v11_0_is_rlc_enabled(struct amdgpu_device *adev) 5394 { 5395 uint32_t rlc_cntl; 5396 5397 /* if RLC is not enabled, do nothing */ 5398 rlc_cntl = RREG32_SOC15(GC, 0, regRLC_CNTL); 5399 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; 5400 } 5401 5402 static void gfx_v11_0_set_safe_mode(struct amdgpu_device *adev, int xcc_id) 5403 { 5404 uint32_t data; 5405 unsigned i; 5406 5407 data = RLC_SAFE_MODE__CMD_MASK; 5408 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 5409 5410 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, data); 5411 5412 /* wait for RLC_SAFE_MODE */ 5413 for (i = 0; i < adev->usec_timeout; i++) { 5414 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, regRLC_SAFE_MODE), 5415 RLC_SAFE_MODE, CMD)) 5416 break; 5417 udelay(1); 5418 } 5419 } 5420 5421 static void gfx_v11_0_unset_safe_mode(struct amdgpu_device *adev, int xcc_id) 5422 { 5423 WREG32_SOC15(GC, 0, regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK); 5424 } 5425 5426 static void gfx_v11_0_update_perf_clk(struct amdgpu_device *adev, 5427 bool enable) 5428 { 5429 uint32_t def, data; 5430 5431 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK)) 5432 return; 5433 5434 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5435 5436 if (enable) 5437 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5438 else 5439 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 5440 5441 if (def != data) 5442 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5443 } 5444 5445 static void gfx_v11_0_update_sram_fgcg(struct amdgpu_device *adev, 5446 bool enable) 5447 { 5448 uint32_t def, data; 5449 5450 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG)) 5451 return; 5452 5453 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5454 5455 if (enable) 5456 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5457 else 5458 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 5459 5460 if (def != data) 5461 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5462 } 5463 5464 static void gfx_v11_0_update_repeater_fgcg(struct amdgpu_device *adev, 5465 bool enable) 5466 { 5467 uint32_t def, data; 5468 5469 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG)) 5470 return; 5471 5472 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5473 5474 if (enable) 5475 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5476 else 5477 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK; 5478 5479 if (def != data) 5480 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5481 } 5482 5483 static void gfx_v11_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 5484 bool enable) 5485 { 5486 uint32_t data, def; 5487 5488 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS))) 5489 return; 5490 5491 /* It is disabled by HW by default */ 5492 if (enable) { 5493 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5494 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 5495 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5496 5497 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5498 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5499 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5500 5501 if (def != data) 5502 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5503 } 5504 } else { 5505 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 5506 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5507 5508 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 5509 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 5510 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 5511 5512 if (def != data) 5513 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5514 } 5515 } 5516 } 5517 5518 static void gfx_v11_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 5519 bool enable) 5520 { 5521 uint32_t def, data; 5522 5523 if (!(adev->cg_flags & 5524 (AMD_CG_SUPPORT_GFX_CGCG | 5525 AMD_CG_SUPPORT_GFX_CGLS | 5526 AMD_CG_SUPPORT_GFX_3D_CGCG | 5527 AMD_CG_SUPPORT_GFX_3D_CGLS))) 5528 return; 5529 5530 if (enable) { 5531 def = data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5532 5533 /* unset CGCG override */ 5534 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5535 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 5536 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5537 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 5538 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG || 5539 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5540 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 5541 5542 /* update CGCG override bits */ 5543 if (def != data) 5544 WREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE, data); 5545 5546 /* enable cgcg FSM(0x0000363F) */ 5547 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5548 5549 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 5550 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK; 5551 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5552 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5553 } 5554 5555 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 5556 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK; 5557 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5558 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5559 } 5560 5561 if (def != data) 5562 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5563 5564 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5565 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5566 5567 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) { 5568 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK; 5569 data |= (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 5570 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5571 } 5572 5573 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) { 5574 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK; 5575 data |= (0xf << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 5576 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5577 } 5578 5579 if (def != data) 5580 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5581 5582 /* set IDLE_POLL_COUNT(0x00900100) */ 5583 def = data = RREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL); 5584 5585 data &= ~(CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK | CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK); 5586 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 5587 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 5588 5589 if (def != data) 5590 WREG32_SOC15(GC, 0, regCP_RB_WPTR_POLL_CNTL, data); 5591 5592 data = RREG32_SOC15(GC, 0, regCP_INT_CNTL); 5593 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 5594 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 5595 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 5596 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 5597 WREG32_SOC15(GC, 0, regCP_INT_CNTL, data); 5598 5599 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5600 data = REG_SET_FIELD(data, SDMA0_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5601 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5602 5603 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5604 if (adev->sdma.num_instances > 1) { 5605 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5606 data = REG_SET_FIELD(data, SDMA1_RLC_CGCG_CTRL, CGCG_INT_ENABLE, 1); 5607 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5608 } 5609 } else { 5610 /* Program RLC_CGCG_CGLS_CTRL */ 5611 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5612 5613 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 5614 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 5615 5616 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 5617 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 5618 5619 if (def != data) 5620 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL, data); 5621 5622 /* Program RLC_CGCG_CGLS_CTRL_3D */ 5623 def = data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5624 5625 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG) 5626 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK; 5627 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 5628 data &= ~RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK; 5629 5630 if (def != data) 5631 WREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D, data); 5632 5633 data = RREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL); 5634 data &= ~SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5635 WREG32_SOC15(GC, 0, regSDMA0_RLC_CGCG_CTRL, data); 5636 5637 /* Some ASICs only have one SDMA instance, not need to configure SDMA1 */ 5638 if (adev->sdma.num_instances > 1) { 5639 data = RREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL); 5640 data &= ~SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK; 5641 WREG32_SOC15(GC, 0, regSDMA1_RLC_CGCG_CTRL, data); 5642 } 5643 } 5644 } 5645 5646 static int gfx_v11_0_update_gfx_clock_gating(struct amdgpu_device *adev, 5647 bool enable) 5648 { 5649 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5650 5651 gfx_v11_0_update_coarse_grain_clock_gating(adev, enable); 5652 5653 gfx_v11_0_update_medium_grain_clock_gating(adev, enable); 5654 5655 gfx_v11_0_update_repeater_fgcg(adev, enable); 5656 5657 gfx_v11_0_update_sram_fgcg(adev, enable); 5658 5659 gfx_v11_0_update_perf_clk(adev, enable); 5660 5661 if (adev->cg_flags & 5662 (AMD_CG_SUPPORT_GFX_MGCG | 5663 AMD_CG_SUPPORT_GFX_CGLS | 5664 AMD_CG_SUPPORT_GFX_CGCG | 5665 AMD_CG_SUPPORT_GFX_3D_CGCG | 5666 AMD_CG_SUPPORT_GFX_3D_CGLS)) 5667 gfx_v11_0_enable_gui_idle_interrupt(adev, enable); 5668 5669 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5670 5671 return 0; 5672 } 5673 5674 static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, int xcc_id, 5675 struct amdgpu_ring *ring, unsigned vmid) 5676 { 5677 u32 reg, pre_data, data; 5678 5679 amdgpu_gfx_off_ctrl(adev, false); 5680 reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL); 5681 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) 5682 pre_data = RREG32_NO_KIQ(reg); 5683 else 5684 pre_data = RREG32(reg); 5685 5686 data = pre_data & (~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK); 5687 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 5688 5689 if (pre_data != data) { 5690 if (amdgpu_sriov_is_pp_one_vf(adev) && !amdgpu_sriov_runtime(adev)) { 5691 WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data); 5692 } else 5693 WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data); 5694 } 5695 amdgpu_gfx_off_ctrl(adev, true); 5696 5697 if (ring 5698 && amdgpu_sriov_is_pp_one_vf(adev) 5699 && (pre_data != data) 5700 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) 5701 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) { 5702 amdgpu_ring_emit_wreg(ring, reg, data); 5703 } 5704 } 5705 5706 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = { 5707 .is_rlc_enabled = gfx_v11_0_is_rlc_enabled, 5708 .set_safe_mode = gfx_v11_0_set_safe_mode, 5709 .unset_safe_mode = gfx_v11_0_unset_safe_mode, 5710 .init = gfx_v11_0_rlc_init, 5711 .get_csb_size = gfx_v11_0_get_csb_size, 5712 .get_csb_buffer = gfx_v11_0_get_csb_buffer, 5713 .resume = gfx_v11_0_rlc_resume, 5714 .stop = gfx_v11_0_rlc_stop, 5715 .reset = gfx_v11_0_rlc_reset, 5716 .start = gfx_v11_0_rlc_start, 5717 .update_spm_vmid = gfx_v11_0_update_spm_vmid, 5718 }; 5719 5720 static void gfx_v11_cntl_power_gating(struct amdgpu_device *adev, bool enable) 5721 { 5722 u32 data = RREG32_SOC15(GC, 0, regRLC_PG_CNTL); 5723 5724 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) 5725 data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5726 else 5727 data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK; 5728 5729 WREG32_SOC15(GC, 0, regRLC_PG_CNTL, data); 5730 5731 // Program RLC_PG_DELAY3 for CGPG hysteresis 5732 if (enable && (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)) { 5733 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5734 case IP_VERSION(11, 0, 1): 5735 case IP_VERSION(11, 0, 4): 5736 case IP_VERSION(11, 5, 0): 5737 case IP_VERSION(11, 5, 1): 5738 case IP_VERSION(11, 5, 2): 5739 case IP_VERSION(11, 5, 3): 5740 case IP_VERSION(11, 5, 4): 5741 case IP_VERSION(11, 5, 6): 5742 WREG32_SOC15(GC, 0, regRLC_PG_DELAY_3, RLC_PG_DELAY_3_DEFAULT_GC_11_0_1); 5743 break; 5744 default: 5745 break; 5746 } 5747 } 5748 } 5749 5750 static void gfx_v11_cntl_pg(struct amdgpu_device *adev, bool enable) 5751 { 5752 amdgpu_gfx_rlc_enter_safe_mode(adev, 0); 5753 5754 gfx_v11_cntl_power_gating(adev, enable); 5755 5756 amdgpu_gfx_rlc_exit_safe_mode(adev, 0); 5757 } 5758 5759 static int gfx_v11_0_set_powergating_state(struct amdgpu_ip_block *ip_block, 5760 enum amd_powergating_state state) 5761 { 5762 struct amdgpu_device *adev = ip_block->adev; 5763 bool enable = (state == AMD_PG_STATE_GATE); 5764 5765 if (amdgpu_sriov_vf(adev)) 5766 return 0; 5767 5768 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5769 case IP_VERSION(11, 0, 0): 5770 case IP_VERSION(11, 0, 2): 5771 case IP_VERSION(11, 0, 3): 5772 amdgpu_gfx_off_ctrl(adev, enable); 5773 break; 5774 case IP_VERSION(11, 0, 1): 5775 case IP_VERSION(11, 0, 4): 5776 case IP_VERSION(11, 5, 0): 5777 case IP_VERSION(11, 5, 1): 5778 case IP_VERSION(11, 5, 2): 5779 case IP_VERSION(11, 5, 3): 5780 case IP_VERSION(11, 5, 4): 5781 case IP_VERSION(11, 5, 6): 5782 if (!enable) 5783 amdgpu_gfx_off_ctrl(adev, false); 5784 5785 gfx_v11_cntl_pg(adev, enable); 5786 5787 if (enable) 5788 amdgpu_gfx_off_ctrl(adev, true); 5789 5790 break; 5791 default: 5792 break; 5793 } 5794 5795 return 0; 5796 } 5797 5798 static int gfx_v11_0_set_clockgating_state(struct amdgpu_ip_block *ip_block, 5799 enum amd_clockgating_state state) 5800 { 5801 struct amdgpu_device *adev = ip_block->adev; 5802 5803 if (amdgpu_sriov_vf(adev)) 5804 return 0; 5805 5806 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 5807 case IP_VERSION(11, 0, 0): 5808 case IP_VERSION(11, 0, 1): 5809 case IP_VERSION(11, 0, 2): 5810 case IP_VERSION(11, 0, 3): 5811 case IP_VERSION(11, 0, 4): 5812 case IP_VERSION(11, 5, 0): 5813 case IP_VERSION(11, 5, 1): 5814 case IP_VERSION(11, 5, 2): 5815 case IP_VERSION(11, 5, 3): 5816 case IP_VERSION(11, 5, 4): 5817 case IP_VERSION(11, 5, 6): 5818 gfx_v11_0_update_gfx_clock_gating(adev, 5819 state == AMD_CG_STATE_GATE); 5820 break; 5821 default: 5822 break; 5823 } 5824 5825 return 0; 5826 } 5827 5828 static void gfx_v11_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 5829 { 5830 struct amdgpu_device *adev = ip_block->adev; 5831 int data; 5832 5833 /* AMD_CG_SUPPORT_GFX_MGCG */ 5834 data = RREG32_SOC15(GC, 0, regRLC_CGTT_MGCG_OVERRIDE); 5835 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 5836 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 5837 5838 /* AMD_CG_SUPPORT_REPEATER_FGCG */ 5839 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK)) 5840 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG; 5841 5842 /* AMD_CG_SUPPORT_GFX_FGCG */ 5843 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK)) 5844 *flags |= AMD_CG_SUPPORT_GFX_FGCG; 5845 5846 /* AMD_CG_SUPPORT_GFX_PERF_CLK */ 5847 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK)) 5848 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK; 5849 5850 /* AMD_CG_SUPPORT_GFX_CGCG */ 5851 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL); 5852 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 5853 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 5854 5855 /* AMD_CG_SUPPORT_GFX_CGLS */ 5856 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 5857 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 5858 5859 /* AMD_CG_SUPPORT_GFX_3D_CGCG */ 5860 data = RREG32_SOC15(GC, 0, regRLC_CGCG_CGLS_CTRL_3D); 5861 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK) 5862 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG; 5863 5864 /* AMD_CG_SUPPORT_GFX_3D_CGLS */ 5865 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK) 5866 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS; 5867 } 5868 5869 static u64 gfx_v11_0_ring_get_rptr_gfx(struct amdgpu_ring *ring) 5870 { 5871 /* gfx11 is 32bit rptr*/ 5872 return *(uint32_t *)ring->rptr_cpu_addr; 5873 } 5874 5875 static u64 gfx_v11_0_ring_get_wptr_gfx(struct amdgpu_ring *ring) 5876 { 5877 struct amdgpu_device *adev = ring->adev; 5878 u64 wptr; 5879 5880 /* XXX check if swapping is necessary on BE */ 5881 if (ring->use_doorbell) { 5882 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5883 } else { 5884 wptr = RREG32_SOC15(GC, 0, regCP_RB0_WPTR); 5885 wptr += (u64)RREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI) << 32; 5886 } 5887 5888 return wptr; 5889 } 5890 5891 static void gfx_v11_0_ring_set_wptr_gfx(struct amdgpu_ring *ring) 5892 { 5893 struct amdgpu_device *adev = ring->adev; 5894 5895 if (ring->use_doorbell) { 5896 /* XXX check if swapping is necessary on BE */ 5897 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5898 ring->wptr); 5899 WDOORBELL64(ring->doorbell_index, ring->wptr); 5900 } else { 5901 WREG32_SOC15(GC, 0, regCP_RB0_WPTR, 5902 lower_32_bits(ring->wptr)); 5903 WREG32_SOC15(GC, 0, regCP_RB0_WPTR_HI, 5904 upper_32_bits(ring->wptr)); 5905 } 5906 } 5907 5908 static u64 gfx_v11_0_ring_get_rptr_compute(struct amdgpu_ring *ring) 5909 { 5910 /* gfx11 hardware is 32bit rptr */ 5911 return *(uint32_t *)ring->rptr_cpu_addr; 5912 } 5913 5914 static u64 gfx_v11_0_ring_get_wptr_compute(struct amdgpu_ring *ring) 5915 { 5916 u64 wptr; 5917 5918 /* XXX check if swapping is necessary on BE */ 5919 if (ring->use_doorbell) 5920 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 5921 else 5922 BUG(); 5923 return wptr; 5924 } 5925 5926 static void gfx_v11_0_ring_set_wptr_compute(struct amdgpu_ring *ring) 5927 { 5928 struct amdgpu_device *adev = ring->adev; 5929 5930 /* XXX check if swapping is necessary on BE */ 5931 if (ring->use_doorbell) { 5932 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 5933 ring->wptr); 5934 WDOORBELL64(ring->doorbell_index, ring->wptr); 5935 } else { 5936 BUG(); /* only DOORBELL method supported on gfx11 now */ 5937 } 5938 } 5939 5940 static void gfx_v11_0_ring_emit_hdp_flush(struct amdgpu_ring *ring) 5941 { 5942 struct amdgpu_device *adev = ring->adev; 5943 u32 ref_and_mask, reg_mem_engine; 5944 5945 if (!adev->gfx.funcs->get_hdp_flush_mask) { 5946 dev_err(adev->dev, "%s: gfx hdp flush is not supported.\n", __func__); 5947 return; 5948 } 5949 5950 adev->gfx.funcs->get_hdp_flush_mask(ring, &ref_and_mask, ®_mem_engine); 5951 gfx_v11_0_wait_reg_mem(ring, reg_mem_engine, 0, 1, 5952 adev->nbio.funcs->get_hdp_flush_req_offset(adev), 5953 adev->nbio.funcs->get_hdp_flush_done_offset(adev), 5954 ref_and_mask, ref_and_mask, 0x20); 5955 } 5956 5957 static void gfx_v11_0_ring_emit_ib_gfx(struct amdgpu_ring *ring, 5958 struct amdgpu_job *job, 5959 struct amdgpu_ib *ib, 5960 uint32_t flags) 5961 { 5962 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5963 u32 header, control = 0; 5964 5965 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2); 5966 5967 control |= ib->length_dw | (vmid << 24); 5968 5969 if (ring->adev->gfx.mcbp && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) { 5970 control |= INDIRECT_BUFFER_PRE_ENB(1); 5971 5972 if (flags & AMDGPU_IB_PREEMPTED) 5973 control |= INDIRECT_BUFFER_PRE_RESUME(1); 5974 5975 if (vmid && !ring->adev->gfx.rs64_enable) 5976 gfx_v11_0_ring_emit_de_meta(ring, 5977 !amdgpu_sriov_vf(ring->adev) && (flags & AMDGPU_IB_PREEMPTED)); 5978 } 5979 5980 amdgpu_ring_write(ring, header); 5981 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 5982 amdgpu_ring_write(ring, 5983 #ifdef __BIG_ENDIAN 5984 (2 << 0) | 5985 #endif 5986 lower_32_bits(ib->gpu_addr)); 5987 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 5988 amdgpu_ring_write(ring, control); 5989 } 5990 5991 static void gfx_v11_0_ring_emit_ib_compute(struct amdgpu_ring *ring, 5992 struct amdgpu_job *job, 5993 struct amdgpu_ib *ib, 5994 uint32_t flags) 5995 { 5996 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 5997 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24); 5998 5999 /* Currently, there is a high possibility to get wave ID mismatch 6000 * between ME and GDS, leading to a hw deadlock, because ME generates 6001 * different wave IDs than the GDS expects. This situation happens 6002 * randomly when at least 5 compute pipes use GDS ordered append. 6003 * The wave IDs generated by ME are also wrong after suspend/resume. 6004 * Those are probably bugs somewhere else in the kernel driver. 6005 * 6006 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 6007 * GDS to 0 for this ring (me/pipe). 6008 */ 6009 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 6010 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 6011 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID); 6012 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id); 6013 } 6014 6015 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 6016 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 6017 amdgpu_ring_write(ring, 6018 #ifdef __BIG_ENDIAN 6019 (2 << 0) | 6020 #endif 6021 lower_32_bits(ib->gpu_addr)); 6022 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 6023 amdgpu_ring_write(ring, control); 6024 } 6025 6026 static void gfx_v11_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 6027 u64 seq, unsigned flags) 6028 { 6029 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 6030 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 6031 6032 /* RELEASE_MEM - flush caches, send int */ 6033 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 6034 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ | 6035 PACKET3_RELEASE_MEM_GCR_GL2_WB | 6036 PACKET3_RELEASE_MEM_GCR_GLM_INV | /* must be set with GLM_WB */ 6037 PACKET3_RELEASE_MEM_GCR_GLM_WB | 6038 PACKET3_RELEASE_MEM_CACHE_POLICY(3) | 6039 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 6040 PACKET3_RELEASE_MEM_EVENT_INDEX(5))); 6041 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) | 6042 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0))); 6043 6044 /* 6045 * the address should be Qword aligned if 64bit write, Dword 6046 * aligned if only send 32bit data low (discard data high) 6047 */ 6048 if (write64bit) 6049 WARN_ON(addr & 0x7); 6050 else 6051 WARN_ON(addr & 0x3); 6052 amdgpu_ring_write(ring, lower_32_bits(addr)); 6053 amdgpu_ring_write(ring, upper_32_bits(addr)); 6054 amdgpu_ring_write(ring, lower_32_bits(seq)); 6055 amdgpu_ring_write(ring, upper_32_bits(seq)); 6056 amdgpu_ring_write(ring, 0); 6057 } 6058 6059 static void gfx_v11_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 6060 { 6061 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6062 uint32_t seq = ring->fence_drv.sync_seq; 6063 uint64_t addr = ring->fence_drv.gpu_addr; 6064 6065 gfx_v11_0_wait_reg_mem(ring, usepfp, 1, 0, lower_32_bits(addr), 6066 upper_32_bits(addr), seq, 0xffffffff, 4); 6067 } 6068 6069 static void gfx_v11_0_ring_invalidate_tlbs(struct amdgpu_ring *ring, 6070 uint16_t pasid, uint32_t flush_type, 6071 bool all_hub, uint8_t dst_sel) 6072 { 6073 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 6074 amdgpu_ring_write(ring, 6075 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) | 6076 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 6077 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 6078 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 6079 } 6080 6081 static void gfx_v11_0_ring_emit_vm_flush(struct amdgpu_ring *ring, 6082 unsigned vmid, uint64_t pd_addr) 6083 { 6084 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 6085 6086 /* compute doesn't have PFP */ 6087 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 6088 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 6089 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 6090 amdgpu_ring_write(ring, 0x0); 6091 } 6092 6093 /* Make sure that we can't skip the SET_Q_MODE packets when the VM 6094 * changed in any way. 6095 */ 6096 ring->set_q_mode_offs = 0; 6097 ring->set_q_mode_ptr = NULL; 6098 } 6099 6100 static void gfx_v11_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 6101 u64 seq, unsigned int flags) 6102 { 6103 struct amdgpu_device *adev = ring->adev; 6104 6105 /* write fence seq to the "addr" */ 6106 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6107 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6108 WRITE_DATA_DST_SEL(5) | WR_CONFIRM)); 6109 amdgpu_ring_write(ring, lower_32_bits(addr)); 6110 amdgpu_ring_write(ring, upper_32_bits(addr)); 6111 amdgpu_ring_write(ring, lower_32_bits(seq)); 6112 6113 if (flags & AMDGPU_FENCE_FLAG_INT) { 6114 /* set register to trigger INT */ 6115 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6116 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) | 6117 WRITE_DATA_DST_SEL(0) | WR_CONFIRM)); 6118 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, regCPC_INT_STATUS)); 6119 amdgpu_ring_write(ring, 0); 6120 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 6121 } 6122 } 6123 6124 static void gfx_v11_0_ring_emit_cntxcntl(struct amdgpu_ring *ring, 6125 uint32_t flags) 6126 { 6127 uint32_t dw2 = 0; 6128 6129 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */ 6130 if (flags & AMDGPU_HAVE_CTX_SWITCH) { 6131 /* set load_global_config & load_global_uconfig */ 6132 dw2 |= 0x8001; 6133 /* set load_cs_sh_regs */ 6134 dw2 |= 0x01000000; 6135 /* set load_per_context_state & load_gfx_sh_regs for GFX */ 6136 dw2 |= 0x10002; 6137 } 6138 6139 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1)); 6140 amdgpu_ring_write(ring, dw2); 6141 amdgpu_ring_write(ring, 0); 6142 } 6143 6144 static unsigned gfx_v11_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring, 6145 uint64_t addr) 6146 { 6147 unsigned ret; 6148 6149 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3)); 6150 amdgpu_ring_write(ring, lower_32_bits(addr)); 6151 amdgpu_ring_write(ring, upper_32_bits(addr)); 6152 /* discard following DWs if *cond_exec_gpu_addr==0 */ 6153 amdgpu_ring_write(ring, 0); 6154 ret = ring->wptr & ring->buf_mask; 6155 /* patch dummy value later */ 6156 amdgpu_ring_write(ring, 0); 6157 6158 return ret; 6159 } 6160 6161 static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring, 6162 u64 shadow_va, u64 csa_va, 6163 u64 gds_va, bool init_shadow, 6164 int vmid) 6165 { 6166 struct amdgpu_device *adev = ring->adev; 6167 unsigned int offs, end; 6168 6169 if (!adev->gfx.cp_gfx_shadow || !ring->ring_obj) 6170 return; 6171 6172 /* 6173 * The logic here isn't easy to understand because we need to keep state 6174 * accross multiple executions of the function as well as between the 6175 * CPU and GPU. The general idea is that the newly written GPU command 6176 * has a condition on the previous one and only executed if really 6177 * necessary. 6178 */ 6179 6180 /* 6181 * The dw in the NOP controls if the next SET_Q_MODE packet should be 6182 * executed or not. Reserve 64bits just to be on the save side. 6183 */ 6184 amdgpu_ring_write(ring, PACKET3(PACKET3_NOP, 1)); 6185 offs = ring->wptr & ring->buf_mask; 6186 6187 /* 6188 * We start with skipping the prefix SET_Q_MODE and always executing 6189 * the postfix SET_Q_MODE packet. This is changed below with a 6190 * WRITE_DATA command when the postfix executed. 6191 */ 6192 amdgpu_ring_write(ring, shadow_va ? 1 : 0); 6193 amdgpu_ring_write(ring, 0); 6194 6195 if (ring->set_q_mode_offs) { 6196 uint64_t addr; 6197 6198 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6199 addr += ring->set_q_mode_offs << 2; 6200 end = gfx_v11_0_ring_emit_init_cond_exec(ring, addr); 6201 } 6202 6203 /* 6204 * When the postfix SET_Q_MODE packet executes we need to make sure that the 6205 * next prefix SET_Q_MODE packet executes as well. 6206 */ 6207 if (!shadow_va) { 6208 uint64_t addr; 6209 6210 addr = amdgpu_bo_gpu_offset(ring->ring_obj); 6211 addr += offs << 2; 6212 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6213 amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM); 6214 amdgpu_ring_write(ring, lower_32_bits(addr)); 6215 amdgpu_ring_write(ring, upper_32_bits(addr)); 6216 amdgpu_ring_write(ring, 0x1); 6217 } 6218 6219 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_Q_PREEMPTION_MODE, 7)); 6220 amdgpu_ring_write(ring, lower_32_bits(shadow_va)); 6221 amdgpu_ring_write(ring, upper_32_bits(shadow_va)); 6222 amdgpu_ring_write(ring, lower_32_bits(gds_va)); 6223 amdgpu_ring_write(ring, upper_32_bits(gds_va)); 6224 amdgpu_ring_write(ring, lower_32_bits(csa_va)); 6225 amdgpu_ring_write(ring, upper_32_bits(csa_va)); 6226 amdgpu_ring_write(ring, shadow_va ? 6227 PACKET3_SET_Q_PREEMPTION_MODE_IB_VMID(vmid) : 0); 6228 amdgpu_ring_write(ring, init_shadow ? 6229 PACKET3_SET_Q_PREEMPTION_MODE_INIT_SHADOW_MEM : 0); 6230 6231 if (ring->set_q_mode_offs) 6232 amdgpu_ring_patch_cond_exec(ring, end); 6233 6234 if (shadow_va) { 6235 uint64_t token = shadow_va ^ csa_va ^ gds_va ^ vmid; 6236 6237 /* 6238 * If the tokens match try to skip the last postfix SET_Q_MODE 6239 * packet to avoid saving/restoring the state all the time. 6240 */ 6241 if (ring->set_q_mode_ptr && ring->set_q_mode_token == token) 6242 *ring->set_q_mode_ptr = 0; 6243 6244 ring->set_q_mode_token = token; 6245 } else { 6246 ring->set_q_mode_ptr = &ring->ring[ring->set_q_mode_offs]; 6247 } 6248 6249 ring->set_q_mode_offs = offs; 6250 } 6251 6252 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume) 6253 { 6254 struct amdgpu_device *adev = ring->adev; 6255 struct v10_de_ib_state de_payload = {0}; 6256 uint64_t offset, gds_addr, de_payload_gpu_addr; 6257 void *de_payload_cpu_addr; 6258 int cnt; 6259 6260 offset = offsetof(struct v10_gfx_meta_data, de_payload); 6261 de_payload_gpu_addr = amdgpu_csa_vaddr(ring->adev) + offset; 6262 de_payload_cpu_addr = adev->virt.csa_cpu_addr + offset; 6263 6264 gds_addr = ALIGN(amdgpu_csa_vaddr(ring->adev) + 6265 AMDGPU_CSA_SIZE - adev->gds.gds_size, 6266 PAGE_SIZE); 6267 6268 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr); 6269 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr); 6270 6271 cnt = (sizeof(de_payload) >> 2) + 4 - 2; 6272 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt)); 6273 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) | 6274 WRITE_DATA_DST_SEL(8) | 6275 WR_CONFIRM) | 6276 WRITE_DATA_CACHE_POLICY(0)); 6277 amdgpu_ring_write(ring, lower_32_bits(de_payload_gpu_addr)); 6278 amdgpu_ring_write(ring, upper_32_bits(de_payload_gpu_addr)); 6279 6280 if (resume) 6281 amdgpu_ring_write_multiple(ring, de_payload_cpu_addr, 6282 sizeof(de_payload) >> 2); 6283 else 6284 amdgpu_ring_write_multiple(ring, (void *)&de_payload, 6285 sizeof(de_payload) >> 2); 6286 } 6287 6288 static void gfx_v11_0_ring_emit_frame_cntl(struct amdgpu_ring *ring, bool start, 6289 bool secure) 6290 { 6291 uint32_t v = secure ? FRAME_TMZ : 0; 6292 6293 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0)); 6294 amdgpu_ring_write(ring, v | FRAME_CMD(start ? 0 : 1)); 6295 } 6296 6297 static void gfx_v11_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 6298 uint32_t reg_val_offs) 6299 { 6300 struct amdgpu_device *adev = ring->adev; 6301 6302 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 6303 amdgpu_ring_write(ring, 0 | /* src: register*/ 6304 (5 << 8) | /* dst: memory */ 6305 (1 << 20)); /* write confirm */ 6306 amdgpu_ring_write(ring, reg); 6307 amdgpu_ring_write(ring, 0); 6308 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 6309 reg_val_offs * 4)); 6310 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 6311 reg_val_offs * 4)); 6312 } 6313 6314 static void gfx_v11_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 6315 uint32_t val) 6316 { 6317 uint32_t cmd = 0; 6318 6319 switch (ring->funcs->type) { 6320 case AMDGPU_RING_TYPE_GFX: 6321 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM; 6322 break; 6323 case AMDGPU_RING_TYPE_KIQ: 6324 cmd = (1 << 16); /* no inc addr */ 6325 break; 6326 default: 6327 cmd = WR_CONFIRM; 6328 break; 6329 } 6330 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 6331 amdgpu_ring_write(ring, cmd); 6332 amdgpu_ring_write(ring, reg); 6333 amdgpu_ring_write(ring, 0); 6334 amdgpu_ring_write(ring, val); 6335 } 6336 6337 static void gfx_v11_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 6338 uint32_t val, uint32_t mask) 6339 { 6340 gfx_v11_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 6341 } 6342 6343 static void gfx_v11_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 6344 uint32_t reg0, uint32_t reg1, 6345 uint32_t ref, uint32_t mask) 6346 { 6347 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX); 6348 6349 gfx_v11_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1, 6350 ref, mask, 0x20); 6351 } 6352 6353 static void 6354 gfx_v11_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev, 6355 uint32_t me, uint32_t pipe, 6356 enum amdgpu_interrupt_state state) 6357 { 6358 uint32_t cp_int_cntl, cp_int_cntl_reg; 6359 6360 if (!me) { 6361 switch (pipe) { 6362 case 0: 6363 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING0); 6364 break; 6365 case 1: 6366 cp_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_INT_CNTL_RING1); 6367 break; 6368 default: 6369 DRM_DEBUG("invalid pipe %d\n", pipe); 6370 return; 6371 } 6372 } else { 6373 DRM_DEBUG("invalid me %d\n", me); 6374 return; 6375 } 6376 6377 switch (state) { 6378 case AMDGPU_IRQ_STATE_DISABLE: 6379 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6380 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6381 TIME_STAMP_INT_ENABLE, 0); 6382 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6383 GENERIC0_INT_ENABLE, 0); 6384 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6385 break; 6386 case AMDGPU_IRQ_STATE_ENABLE: 6387 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6388 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6389 TIME_STAMP_INT_ENABLE, 1); 6390 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6391 GENERIC0_INT_ENABLE, 1); 6392 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6393 break; 6394 default: 6395 break; 6396 } 6397 } 6398 6399 static void gfx_v11_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 6400 int me, int pipe, 6401 enum amdgpu_interrupt_state state) 6402 { 6403 u32 mec_int_cntl, mec_int_cntl_reg; 6404 6405 /* 6406 * amdgpu controls only the first MEC. That's why this function only 6407 * handles the setting of interrupts for this specific MEC. All other 6408 * pipes' interrupts are set by amdkfd. 6409 */ 6410 6411 if (me == 1) { 6412 switch (pipe) { 6413 case 0: 6414 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6415 break; 6416 case 1: 6417 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE1_INT_CNTL); 6418 break; 6419 case 2: 6420 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE2_INT_CNTL); 6421 break; 6422 case 3: 6423 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE3_INT_CNTL); 6424 break; 6425 default: 6426 DRM_DEBUG("invalid pipe %d\n", pipe); 6427 return; 6428 } 6429 } else { 6430 DRM_DEBUG("invalid me %d\n", me); 6431 return; 6432 } 6433 6434 switch (state) { 6435 case AMDGPU_IRQ_STATE_DISABLE: 6436 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6437 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6438 TIME_STAMP_INT_ENABLE, 0); 6439 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6440 GENERIC0_INT_ENABLE, 0); 6441 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6442 break; 6443 case AMDGPU_IRQ_STATE_ENABLE: 6444 mec_int_cntl = RREG32_SOC15_IP(GC, mec_int_cntl_reg); 6445 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6446 TIME_STAMP_INT_ENABLE, 1); 6447 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6448 GENERIC0_INT_ENABLE, 1); 6449 WREG32_SOC15_IP(GC, mec_int_cntl_reg, mec_int_cntl); 6450 break; 6451 default: 6452 break; 6453 } 6454 } 6455 6456 static int gfx_v11_0_set_eop_interrupt_state(struct amdgpu_device *adev, 6457 struct amdgpu_irq_src *src, 6458 unsigned type, 6459 enum amdgpu_interrupt_state state) 6460 { 6461 switch (type) { 6462 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP: 6463 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 0, state); 6464 break; 6465 case AMDGPU_CP_IRQ_GFX_ME0_PIPE1_EOP: 6466 gfx_v11_0_set_gfx_eop_interrupt_state(adev, 0, 1, state); 6467 break; 6468 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 6469 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 0, state); 6470 break; 6471 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 6472 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 1, state); 6473 break; 6474 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 6475 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 2, state); 6476 break; 6477 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 6478 gfx_v11_0_set_compute_eop_interrupt_state(adev, 1, 3, state); 6479 break; 6480 default: 6481 break; 6482 } 6483 return 0; 6484 } 6485 6486 static int gfx_v11_0_eop_irq(struct amdgpu_device *adev, 6487 struct amdgpu_irq_src *source, 6488 struct amdgpu_iv_entry *entry) 6489 { 6490 u32 doorbell_offset = entry->src_data[0]; 6491 6492 DRM_DEBUG("IH: CP EOP\n"); 6493 6494 if (!adev->gfx.disable_kq) { 6495 u8 me_id = (entry->ring_id & 0x0c) >> 2; 6496 u8 pipe_id = (entry->ring_id & 0x03) >> 0; 6497 u8 queue_id = (entry->ring_id & 0x70) >> 4; 6498 struct amdgpu_ring *ring; 6499 int i; 6500 6501 switch (me_id) { 6502 case 0: 6503 /* 6504 * MES splits gfx HQDs per (me,pipe): KGQ owns queue=0, 6505 * userq gfx owns queue>=1 (see amdgpu_mes_get_hqd_mask). 6506 * Require a strict (me,pipe,queue) match so userq gfx 6507 * EOPs fall through to amdgpu_userq_process_fence_irq(). 6508 */ 6509 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6510 ring = &adev->gfx.gfx_ring[i]; 6511 if ((ring->me == me_id) && 6512 (ring->pipe == pipe_id) && 6513 (ring->queue == queue_id)) { 6514 amdgpu_fence_process(ring); 6515 return 0; 6516 } 6517 } 6518 break; 6519 case 1: 6520 case 2: 6521 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6522 ring = &adev->gfx.compute_ring[i]; 6523 /* Per-queue interrupt is supported for MEC starting from VI. 6524 * The interrupt can only be enabled/disabled per pipe instead 6525 * of per queue. 6526 */ 6527 if ((ring->me == me_id) && 6528 (ring->pipe == pipe_id) && 6529 (ring->queue == queue_id)) { 6530 amdgpu_fence_process(ring); 6531 return 0; 6532 } 6533 } 6534 break; 6535 default: 6536 break; 6537 } 6538 } 6539 6540 if (adev->enable_mes && doorbell_offset) 6541 amdgpu_userq_process_fence_irq(adev, doorbell_offset); 6542 6543 return 0; 6544 } 6545 6546 static int gfx_v11_0_set_priv_reg_fault_state(struct amdgpu_device *adev, 6547 struct amdgpu_irq_src *source, 6548 unsigned int type, 6549 enum amdgpu_interrupt_state state) 6550 { 6551 u32 cp_int_cntl_reg, cp_int_cntl; 6552 int i, j; 6553 6554 switch (state) { 6555 case AMDGPU_IRQ_STATE_DISABLE: 6556 case AMDGPU_IRQ_STATE_ENABLE: 6557 for (i = 0; i < adev->gfx.me.num_me; i++) { 6558 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6559 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6560 6561 if (cp_int_cntl_reg) { 6562 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6563 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6564 PRIV_REG_INT_ENABLE, 6565 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6566 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6567 } 6568 } 6569 } 6570 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6571 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6572 /* MECs start at 1 */ 6573 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6574 6575 if (cp_int_cntl_reg) { 6576 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6577 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6578 PRIV_REG_INT_ENABLE, 6579 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6580 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6581 } 6582 } 6583 } 6584 break; 6585 default: 6586 break; 6587 } 6588 6589 return 0; 6590 } 6591 6592 static int gfx_v11_0_set_bad_op_fault_state(struct amdgpu_device *adev, 6593 struct amdgpu_irq_src *source, 6594 unsigned type, 6595 enum amdgpu_interrupt_state state) 6596 { 6597 u32 cp_int_cntl_reg, cp_int_cntl; 6598 int i, j; 6599 6600 switch (state) { 6601 case AMDGPU_IRQ_STATE_DISABLE: 6602 case AMDGPU_IRQ_STATE_ENABLE: 6603 for (i = 0; i < adev->gfx.me.num_me; i++) { 6604 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6605 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6606 6607 if (cp_int_cntl_reg) { 6608 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6609 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6610 OPCODE_ERROR_INT_ENABLE, 6611 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6612 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6613 } 6614 } 6615 } 6616 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6617 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6618 /* MECs start at 1 */ 6619 cp_int_cntl_reg = gfx_v11_0_get_cpc_int_cntl(adev, i + 1, j); 6620 6621 if (cp_int_cntl_reg) { 6622 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6623 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_ME1_PIPE0_INT_CNTL, 6624 OPCODE_ERROR_INT_ENABLE, 6625 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6626 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6627 } 6628 } 6629 } 6630 break; 6631 default: 6632 break; 6633 } 6634 return 0; 6635 } 6636 6637 static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev, 6638 struct amdgpu_irq_src *source, 6639 unsigned int type, 6640 enum amdgpu_interrupt_state state) 6641 { 6642 u32 cp_int_cntl_reg, cp_int_cntl; 6643 int i, j; 6644 6645 switch (state) { 6646 case AMDGPU_IRQ_STATE_DISABLE: 6647 case AMDGPU_IRQ_STATE_ENABLE: 6648 for (i = 0; i < adev->gfx.me.num_me; i++) { 6649 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6650 cp_int_cntl_reg = gfx_v11_0_get_cpg_int_cntl(adev, i, j); 6651 6652 if (cp_int_cntl_reg) { 6653 cp_int_cntl = RREG32_SOC15_IP(GC, cp_int_cntl_reg); 6654 cp_int_cntl = REG_SET_FIELD(cp_int_cntl, CP_INT_CNTL_RING0, 6655 PRIV_INSTR_INT_ENABLE, 6656 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 6657 WREG32_SOC15_IP(GC, cp_int_cntl_reg, cp_int_cntl); 6658 } 6659 } 6660 } 6661 break; 6662 default: 6663 break; 6664 } 6665 6666 return 0; 6667 } 6668 6669 static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, 6670 struct amdgpu_iv_entry *entry) 6671 { 6672 u32 doorbell_offset = entry->src_data[0] & AMDGPU_CTXID0_DOORBELL_ID_MASK; 6673 6674 /* 6675 * Try KQ first by ring_id (HW slot is authoritative). The 6676 * KMD compute_hqd_mask contract guarantees KCQ and user queues 6677 * never share a HW slot. 6678 */ 6679 if (!adev->gfx.disable_kq) { 6680 u8 me_id = (entry->ring_id & 0x0c) >> 2; 6681 u8 pipe_id = (entry->ring_id & 0x03) >> 0; 6682 u8 queue_id = (entry->ring_id & 0x70) >> 4; 6683 struct amdgpu_ring *ring; 6684 int i; 6685 6686 switch (me_id) { 6687 case 0: 6688 for (i = 0; i < adev->gfx.num_gfx_rings; i++) { 6689 ring = &adev->gfx.gfx_ring[i]; 6690 if (ring->me == me_id && ring->pipe == pipe_id && 6691 ring->queue == queue_id) { 6692 drm_sched_fault(&ring->sched); 6693 return; 6694 } 6695 } 6696 break; 6697 case 1: 6698 case 2: 6699 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 6700 ring = &adev->gfx.compute_ring[i]; 6701 if (ring->me == me_id && ring->pipe == pipe_id && 6702 ring->queue == queue_id) { 6703 drm_sched_fault(&ring->sched); 6704 return; 6705 } 6706 } 6707 break; 6708 default: 6709 BUG(); 6710 break; 6711 } 6712 } 6713 6714 /* No KQ matched: HW slot is a MES-scheduled user queue. */ 6715 if (adev->enable_mes && doorbell_offset) 6716 amdgpu_userq_process_reset_irq(adev, entry->pasid, 6717 doorbell_offset); 6718 } 6719 6720 static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev, 6721 struct amdgpu_irq_src *source, 6722 struct amdgpu_iv_entry *entry) 6723 { 6724 DRM_ERROR("Illegal register access in command stream\n"); 6725 gfx_v11_0_handle_priv_fault(adev, entry); 6726 return 0; 6727 } 6728 6729 static int gfx_v11_0_bad_op_irq(struct amdgpu_device *adev, 6730 struct amdgpu_irq_src *source, 6731 struct amdgpu_iv_entry *entry) 6732 { 6733 DRM_ERROR("Illegal opcode in command stream\n"); 6734 gfx_v11_0_handle_priv_fault(adev, entry); 6735 return 0; 6736 } 6737 6738 static int gfx_v11_0_priv_inst_irq(struct amdgpu_device *adev, 6739 struct amdgpu_irq_src *source, 6740 struct amdgpu_iv_entry *entry) 6741 { 6742 DRM_ERROR("Illegal instruction in command stream\n"); 6743 gfx_v11_0_handle_priv_fault(adev, entry); 6744 return 0; 6745 } 6746 6747 static int gfx_v11_0_rlc_gc_fed_irq(struct amdgpu_device *adev, 6748 struct amdgpu_irq_src *source, 6749 struct amdgpu_iv_entry *entry) 6750 { 6751 if (adev->gfx.ras && adev->gfx.ras->rlc_gc_fed_irq) 6752 return adev->gfx.ras->rlc_gc_fed_irq(adev, source, entry); 6753 6754 return 0; 6755 } 6756 6757 #if 0 6758 static int gfx_v11_0_kiq_set_interrupt_state(struct amdgpu_device *adev, 6759 struct amdgpu_irq_src *src, 6760 unsigned int type, 6761 enum amdgpu_interrupt_state state) 6762 { 6763 uint32_t tmp, target; 6764 struct amdgpu_ring *ring = &(adev->gfx.kiq[0].ring); 6765 6766 target = SOC15_REG_OFFSET(GC, 0, regCP_ME1_PIPE0_INT_CNTL); 6767 target += ring->pipe; 6768 6769 switch (type) { 6770 case AMDGPU_CP_KIQ_IRQ_DRIVER0: 6771 if (state == AMDGPU_IRQ_STATE_DISABLE) { 6772 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6773 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6774 GENERIC2_INT_ENABLE, 0); 6775 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6776 6777 tmp = RREG32_SOC15_IP(GC, target); 6778 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6779 GENERIC2_INT_ENABLE, 0); 6780 WREG32_SOC15_IP(GC, target, tmp); 6781 } else { 6782 tmp = RREG32_SOC15(GC, 0, regCPC_INT_CNTL); 6783 tmp = REG_SET_FIELD(tmp, CPC_INT_CNTL, 6784 GENERIC2_INT_ENABLE, 1); 6785 WREG32_SOC15(GC, 0, regCPC_INT_CNTL, tmp); 6786 6787 tmp = RREG32_SOC15_IP(GC, target); 6788 tmp = REG_SET_FIELD(tmp, CP_ME1_PIPE0_INT_CNTL, 6789 GENERIC2_INT_ENABLE, 1); 6790 WREG32_SOC15_IP(GC, target, tmp); 6791 } 6792 break; 6793 default: 6794 BUG(); /* kiq only support GENERIC2_INT now */ 6795 break; 6796 } 6797 return 0; 6798 } 6799 #endif 6800 6801 static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring) 6802 { 6803 const unsigned int gcr_cntl = 6804 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) | 6805 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) | 6806 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_INV(1) | 6807 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLM_WB(1) | 6808 PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_INV(1) | 6809 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) | 6810 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) | 6811 PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1); 6812 6813 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ 6814 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6)); 6815 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */ 6816 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 6817 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 6818 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 6819 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 6820 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 6821 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ 6822 } 6823 6824 static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, 6825 unsigned int vmid, 6826 struct amdgpu_fence *timedout_fence) 6827 { 6828 struct amdgpu_device *adev = ring->adev; 6829 bool use_mmio = adev->gfx.me.use_mmio_for_reset; 6830 6831 return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio); 6832 } 6833 6834 static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, 6835 unsigned int vmid, 6836 struct amdgpu_fence *timedout_fence) 6837 { 6838 struct amdgpu_device *adev = ring->adev; 6839 6840 return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL); 6841 } 6842 6843 static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 6844 { 6845 struct amdgpu_device *adev = ip_block->adev; 6846 uint32_t i, j, k, reg, index = 0; 6847 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6848 6849 if (!adev->gfx.ip_dump_core) 6850 return; 6851 6852 for (i = 0; i < reg_count; i++) 6853 drm_printf(p, "%-50s \t 0x%08x\n", 6854 gc_reg_list_11_0[i].reg_name, 6855 adev->gfx.ip_dump_core[i]); 6856 6857 /* print compute queue registers for all instances */ 6858 if (!adev->gfx.ip_dump_compute_queues) 6859 return; 6860 6861 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6862 drm_printf(p, "\nnum_mec: %d num_pipe: %d num_queue: %d\n", 6863 adev->gfx.mec.num_mec, 6864 adev->gfx.mec.num_pipe_per_mec, 6865 adev->gfx.mec.num_queue_per_pipe); 6866 6867 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6868 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6869 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6870 drm_printf(p, "\nmec %d, pipe %d, queue %d\n", i, j, k); 6871 for (reg = 0; reg < reg_count; reg++) { 6872 if (i && gc_cp_reg_list_11[reg].reg_offset == regCP_MEC_ME1_HEADER_DUMP) 6873 drm_printf(p, "%-50s \t 0x%08x\n", 6874 "regCP_MEC_ME2_HEADER_DUMP", 6875 adev->gfx.ip_dump_compute_queues[index + reg]); 6876 else 6877 drm_printf(p, "%-50s \t 0x%08x\n", 6878 gc_cp_reg_list_11[reg].reg_name, 6879 adev->gfx.ip_dump_compute_queues[index + reg]); 6880 } 6881 index += reg_count; 6882 } 6883 } 6884 } 6885 6886 /* print gfx queue registers for all instances */ 6887 if (!adev->gfx.ip_dump_gfx_queues) 6888 return; 6889 6890 index = 0; 6891 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 6892 drm_printf(p, "\nnum_me: %d num_pipe: %d num_queue: %d\n", 6893 adev->gfx.me.num_me, 6894 adev->gfx.me.num_pipe_per_me, 6895 adev->gfx.me.num_queue_per_pipe); 6896 6897 for (i = 0; i < adev->gfx.me.num_me; i++) { 6898 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6899 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 6900 drm_printf(p, "\nme %d, pipe %d, queue %d\n", i, j, k); 6901 for (reg = 0; reg < reg_count; reg++) { 6902 drm_printf(p, "%-50s \t 0x%08x\n", 6903 gc_gfx_queue_reg_list_11[reg].reg_name, 6904 adev->gfx.ip_dump_gfx_queues[index + reg]); 6905 } 6906 index += reg_count; 6907 } 6908 } 6909 } 6910 } 6911 6912 static void gfx_v11_ip_dump(struct amdgpu_ip_block *ip_block) 6913 { 6914 struct amdgpu_device *adev = ip_block->adev; 6915 uint32_t i, j, k, reg, index = 0; 6916 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_11_0); 6917 6918 if (!adev->gfx.ip_dump_core) 6919 return; 6920 6921 amdgpu_gfx_off_ctrl(adev, false); 6922 for (i = 0; i < reg_count; i++) 6923 adev->gfx.ip_dump_core[i] = RREG32(SOC15_REG_ENTRY_OFFSET(gc_reg_list_11_0[i])); 6924 amdgpu_gfx_off_ctrl(adev, true); 6925 6926 /* dump compute queue registers for all instances */ 6927 if (!adev->gfx.ip_dump_compute_queues) 6928 return; 6929 6930 reg_count = ARRAY_SIZE(gc_cp_reg_list_11); 6931 amdgpu_gfx_off_ctrl(adev, false); 6932 mutex_lock(&adev->srbm_mutex); 6933 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 6934 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 6935 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 6936 /* ME0 is for GFX so start from 1 for CP */ 6937 soc21_grbm_select(adev, adev->gfx.me.num_me + i, j, k, 0); 6938 for (reg = 0; reg < reg_count; reg++) { 6939 if (i && 6940 gc_cp_reg_list_11[reg].reg_offset == 6941 regCP_MEC_ME1_HEADER_DUMP) 6942 adev->gfx.ip_dump_compute_queues[index + reg] = 6943 RREG32(SOC15_REG_OFFSET(GC, 0, 6944 regCP_MEC_ME2_HEADER_DUMP)); 6945 else 6946 adev->gfx.ip_dump_compute_queues[index + reg] = 6947 RREG32(SOC15_REG_ENTRY_OFFSET( 6948 gc_cp_reg_list_11[reg])); 6949 } 6950 index += reg_count; 6951 } 6952 } 6953 } 6954 soc21_grbm_select(adev, 0, 0, 0, 0); 6955 mutex_unlock(&adev->srbm_mutex); 6956 amdgpu_gfx_off_ctrl(adev, true); 6957 6958 /* dump gfx queue registers for all instances */ 6959 if (!adev->gfx.ip_dump_gfx_queues) 6960 return; 6961 6962 index = 0; 6963 reg_count = ARRAY_SIZE(gc_gfx_queue_reg_list_11); 6964 amdgpu_gfx_off_ctrl(adev, false); 6965 mutex_lock(&adev->srbm_mutex); 6966 for (i = 0; i < adev->gfx.me.num_me; i++) { 6967 for (j = 0; j < adev->gfx.me.num_pipe_per_me; j++) { 6968 for (k = 0; k < adev->gfx.me.num_queue_per_pipe; k++) { 6969 soc21_grbm_select(adev, i, j, k, 0); 6970 6971 for (reg = 0; reg < reg_count; reg++) { 6972 adev->gfx.ip_dump_gfx_queues[index + reg] = 6973 RREG32(SOC15_REG_ENTRY_OFFSET( 6974 gc_gfx_queue_reg_list_11[reg])); 6975 } 6976 index += reg_count; 6977 } 6978 } 6979 } 6980 soc21_grbm_select(adev, 0, 0, 0, 0); 6981 mutex_unlock(&adev->srbm_mutex); 6982 amdgpu_gfx_off_ctrl(adev, true); 6983 } 6984 6985 static void gfx_v11_0_ring_emit_cleaner_shader(struct amdgpu_ring *ring) 6986 { 6987 /* Emit the cleaner shader */ 6988 amdgpu_ring_write(ring, PACKET3(PACKET3_RUN_CLEANER_SHADER, 0)); 6989 amdgpu_ring_write(ring, 0); /* RESERVED field, programmed to zero */ 6990 } 6991 6992 static void gfx_v11_0_ring_begin_use(struct amdgpu_ring *ring) 6993 { 6994 amdgpu_gfx_profile_ring_begin_use(ring); 6995 6996 amdgpu_gfx_enforce_isolation_ring_begin_use(ring); 6997 } 6998 6999 static void gfx_v11_0_ring_end_use(struct amdgpu_ring *ring) 7000 { 7001 amdgpu_gfx_profile_ring_end_use(ring); 7002 7003 amdgpu_gfx_enforce_isolation_ring_end_use(ring); 7004 } 7005 7006 static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { 7007 .name = "gfx_v11_0", 7008 .early_init = gfx_v11_0_early_init, 7009 .sw_init = gfx_v11_0_sw_init, 7010 .sw_fini = gfx_v11_0_sw_fini, 7011 .hw_init = gfx_v11_0_hw_init, 7012 .hw_fini = gfx_v11_0_hw_fini, 7013 .suspend = gfx_v11_0_suspend, 7014 .resume = gfx_v11_0_resume, 7015 .is_idle = gfx_v11_0_is_idle, 7016 .wait_for_idle = gfx_v11_0_wait_for_idle, 7017 .soft_reset = gfx_v11_0_soft_reset, 7018 .set_clockgating_state = gfx_v11_0_set_clockgating_state, 7019 .set_powergating_state = gfx_v11_0_set_powergating_state, 7020 .get_clockgating_state = gfx_v11_0_get_clockgating_state, 7021 .dump_ip_state = gfx_v11_ip_dump, 7022 .print_ip_state = gfx_v11_ip_print, 7023 }; 7024 7025 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = { 7026 .type = AMDGPU_RING_TYPE_GFX, 7027 .align_mask = 0xff, 7028 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7029 .support_64bit_ptrs = true, 7030 .secure_submission_supported = true, 7031 .get_rptr = gfx_v11_0_ring_get_rptr_gfx, 7032 .get_wptr = gfx_v11_0_ring_get_wptr_gfx, 7033 .set_wptr = gfx_v11_0_ring_set_wptr_gfx, 7034 .emit_frame_size = /* totally 247 maximum if 16 IBs */ 7035 5 + /* update_spm_vmid */ 7036 5 + /* COND_EXEC */ 7037 22 + /* SET_Q_PREEMPTION_MODE */ 7038 7 + /* PIPELINE_SYNC */ 7039 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7040 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7041 4 + /* VM_FLUSH */ 7042 8 + /* FENCE for VM_FLUSH */ 7043 20 + /* GDS switch */ 7044 5 + /* COND_EXEC */ 7045 7 + /* HDP_flush */ 7046 4 + /* VGT_flush */ 7047 31 + /* DE_META */ 7048 3 + /* CNTX_CTRL */ 7049 5 + /* HDP_INVL */ 7050 22 + /* SET_Q_PREEMPTION_MODE */ 7051 8 + 8 + /* FENCE x2 */ 7052 8 + /* gfx_v11_0_emit_mem_sync */ 7053 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7054 .emit_ib_size = 4, /* gfx_v11_0_ring_emit_ib_gfx */ 7055 .emit_ib = gfx_v11_0_ring_emit_ib_gfx, 7056 .emit_fence = gfx_v11_0_ring_emit_fence, 7057 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7058 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7059 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7060 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7061 .test_ring = gfx_v11_0_ring_test_ring, 7062 .test_ib = gfx_v11_0_ring_test_ib, 7063 .insert_nop = gfx_v11_ring_insert_nop, 7064 .pad_ib = amdgpu_ring_generic_pad_ib, 7065 .emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl, 7066 .emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow, 7067 .init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec, 7068 .preempt_ib = amdgpu_gfx_ring_preempt_ib, 7069 .emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl, 7070 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7071 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7072 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7073 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7074 .reset = gfx_v11_0_reset_kgq, 7075 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7076 .begin_use = gfx_v11_0_ring_begin_use, 7077 .end_use = gfx_v11_0_ring_end_use, 7078 }; 7079 7080 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_compute = { 7081 .type = AMDGPU_RING_TYPE_COMPUTE, 7082 .align_mask = 0xff, 7083 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7084 .support_64bit_ptrs = true, 7085 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7086 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7087 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7088 .emit_frame_size = 7089 5 + /* update_spm_vmid */ 7090 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7091 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7092 5 + /* hdp invalidate */ 7093 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7094 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7095 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7096 2 + /* gfx_v11_0_ring_emit_vm_flush */ 7097 8 + 8 + 8 + /* gfx_v11_0_ring_emit_fence x3 for user fence, vm fence */ 7098 8 + /* gfx_v11_0_emit_mem_sync */ 7099 2, /* gfx_v11_0_ring_emit_cleaner_shader */ 7100 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7101 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7102 .emit_fence = gfx_v11_0_ring_emit_fence, 7103 .emit_pipeline_sync = gfx_v11_0_ring_emit_pipeline_sync, 7104 .emit_vm_flush = gfx_v11_0_ring_emit_vm_flush, 7105 .emit_gds_switch = gfx_v11_0_ring_emit_gds_switch, 7106 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7107 .test_ring = gfx_v11_0_ring_test_ring, 7108 .test_ib = gfx_v11_0_ring_test_ib, 7109 .insert_nop = gfx_v11_ring_insert_nop, 7110 .pad_ib = amdgpu_ring_generic_pad_ib, 7111 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7112 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7113 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7114 .emit_mem_sync = gfx_v11_0_emit_mem_sync, 7115 .reset = gfx_v11_0_reset_kcq, 7116 .emit_cleaner_shader = gfx_v11_0_ring_emit_cleaner_shader, 7117 .begin_use = gfx_v11_0_ring_begin_use, 7118 .end_use = gfx_v11_0_ring_end_use, 7119 }; 7120 7121 static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_kiq = { 7122 .type = AMDGPU_RING_TYPE_KIQ, 7123 .align_mask = 0xff, 7124 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 7125 .support_64bit_ptrs = true, 7126 .get_rptr = gfx_v11_0_ring_get_rptr_compute, 7127 .get_wptr = gfx_v11_0_ring_get_wptr_compute, 7128 .set_wptr = gfx_v11_0_ring_set_wptr_compute, 7129 .emit_frame_size = 7130 20 + /* gfx_v11_0_ring_emit_gds_switch */ 7131 7 + /* gfx_v11_0_ring_emit_hdp_flush */ 7132 5 + /*hdp invalidate */ 7133 7 + /* gfx_v11_0_ring_emit_pipeline_sync */ 7134 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 7135 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 7136 8 + 8 + 8, /* gfx_v11_0_ring_emit_fence_kiq x3 for user fence, vm fence */ 7137 .emit_ib_size = 7, /* gfx_v11_0_ring_emit_ib_compute */ 7138 .emit_ib = gfx_v11_0_ring_emit_ib_compute, 7139 .emit_fence = gfx_v11_0_ring_emit_fence_kiq, 7140 .test_ring = gfx_v11_0_ring_test_ring, 7141 .test_ib = gfx_v11_0_ring_test_ib, 7142 .insert_nop = amdgpu_ring_insert_nop, 7143 .pad_ib = amdgpu_ring_generic_pad_ib, 7144 .emit_rreg = gfx_v11_0_ring_emit_rreg, 7145 .emit_wreg = gfx_v11_0_ring_emit_wreg, 7146 .emit_reg_wait = gfx_v11_0_ring_emit_reg_wait, 7147 .emit_reg_write_reg_wait = gfx_v11_0_ring_emit_reg_write_reg_wait, 7148 .emit_hdp_flush = gfx_v11_0_ring_emit_hdp_flush, 7149 }; 7150 7151 static void gfx_v11_0_set_ring_funcs(struct amdgpu_device *adev) 7152 { 7153 int i; 7154 7155 adev->gfx.kiq[0].ring.funcs = &gfx_v11_0_ring_funcs_kiq; 7156 7157 for (i = 0; i < adev->gfx.num_gfx_rings; i++) 7158 adev->gfx.gfx_ring[i].funcs = &gfx_v11_0_ring_funcs_gfx; 7159 7160 for (i = 0; i < adev->gfx.num_compute_rings; i++) 7161 adev->gfx.compute_ring[i].funcs = &gfx_v11_0_ring_funcs_compute; 7162 } 7163 7164 static const struct amdgpu_irq_src_funcs gfx_v11_0_eop_irq_funcs = { 7165 .set = gfx_v11_0_set_eop_interrupt_state, 7166 .process = gfx_v11_0_eop_irq, 7167 }; 7168 7169 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_reg_irq_funcs = { 7170 .set = gfx_v11_0_set_priv_reg_fault_state, 7171 .process = gfx_v11_0_priv_reg_irq, 7172 }; 7173 7174 static const struct amdgpu_irq_src_funcs gfx_v11_0_bad_op_irq_funcs = { 7175 .set = gfx_v11_0_set_bad_op_fault_state, 7176 .process = gfx_v11_0_bad_op_irq, 7177 }; 7178 7179 static const struct amdgpu_irq_src_funcs gfx_v11_0_priv_inst_irq_funcs = { 7180 .set = gfx_v11_0_set_priv_inst_fault_state, 7181 .process = gfx_v11_0_priv_inst_irq, 7182 }; 7183 7184 static const struct amdgpu_irq_src_funcs gfx_v11_0_rlc_gc_fed_irq_funcs = { 7185 .process = gfx_v11_0_rlc_gc_fed_irq, 7186 }; 7187 7188 static void gfx_v11_0_set_irq_funcs(struct amdgpu_device *adev) 7189 { 7190 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 7191 adev->gfx.eop_irq.funcs = &gfx_v11_0_eop_irq_funcs; 7192 7193 adev->gfx.priv_reg_irq.num_types = 1; 7194 adev->gfx.priv_reg_irq.funcs = &gfx_v11_0_priv_reg_irq_funcs; 7195 7196 adev->gfx.bad_op_irq.num_types = 1; 7197 adev->gfx.bad_op_irq.funcs = &gfx_v11_0_bad_op_irq_funcs; 7198 7199 adev->gfx.priv_inst_irq.num_types = 1; 7200 adev->gfx.priv_inst_irq.funcs = &gfx_v11_0_priv_inst_irq_funcs; 7201 7202 adev->gfx.rlc_gc_fed_irq.num_types = 1; /* 0x80 FED error */ 7203 adev->gfx.rlc_gc_fed_irq.funcs = &gfx_v11_0_rlc_gc_fed_irq_funcs; 7204 7205 } 7206 7207 static void gfx_v11_0_set_imu_funcs(struct amdgpu_device *adev) 7208 { 7209 if (adev->flags & AMD_IS_APU) 7210 adev->gfx.imu.mode = MISSION_MODE; 7211 else 7212 adev->gfx.imu.mode = DEBUG_MODE; 7213 7214 adev->gfx.imu.funcs = &gfx_v11_0_imu_funcs; 7215 } 7216 7217 static void gfx_v11_0_set_rlc_funcs(struct amdgpu_device *adev) 7218 { 7219 adev->gfx.rlc.funcs = &gfx_v11_0_rlc_funcs; 7220 } 7221 7222 static void gfx_v11_0_set_gds_init(struct amdgpu_device *adev) 7223 { 7224 unsigned total_cu = adev->gfx.config.max_cu_per_sh * 7225 adev->gfx.config.max_sh_per_se * 7226 adev->gfx.config.max_shader_engines; 7227 7228 adev->gds.gds_size = 0x1000; 7229 adev->gds.gds_compute_max_wave_id = total_cu * 32 - 1; 7230 adev->gds.gws_size = 64; 7231 adev->gds.oa_size = 16; 7232 } 7233 7234 static void gfx_v11_0_set_mqd_funcs(struct amdgpu_device *adev) 7235 { 7236 /* set gfx eng mqd */ 7237 adev->mqds[AMDGPU_HW_IP_GFX].mqd_size = 7238 sizeof(struct v11_gfx_mqd); 7239 adev->mqds[AMDGPU_HW_IP_GFX].init_mqd = 7240 gfx_v11_0_gfx_mqd_init; 7241 /* set compute eng mqd */ 7242 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size = 7243 sizeof(struct v11_compute_mqd); 7244 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd = 7245 gfx_v11_0_compute_mqd_init; 7246 } 7247 7248 static void gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh(struct amdgpu_device *adev, 7249 u32 bitmap) 7250 { 7251 u32 data; 7252 7253 if (!bitmap) 7254 return; 7255 7256 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7257 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7258 7259 WREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG, data); 7260 } 7261 7262 static u32 gfx_v11_0_get_wgp_active_bitmap_per_sh(struct amdgpu_device *adev) 7263 { 7264 u32 data, wgp_bitmask; 7265 data = RREG32_SOC15(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG); 7266 data |= RREG32_SOC15(GC, 0, regGC_USER_SHADER_ARRAY_CONFIG); 7267 7268 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 7269 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 7270 7271 wgp_bitmask = 7272 amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh >> 1); 7273 7274 return (~data) & wgp_bitmask; 7275 } 7276 7277 static u32 gfx_v11_0_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev) 7278 { 7279 u32 wgp_idx, wgp_active_bitmap; 7280 u32 cu_bitmap_per_wgp, cu_active_bitmap; 7281 7282 wgp_active_bitmap = gfx_v11_0_get_wgp_active_bitmap_per_sh(adev); 7283 cu_active_bitmap = 0; 7284 7285 for (wgp_idx = 0; wgp_idx < 16; wgp_idx++) { 7286 /* if there is one WGP enabled, it means 2 CUs will be enabled */ 7287 cu_bitmap_per_wgp = 3 << (2 * wgp_idx); 7288 if (wgp_active_bitmap & (1 << wgp_idx)) 7289 cu_active_bitmap |= cu_bitmap_per_wgp; 7290 } 7291 7292 return cu_active_bitmap; 7293 } 7294 7295 static int gfx_v11_0_get_cu_info(struct amdgpu_device *adev, 7296 struct amdgpu_cu_info *cu_info) 7297 { 7298 int i, j, k, counter, active_cu_number = 0; 7299 u32 mask, bitmap; 7300 unsigned disable_masks[8 * 2]; 7301 7302 if (!adev || !cu_info) 7303 return -EINVAL; 7304 7305 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 8, 2); 7306 7307 mutex_lock(&adev->grbm_idx_mutex); 7308 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 7309 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 7310 bitmap = i * adev->gfx.config.max_sh_per_se + j; 7311 if (!((gfx_v11_0_get_sa_active_bitmap(adev) >> bitmap) & 1)) 7312 continue; 7313 mask = 1; 7314 counter = 0; 7315 gfx_v11_0_select_se_sh(adev, i, j, 0xffffffff, 0); 7316 if (i < 8 && j < 2) 7317 gfx_v11_0_set_user_wgp_inactive_bitmap_per_sh( 7318 adev, disable_masks[i * 2 + j]); 7319 bitmap = gfx_v11_0_get_cu_active_bitmap_per_sh(adev); 7320 7321 /** 7322 * GFX11 could support more than 4 SEs, while the bitmap 7323 * in cu_info struct is 4x4 and ioctl interface struct 7324 * drm_amdgpu_info_device should keep stable. 7325 * So we use last two columns of bitmap to store cu mask for 7326 * SEs 4 to 7, the layout of the bitmap is as below: 7327 * SE0: {SH0,SH1} --> {bitmap[0][0], bitmap[0][1]} 7328 * SE1: {SH0,SH1} --> {bitmap[1][0], bitmap[1][1]} 7329 * SE2: {SH0,SH1} --> {bitmap[2][0], bitmap[2][1]} 7330 * SE3: {SH0,SH1} --> {bitmap[3][0], bitmap[3][1]} 7331 * SE4: {SH0,SH1} --> {bitmap[0][2], bitmap[0][3]} 7332 * SE5: {SH0,SH1} --> {bitmap[1][2], bitmap[1][3]} 7333 * SE6: {SH0,SH1} --> {bitmap[2][2], bitmap[2][3]} 7334 * SE7: {SH0,SH1} --> {bitmap[3][2], bitmap[3][3]} 7335 */ 7336 cu_info->bitmap[0][i % 4][j + (i / 4) * 2] = bitmap; 7337 7338 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { 7339 if (bitmap & mask) 7340 counter++; 7341 7342 mask <<= 1; 7343 } 7344 active_cu_number += counter; 7345 } 7346 } 7347 gfx_v11_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0); 7348 mutex_unlock(&adev->grbm_idx_mutex); 7349 7350 cu_info->number = active_cu_number; 7351 cu_info->simd_per_cu = NUM_SIMD_PER_CU; 7352 7353 return 0; 7354 } 7355 7356 const struct amdgpu_ip_block_version gfx_v11_0_ip_block = 7357 { 7358 .type = AMD_IP_BLOCK_TYPE_GFX, 7359 .major = 11, 7360 .minor = 0, 7361 .rev = 0, 7362 .funcs = &gfx_v11_0_ip_funcs, 7363 }; 7364