1 /* 2 * Copyright 2025 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/delay.h> 24 #include <linux/kernel.h> 25 #include <linux/firmware.h> 26 #include <linux/module.h> 27 #include <linux/pci.h> 28 #include "amdgpu.h" 29 #include "amdgpu_gfx.h" 30 #include "amdgpu_psp.h" 31 #include "amdgpu_smu.h" 32 #include "amdgpu_atomfirmware.h" 33 #include "amdgpu_userq_fence.h" 34 #include "imu_v12_1.h" 35 #include "soc_v1_0.h" 36 #include "gfx_v12_1_pkt.h" 37 38 #include "gc/gc_12_1_0_offset.h" 39 #include "gc/gc_12_1_0_sh_mask.h" 40 #include "soc24_enum.h" 41 #include "ivsrcid/gfx/irqsrcs_gfx_12_1_0.h" 42 43 #include "soc15.h" 44 #include "clearstate_gfx12.h" 45 #include "v12_structs.h" 46 #include "gfx_v12_1.h" 47 #include "mes_v12_1.h" 48 #include "amdgpu_ras_mgr.h" 49 50 #define GFX12_MEC_HPD_SIZE 2048 51 #define NUM_SIMD_PER_CU_GFX12_1 4 52 53 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L 54 55 #define regCP_HQD_EOP_CONTROL_DEFAULT 0x00000000 56 #define regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT 0x00000000 57 #define regCP_MQD_CONTROL_DEFAULT 0x00000100 58 #define regCP_HQD_PQ_CONTROL_DEFAULT 0x00308509 59 #define regCP_HQD_PQ_RPTR_DEFAULT 0x00000000 60 #define regCP_HQD_PERSISTENT_STATE_DEFAULT 0x0ae06301 61 #define regCP_HQD_IB_CONTROL_DEFAULT 0x00100000 62 63 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin"); 64 MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc_1.bin"); 65 66 #define SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 0x00000001 67 #define DEFAULT_SH_MEM_CONFIG \ 68 ((SH_MEM_ADDRESS_MODE_64 << SH_MEM_CONFIG__ADDRESS_MODE__SHIFT) | \ 69 (SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0 << SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT) | \ 70 (3 << SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT)) 71 72 73 static const struct amdgpu_hwip_reg_entry gc_reg_list_12_1[] = { 74 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS), 75 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS2), 76 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS3), 77 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT1), 78 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT2), 79 SOC15_REG_ENTRY_STR(GC, 0, regCP_STALLED_STAT3), 80 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STALLED_STAT1), 81 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STALLED_STAT1), 82 SOC15_REG_ENTRY_STR(GC, 0, regCP_BUSY_STAT), 83 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT), 84 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT), 85 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_BUSY_STAT2), 86 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_BUSY_STAT2), 87 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPF_STATUS), 88 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_ERROR), 89 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_HPD_STATUS0), 90 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_BASE), 91 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_RPTR), 92 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB_WPTR), 93 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_BASE), 94 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_RPTR), 95 SOC15_REG_ENTRY_STR(GC, 0, regCP_RB0_WPTR), 96 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_CMD_BUFSZ), 97 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_CMD_BUFSZ), 98 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_LO), 99 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BASE_HI), 100 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB1_BUFSZ), 101 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_LO), 102 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BASE_HI), 103 SOC15_REG_ENTRY_STR(GC, 0, regCP_IB2_BUFSZ), 104 SOC15_REG_ENTRY_STR(GC, 0, regCPF_UTCL1_STATUS), 105 SOC15_REG_ENTRY_STR(GC, 0, regCPC_UTCL1_STATUS), 106 SOC15_REG_ENTRY_STR(GC, 0, regCPG_UTCL1_STATUS), 107 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS), 108 SOC15_REG_ENTRY_STR(GC, 0, regIA_UTCL1_STATUS_2), 109 SOC15_REG_ENTRY_STR(GC, 0, regPA_CL_CNTL_STATUS), 110 SOC15_REG_ENTRY_STR(GC, 0, regRMI_UTCL1_STATUS), 111 SOC15_REG_ENTRY_STR(GC, 0, regSQC_CACHES), 112 SOC15_REG_ENTRY_STR(GC, 0, regSQG_STATUS), 113 SOC15_REG_ENTRY_STR(GC, 0, regWD_UTCL1_STATUS), 114 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2), 115 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_LO32), 116 SOC15_REG_ENTRY_STR(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS_HI32), 117 SOC15_REG_ENTRY_STR(GC, 0, regCP_DEBUG), 118 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_CNTL), 119 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_CNTL), 120 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_INSTR_PNTR), 121 SOC15_REG_ENTRY_STR(GC, 0, regCP_ME_INSTR_PNTR), 122 SOC15_REG_ENTRY_STR(GC, 0, regCP_PFP_INSTR_PNTR), 123 SOC15_REG_ENTRY_STR(GC, 0, regCP_CPC_STATUS), 124 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR0), 125 SOC15_REG_ENTRY_STR(GC, 0, regCP_GFX_RS64_INSTR_PNTR1), 126 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_RS64_INSTR_PNTR), 127 /* cp header registers */ 128 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 129 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 130 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 131 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 132 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 133 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 134 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 135 SOC15_REG_ENTRY_STR(GC, 0, regCP_MES_HEADER_DUMP), 136 /* SE status registers */ 137 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE0), 138 SOC15_REG_ENTRY_STR(GC, 0, regGRBM_STATUS_SE1), 139 }; 140 141 static const struct amdgpu_hwip_reg_entry gc_cp_reg_list_12_1[] = { 142 /* compute registers */ 143 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_VMID), 144 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PERSISTENT_STATE), 145 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PIPE_PRIORITY), 146 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUEUE_PRIORITY), 147 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_QUANTUM), 148 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE), 149 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_BASE_HI), 150 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_RPTR), 151 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR), 152 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_POLL_ADDR_HI), 153 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_DOORBELL_CONTROL), 154 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_CONTROL), 155 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR), 156 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_BASE_ADDR_HI), 157 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_RPTR), 158 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_IB_CONTROL), 159 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_REQUEST), 160 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR), 161 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_BASE_ADDR_HI), 162 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_CONTROL), 163 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_RPTR), 164 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR), 165 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_EVENTS), 166 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_LO), 167 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_BASE_ADDR_HI), 168 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_CONTROL), 169 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_OFFSET), 170 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CNTL_STACK_SIZE), 171 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_WG_STATE_OFFSET), 172 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_CTX_SAVE_SIZE), 173 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_GDS_RESOURCE_STATE), 174 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_ERROR), 175 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_EOP_WPTR_MEM), 176 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_LO), 177 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_PQ_WPTR_HI), 178 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_OFFSET), 179 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT), 180 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_SUSPEND_WG_STATE_OFFSET), 181 SOC15_REG_ENTRY_STR(GC, 0, regCP_HQD_DEQUEUE_STATUS), 182 /* cp header registers */ 183 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 184 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 185 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 186 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 187 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 188 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 189 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 190 SOC15_REG_ENTRY_STR(GC, 0, regCP_MEC_ME1_HEADER_DUMP), 191 }; 192 193 static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev, int xcc_id); 194 static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev); 195 static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev); 196 static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev); 197 static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev); 198 static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev); 199 static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev, 200 struct amdgpu_cu_info *cu_info); 201 static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev); 202 static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num, 203 u32 sh_num, u32 instance, int xcc_id); 204 static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 205 uint32_t val); 206 static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev); 207 static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring, 208 uint16_t pasid, uint32_t flush_type, 209 bool all_hub, uint8_t dst_sel); 210 static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev, int xcc_id); 211 static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev, int xcc_id); 212 static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev, 213 bool enable); 214 static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev, 215 bool enable, int xcc_id); 216 static int gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device *adev); 217 218 static void gfx_v12_1_kiq_set_resources(struct amdgpu_ring *kiq_ring, 219 uint64_t queue_mask) 220 { 221 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6)); 222 amdgpu_ring_write(kiq_ring, PACKET3_SET_RESOURCES_VMID_MASK(0) | 223 PACKET3_SET_RESOURCES_QUEUE_TYPE(0)); /* vmid_mask:0 queue_type:0 (KIQ) */ 224 amdgpu_ring_write(kiq_ring, lower_32_bits(queue_mask)); /* queue mask lo */ 225 amdgpu_ring_write(kiq_ring, upper_32_bits(queue_mask)); /* queue mask hi */ 226 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */ 227 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */ 228 amdgpu_ring_write(kiq_ring, 0); /* oac mask */ 229 amdgpu_ring_write(kiq_ring, 0); 230 } 231 232 static void gfx_v12_1_kiq_map_queues(struct amdgpu_ring *kiq_ring, 233 struct amdgpu_ring *ring) 234 { 235 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); 236 uint64_t wptr_addr = ring->wptr_gpu_addr; 237 uint32_t me = 0, eng_sel = 0; 238 239 switch (ring->funcs->type) { 240 case AMDGPU_RING_TYPE_COMPUTE: 241 me = 1; 242 eng_sel = 0; 243 break; 244 case AMDGPU_RING_TYPE_MES: 245 me = 2; 246 eng_sel = 5; 247 break; 248 default: 249 WARN_ON(1); 250 } 251 252 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5)); 253 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/ 254 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 255 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */ 256 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */ 257 PACKET3_MAP_QUEUES_QUEUE(ring->queue) | 258 PACKET3_MAP_QUEUES_PIPE(ring->pipe) | 259 PACKET3_MAP_QUEUES_ME((me)) | 260 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) | /*queue_type: normal compute queue */ 261 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) | 262 PACKET3_MAP_QUEUES_NUM_QUEUES(1)); /* num_queues: must be 1 */ 263 amdgpu_ring_write(kiq_ring, PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index)); 264 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr)); 265 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr)); 266 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr)); 267 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr)); 268 } 269 270 static void gfx_v12_1_kiq_unmap_queues(struct amdgpu_ring *kiq_ring, 271 struct amdgpu_ring *ring, 272 enum amdgpu_unmap_queues_action action, 273 u64 gpu_addr, u64 seq) 274 { 275 struct amdgpu_device *adev = kiq_ring->adev; 276 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 277 278 if (adev->enable_mes && !adev->gfx.kiq[0].ring.sched.ready) { 279 amdgpu_mes_unmap_legacy_queue(adev, ring, action, gpu_addr, 280 seq, kiq_ring->xcc_id); 281 return; 282 } 283 284 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4)); 285 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 286 PACKET3_UNMAP_QUEUES_ACTION(action) | 287 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) | 288 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) | 289 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1)); 290 amdgpu_ring_write(kiq_ring, 291 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index)); 292 293 if (action == PREEMPT_QUEUES_NO_UNMAP) { 294 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr)); 295 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr)); 296 amdgpu_ring_write(kiq_ring, seq); 297 } else { 298 amdgpu_ring_write(kiq_ring, 0); 299 amdgpu_ring_write(kiq_ring, 0); 300 amdgpu_ring_write(kiq_ring, 0); 301 } 302 } 303 304 static void gfx_v12_1_kiq_query_status(struct amdgpu_ring *kiq_ring, 305 struct amdgpu_ring *ring, 306 u64 addr, u64 seq) 307 { 308 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0; 309 310 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5)); 311 amdgpu_ring_write(kiq_ring, 312 PACKET3_QUERY_STATUS_CONTEXT_ID(0) | 313 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) | 314 PACKET3_QUERY_STATUS_COMMAND(2)); 315 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */ 316 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) | 317 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel)); 318 amdgpu_ring_write(kiq_ring, lower_32_bits(addr)); 319 amdgpu_ring_write(kiq_ring, upper_32_bits(addr)); 320 amdgpu_ring_write(kiq_ring, lower_32_bits(seq)); 321 amdgpu_ring_write(kiq_ring, upper_32_bits(seq)); 322 } 323 324 static void gfx_v12_1_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring, 325 uint16_t pasid, 326 uint32_t flush_type, 327 bool all_hub) 328 { 329 gfx_v12_1_ring_invalidate_tlbs(kiq_ring, pasid, flush_type, all_hub, 1); 330 } 331 332 static const struct kiq_pm4_funcs gfx_v12_1_kiq_pm4_funcs = { 333 .kiq_set_resources = gfx_v12_1_kiq_set_resources, 334 .kiq_map_queues = gfx_v12_1_kiq_map_queues, 335 .kiq_unmap_queues = gfx_v12_1_kiq_unmap_queues, 336 .kiq_query_status = gfx_v12_1_kiq_query_status, 337 .kiq_invalidate_tlbs = gfx_v12_1_kiq_invalidate_tlbs, 338 .set_resources_size = 8, 339 .map_queues_size = 7, 340 .unmap_queues_size = 6, 341 .query_status_size = 7, 342 .invalidate_tlbs_size = 2, 343 }; 344 345 static void gfx_v12_1_set_kiq_pm4_funcs(struct amdgpu_device *adev) 346 { 347 int i, num_xcc; 348 349 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 350 for (i =0; i < num_xcc; i++) 351 adev->gfx.kiq[i].pmf = &gfx_v12_1_kiq_pm4_funcs; 352 } 353 354 static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel, 355 int mem_space, int opt, uint32_t addr0, 356 uint32_t addr1, uint32_t ref, 357 uint32_t mask, uint32_t inv) 358 { 359 if (mem_space == 0) { 360 addr0 = soc_v1_0_normalize_xcc_reg_offset(addr0); 361 addr1 = soc_v1_0_normalize_xcc_reg_offset(addr1); 362 } 363 364 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5)); 365 amdgpu_ring_write(ring, 366 /* memory (1) or register (0) */ 367 (PACKET3_WAIT_REG_MEM__MEM_SPACE(mem_space) | 368 PACKET3_WAIT_REG_MEM__OPERATION(opt) | /* wait */ 369 PACKET3_WAIT_REG_MEM__FUNCTION(3))); /* equal */ 370 371 if (mem_space) 372 WARN_ON(addr0 & 0x3); /* Dword align */ 373 amdgpu_ring_write(ring, addr0); 374 amdgpu_ring_write(ring, addr1); 375 amdgpu_ring_write(ring, ref); 376 amdgpu_ring_write(ring, mask); 377 amdgpu_ring_write(ring, inv); /* poll interval */ 378 } 379 380 static int gfx_v12_1_ring_test_ring(struct amdgpu_ring *ring) 381 { 382 struct amdgpu_device *adev = ring->adev; 383 uint32_t scratch_reg0_offset, xcc_offset; 384 uint32_t tmp = 0; 385 unsigned i; 386 int r; 387 388 /* Use register offset which is local to XCC in the packet */ 389 xcc_offset = SOC15_REG_OFFSET(GC, 0, regSCRATCH_REG0); 390 scratch_reg0_offset = SOC15_REG_OFFSET(GC, GET_INST(GC, ring->xcc_id), regSCRATCH_REG0); 391 WREG32(scratch_reg0_offset, 0xCAFEDEAD); 392 tmp = RREG32(scratch_reg0_offset); 393 394 r = amdgpu_ring_alloc(ring, 5); 395 if (r) { 396 dev_err(adev->dev, 397 "amdgpu: cp failed to lock ring %d (%d).\n", 398 ring->idx, r); 399 return r; 400 } 401 402 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ) { 403 gfx_v12_1_ring_emit_wreg(ring, xcc_offset, 0xDEADBEEF); 404 } else { 405 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1)); 406 amdgpu_ring_write(ring, xcc_offset - 407 PACKET3_SET_UCONFIG_REG_START); 408 amdgpu_ring_write(ring, 0xDEADBEEF); 409 } 410 amdgpu_ring_commit(ring); 411 412 for (i = 0; i < adev->usec_timeout; i++) { 413 tmp = RREG32(scratch_reg0_offset); 414 if (tmp == 0xDEADBEEF) 415 break; 416 if (amdgpu_emu_mode == 1) 417 msleep(1); 418 else 419 udelay(1); 420 } 421 422 if (i >= adev->usec_timeout) 423 r = -ETIMEDOUT; 424 return r; 425 } 426 427 static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout) 428 { 429 struct amdgpu_device *adev = ring->adev; 430 struct amdgpu_ib ib; 431 struct dma_fence *f = NULL; 432 unsigned index; 433 uint64_t gpu_addr; 434 volatile uint32_t *cpu_ptr; 435 long r; 436 437 /* MES KIQ fw hasn't indirect buffer support for now */ 438 if (adev->enable_mes_kiq && 439 ring->funcs->type == AMDGPU_RING_TYPE_KIQ) 440 return 0; 441 442 memset(&ib, 0, sizeof(ib)); 443 444 r = amdgpu_wb_get(adev, &index); 445 if (r) 446 return r; 447 448 gpu_addr = adev->wb.gpu_addr + (index * 4); 449 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD); 450 cpu_ptr = &adev->wb.wb[index]; 451 452 r = amdgpu_ib_get(adev, NULL, 16, AMDGPU_IB_POOL_DIRECT, &ib); 453 if (r) { 454 dev_err(adev->dev, "amdgpu: failed to get ib (%ld).\n", r); 455 goto err1; 456 } 457 458 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3); 459 ib.ptr[1] = PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1); 460 ib.ptr[2] = lower_32_bits(gpu_addr); 461 ib.ptr[3] = upper_32_bits(gpu_addr); 462 ib.ptr[4] = 0xDEADBEEF; 463 ib.length_dw = 5; 464 465 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f); 466 if (r) 467 goto err2; 468 469 r = dma_fence_wait_timeout(f, false, timeout); 470 if (r == 0) { 471 r = -ETIMEDOUT; 472 goto err2; 473 } else if (r < 0) { 474 goto err2; 475 } 476 477 if (le32_to_cpu(*cpu_ptr) == 0xDEADBEEF) 478 r = 0; 479 else 480 r = -EINVAL; 481 err2: 482 amdgpu_ib_free(&ib, NULL); 483 dma_fence_put(f); 484 err1: 485 amdgpu_wb_free(adev, index); 486 return r; 487 } 488 489 static void gfx_v12_1_free_microcode(struct amdgpu_device *adev) 490 { 491 amdgpu_ucode_release(&adev->gfx.rlc_fw); 492 amdgpu_ucode_release(&adev->gfx.mec_fw); 493 494 kfree(adev->gfx.rlc.register_list_format); 495 } 496 497 static int gfx_v12_1_init_toc_microcode(struct amdgpu_device *adev, const char *ucode_prefix) 498 { 499 const struct psp_firmware_header_v1_0 *toc_hdr; 500 int err = 0; 501 502 err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, 503 AMDGPU_UCODE_REQUIRED, 504 "amdgpu/%s_toc.bin", ucode_prefix); 505 if (err) 506 goto out; 507 508 toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data; 509 adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version); 510 adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version); 511 adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes); 512 adev->psp.toc.start_addr = (uint8_t *)toc_hdr + 513 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes); 514 return 0; 515 out: 516 amdgpu_ucode_release(&adev->psp.toc_fw); 517 return err; 518 } 519 520 static int gfx_v12_1_init_microcode(struct amdgpu_device *adev) 521 { 522 char ucode_prefix[15]; 523 int err; 524 const struct rlc_firmware_header_v2_0 *rlc_hdr; 525 uint16_t version_major; 526 uint16_t version_minor; 527 528 DRM_DEBUG("\n"); 529 530 amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix)); 531 532 if (!amdgpu_sriov_vf(adev)) { 533 if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) && 534 adev->rev_id == 0) 535 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 536 AMDGPU_UCODE_REQUIRED, 537 "amdgpu/%s_rlc_1.bin", ucode_prefix); 538 else 539 err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw, 540 AMDGPU_UCODE_REQUIRED, 541 "amdgpu/%s_rlc.bin", ucode_prefix); 542 if (err) 543 goto out; 544 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 545 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 546 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 547 err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor); 548 if (err) 549 goto out; 550 } 551 552 err = amdgpu_ucode_request(adev, &adev->gfx.mec_fw, 553 AMDGPU_UCODE_REQUIRED, 554 "amdgpu/%s_mec.bin", ucode_prefix); 555 if (err) 556 goto out; 557 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC); 558 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK); 559 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK); 560 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK); 561 amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK); 562 563 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 564 err = gfx_v12_1_init_toc_microcode(adev, ucode_prefix); 565 566 /* only one MEC for gfx 12 */ 567 adev->gfx.mec2_fw = NULL; 568 569 if (adev->gfx.imu.funcs) { 570 if (adev->gfx.imu.funcs->init_microcode) { 571 err = adev->gfx.imu.funcs->init_microcode(adev); 572 if (err) 573 dev_err(adev->dev, "Failed to load imu firmware!\n"); 574 } 575 } 576 577 out: 578 if (err) { 579 amdgpu_ucode_release(&adev->gfx.rlc_fw); 580 amdgpu_ucode_release(&adev->gfx.mec_fw); 581 } 582 583 return err; 584 } 585 586 static u32 gfx_v12_1_get_csb_size(struct amdgpu_device *adev) 587 { 588 u32 count = 0; 589 const struct cs_section_def *sect = NULL; 590 const struct cs_extent_def *ext = NULL; 591 592 count += 1; 593 594 for (sect = gfx12_cs_data; sect->section != NULL; ++sect) { 595 if (sect->id == SECT_CONTEXT) { 596 for (ext = sect->section; ext->extent != NULL; ++ext) 597 count += 2 + ext->reg_count; 598 } else 599 return 0; 600 } 601 602 return count; 603 } 604 605 static void gfx_v12_1_get_csb_buffer(struct amdgpu_device *adev, u32 *buffer) 606 { 607 u32 count = 0, clustercount = 0, i; 608 const struct cs_section_def *sect = NULL; 609 const struct cs_extent_def *ext = NULL; 610 611 if (adev->gfx.rlc.cs_data == NULL) 612 return; 613 if (buffer == NULL) 614 return; 615 616 count += 1; 617 618 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) { 619 if (sect->id == SECT_CONTEXT) { 620 for (ext = sect->section; ext->extent != NULL; ++ext) { 621 clustercount++; 622 buffer[count++] = ext->reg_count; 623 buffer[count++] = ext->reg_index; 624 625 for (i = 0; i < ext->reg_count; i++) 626 buffer[count++] = cpu_to_le32(ext->extent[i]); 627 } 628 } else 629 return; 630 } 631 632 buffer[0] = clustercount; 633 } 634 635 static void gfx_v12_1_rlc_fini(struct amdgpu_device *adev) 636 { 637 /* clear state block */ 638 amdgpu_bo_free_kernel(&adev->gfx.rlc.clear_state_obj, 639 &adev->gfx.rlc.clear_state_gpu_addr, 640 (void **)&adev->gfx.rlc.cs_ptr); 641 642 /* jump table block */ 643 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj, 644 &adev->gfx.rlc.cp_table_gpu_addr, 645 (void **)&adev->gfx.rlc.cp_table_ptr); 646 } 647 648 static void gfx_v12_1_init_rlcg_reg_access_ctrl(struct amdgpu_device *adev) 649 { 650 int xcc_id, num_xcc; 651 struct amdgpu_rlcg_reg_access_ctrl *reg_access_ctrl; 652 653 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 654 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 655 reg_access_ctrl = &adev->gfx.rlc.reg_access_ctrl[GET_INST(GC, xcc_id)]; 656 657 reg_access_ctrl->grbm_cntl = 658 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_CNTL); 659 reg_access_ctrl->grbm_idx = 660 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX); 661 662 reg_access_ctrl->vfi_cmd = 663 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_CMD); 664 reg_access_ctrl->vfi_stat = 665 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_STAT); 666 reg_access_ctrl->vfi_addr = 667 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_ADDR); 668 reg_access_ctrl->vfi_data = 669 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_DATA); 670 reg_access_ctrl->vfi_grbm_cntl = 671 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_GRBM_GFX_CNTL); 672 reg_access_ctrl->vfi_grbm_idx = 673 SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_VFI_GRBM_GFX_INDEX); 674 reg_access_ctrl->vfi_grbm_cntl_data = 0; 675 reg_access_ctrl->vfi_grbm_idx_data = 0; 676 } 677 adev->gfx.rlc.rlcg_reg_access_supported = true; 678 } 679 680 static int gfx_v12_1_rlc_init(struct amdgpu_device *adev) 681 { 682 const struct cs_section_def *cs_data; 683 int r, i, num_xcc; 684 685 adev->gfx.rlc.cs_data = gfx12_cs_data; 686 687 cs_data = adev->gfx.rlc.cs_data; 688 689 if (cs_data) { 690 /* init clear state block */ 691 r = amdgpu_gfx_rlc_init_csb(adev); 692 if (r) 693 return r; 694 } 695 696 /* init spm vmid with 0xf */ 697 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 698 for (i = 0; i < num_xcc; i++) { 699 if (adev->gfx.rlc.funcs->update_spm_vmid) 700 adev->gfx.rlc.funcs->update_spm_vmid(adev, i, NULL, 0xf); 701 } 702 703 return 0; 704 } 705 706 static void gfx_v12_1_mec_fini(struct amdgpu_device *adev) 707 { 708 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL); 709 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL); 710 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_data_obj, NULL, NULL); 711 } 712 713 static int gfx_v12_1_mec_init(struct amdgpu_device *adev) 714 { 715 int r, i, num_xcc; 716 u32 *hpd; 717 size_t mec_hpd_size; 718 719 bitmap_zero(adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 720 721 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 722 for (i = 0; i < num_xcc; i++) 723 bitmap_zero(adev->gfx.mec_bitmap[i].queue_bitmap, 724 AMDGPU_MAX_COMPUTE_QUEUES); 725 726 /* take ownership of the relevant compute queues */ 727 amdgpu_gfx_compute_queue_acquire(adev); 728 mec_hpd_size = adev->gfx.num_compute_rings * 729 GFX12_MEC_HPD_SIZE * num_xcc; 730 731 if (mec_hpd_size) { 732 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE, 733 AMDGPU_GEM_DOMAIN_GTT, 734 &adev->gfx.mec.hpd_eop_obj, 735 &adev->gfx.mec.hpd_eop_gpu_addr, 736 (void **)&hpd); 737 if (r) { 738 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r); 739 gfx_v12_1_mec_fini(adev); 740 return r; 741 } 742 743 memset(hpd, 0, mec_hpd_size); 744 745 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj); 746 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj); 747 } 748 749 return 0; 750 } 751 752 static uint32_t wave_read_ind(struct amdgpu_device *adev, 753 uint32_t xcc_id, uint32_t wave, 754 uint32_t address) 755 { 756 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX, 757 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 758 (address << SQ_IND_INDEX__INDEX__SHIFT)); 759 return RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA); 760 } 761 762 static void wave_read_regs(struct amdgpu_device *adev, 763 uint32_t xcc_id, uint32_t wave, 764 uint32_t thread, uint32_t regno, 765 uint32_t num, uint32_t *out) 766 { 767 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_INDEX, 768 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) | 769 (regno << SQ_IND_INDEX__INDEX__SHIFT) | 770 (thread << SQ_IND_INDEX__WORKITEM_ID__SHIFT) | 771 (SQ_IND_INDEX__AUTO_INCR_MASK)); 772 while (num--) 773 *(out++) = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_IND_DATA); 774 } 775 776 static void gfx_v12_1_read_wave_data(struct amdgpu_device *adev, 777 uint32_t xcc_id, 778 uint32_t simd, uint32_t wave, 779 uint32_t *dst, int *no_fields) 780 { 781 /* in gfx12 the SIMD_ID is specified as part of the INSTANCE 782 * field when performing a select_se_sh so it should be 783 * zero here */ 784 WARN_ON(simd != 0); 785 786 /* type 4 wave data */ 787 dst[(*no_fields)++] = 4; 788 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATUS); 789 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_LO); 790 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_PC_HI); 791 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_LO); 792 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXEC_HI); 793 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID1); 794 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_HW_ID2); 795 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_GPR_ALLOC); 796 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_LDS_ALLOC); 797 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS); 798 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_STS2); 799 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_IB_DBG1); 800 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_M0); 801 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_MODE); 802 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_STATE_PRIV); 803 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_PRIV); 804 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_EXCP_FLAG_USER); 805 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_TRAP_CTRL); 806 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_ACTIVE); 807 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_VALID_AND_IDLE); 808 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_LO); 809 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_DVGPR_ALLOC_HI); 810 dst[(*no_fields)++] = wave_read_ind(adev, xcc_id, wave, ixSQ_WAVE_SCHED_MODE); 811 } 812 813 static void gfx_v12_1_read_wave_sgprs(struct amdgpu_device *adev, 814 uint32_t xcc_id, uint32_t simd, 815 uint32_t wave, uint32_t start, 816 uint32_t size, uint32_t *dst) 817 { 818 WARN_ON(simd != 0); 819 820 wave_read_regs(adev, xcc_id, wave, 0, 821 start + SQIND_WAVE_SGPRS_OFFSET, 822 size, dst); 823 } 824 825 static void gfx_v12_1_read_wave_vgprs(struct amdgpu_device *adev, 826 uint32_t xcc_id, uint32_t simd, 827 uint32_t wave, uint32_t thread, 828 uint32_t start, uint32_t size, 829 uint32_t *dst) 830 { 831 wave_read_regs(adev, xcc_id, wave, thread, 832 start + SQIND_WAVE_VGPRS_OFFSET, 833 size, dst); 834 } 835 836 static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev, 837 u32 me, u32 pipe, u32 q, u32 vm, u32 xcc_id) 838 { 839 soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id)); 840 } 841 842 #define regGFX_IMU_PARTITION_SWITCH 0x5f8c 843 #define regGFX_IMU_PARTITION_SWITCH_BASE_IDX 1 844 #define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP__SHIFT 0x2 845 #define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP_MASK 0x0000003CL 846 847 static int gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device *adev) 848 { 849 u32 reg_data; 850 851 /* the register data is expected to be the same on all instances */ 852 reg_data = RREG32_SOC15(GC, GET_INST(GC, 0), regGFX_IMU_PARTITION_SWITCH); 853 854 return REG_GET_FIELD(reg_data, GFX_IMU_PARTITION_SWITCH, TOTAL_XCCS_IN_XCP); 855 } 856 857 static int gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node) 858 { 859 int logic_xcc; 860 int xcc = (ih_node & 0x7) - 2 + (ih_node >> 3) * 4; 861 862 for (logic_xcc = 0; logic_xcc < NUM_XCC(adev->gfx.xcc_mask); logic_xcc++) { 863 if (xcc == GET_INST(GC, logic_xcc)) 864 return logic_xcc; 865 } 866 867 dev_err(adev->dev, "Couldn't find xcc mapping from IH node"); 868 return -EINVAL; 869 } 870 871 static const struct amdgpu_gfx_funcs gfx_v12_1_gfx_funcs = { 872 .get_gpu_clock_counter = &gfx_v12_1_get_gpu_clock_counter, 873 .select_se_sh = &gfx_v12_1_xcc_select_se_sh, 874 .read_wave_data = &gfx_v12_1_read_wave_data, 875 .read_wave_sgprs = &gfx_v12_1_read_wave_sgprs, 876 .read_wave_vgprs = &gfx_v12_1_read_wave_vgprs, 877 .select_me_pipe_q = &gfx_v12_1_select_me_pipe_q, 878 .update_perfmon_mgcg = &gfx_v12_1_update_perf_clk, 879 .get_xccs_per_xcp = &gfx_v12_1_get_xccs_per_xcp, 880 .ih_node_to_logical_xcc = &gfx_v12_1_ih_to_xcc_inst, 881 }; 882 883 static int gfx_v12_1_gpu_early_init(struct amdgpu_device *adev) 884 { 885 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 886 case IP_VERSION(12, 1, 0): 887 adev->gfx.config.max_hw_contexts = 8; 888 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20; 889 adev->gfx.config.sc_prim_fifo_size_backend = 0x100; 890 adev->gfx.config.sc_hiz_tile_fifo_size = 0; 891 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0; 892 break; 893 default: 894 dev_warn(adev->dev, "Unsupported GC version 0x%08x\n", 895 amdgpu_ip_version(adev, GC_HWIP, 0)); 896 return -EINVAL; 897 } 898 899 return 0; 900 } 901 902 static int gfx_v12_1_compute_ring_init(struct amdgpu_device *adev, int ring_id, 903 int xcc_id, int mec, int pipe, int queue) 904 { 905 int r; 906 unsigned irq_type; 907 struct amdgpu_ring *ring; 908 unsigned int hw_prio; 909 uint32_t xcc_doorbell_start; 910 911 ring = &adev->gfx.compute_ring[xcc_id * adev->gfx.num_compute_rings + 912 ring_id]; 913 914 /* mec0 is me1 */ 915 ring->xcc_id = xcc_id; 916 ring->me = mec + 1; 917 ring->pipe = pipe; 918 ring->queue = queue; 919 920 ring->ring_obj = NULL; 921 ring->use_doorbell = true; 922 xcc_doorbell_start = adev->doorbell_index.mec_ring0 + 923 xcc_id * adev->doorbell_index.xcc_doorbell_range; 924 ring->doorbell_index = (xcc_doorbell_start + ring_id) << 1; 925 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr + 926 (ring_id + xcc_id * adev->gfx.num_compute_rings) * 927 GFX12_MEC_HPD_SIZE; 928 ring->vm_hub = AMDGPU_GFXHUB(xcc_id); 929 sprintf(ring->name, "comp_%d.%d.%d.%d", 930 ring->xcc_id, ring->me, ring->pipe, ring->queue); 931 932 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 933 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec) 934 + ring->pipe; 935 hw_prio = amdgpu_gfx_is_high_priority_compute_queue(adev, ring) ? 936 AMDGPU_GFX_PIPE_PRIO_HIGH : AMDGPU_GFX_PIPE_PRIO_NORMAL; 937 /* type-2 packets are deprecated on MEC, use type-3 instead */ 938 r = amdgpu_ring_init(adev, ring, 1024, &adev->gfx.eop_irq, irq_type, 939 hw_prio, NULL); 940 if (r) 941 return r; 942 943 return 0; 944 } 945 946 static struct { 947 SOC24_FIRMWARE_ID id; 948 unsigned int offset; 949 unsigned int size; 950 unsigned int size_x16; 951 unsigned int num_inst; 952 } rlc_autoload_info[SOC24_FIRMWARE_ID_MAX]; 953 954 #define RLC_TOC_OFFSET_DWUNIT 8 955 #define RLC_SIZE_MULTIPLE 1024 956 #define RLC_TOC_UMF_SIZE_inM 23ULL 957 #define RLC_TOC_FORMAT_API 165ULL 958 959 #define RLC_NUM_INS_CODE0 1 960 #define RLC_NUM_INS_CODE1 8 961 #define RLC_NUM_INS_CODE2 2 962 #define RLC_NUM_INS_CODE3 16 963 964 static void gfx_v12_1_parse_rlc_toc(struct amdgpu_device *adev, void *rlc_toc) 965 { 966 RLC_TABLE_OF_CONTENT_V2 *ucode = rlc_toc; 967 968 while (ucode && (ucode->id > SOC24_FIRMWARE_ID_INVALID)) { 969 rlc_autoload_info[ucode->id].id = ucode->id; 970 rlc_autoload_info[ucode->id].offset = 971 ucode->offset * RLC_TOC_OFFSET_DWUNIT * 4; 972 rlc_autoload_info[ucode->id].size = 973 ucode->size_x16 ? ucode->size * RLC_SIZE_MULTIPLE * 4 : 974 ucode->size * 4; 975 switch (ucode->vfflr_image_code) { 976 case 0: 977 rlc_autoload_info[ucode->id].num_inst = 978 RLC_NUM_INS_CODE0; 979 break; 980 case 1: 981 rlc_autoload_info[ucode->id].num_inst = 982 RLC_NUM_INS_CODE1; 983 break; 984 case 2: 985 rlc_autoload_info[ucode->id].num_inst = 986 RLC_NUM_INS_CODE2; 987 break; 988 case 3: 989 rlc_autoload_info[ucode->id].num_inst = 990 RLC_NUM_INS_CODE3; 991 break; 992 default: 993 dev_err(adev->dev, 994 "Invalid Instance number detected\n"); 995 break; 996 } 997 ucode++; 998 } 999 } 1000 1001 static uint32_t gfx_v12_1_calc_toc_total_size(struct amdgpu_device *adev) 1002 { 1003 uint32_t total_size = 0; 1004 SOC24_FIRMWARE_ID id; 1005 1006 gfx_v12_1_parse_rlc_toc(adev, adev->psp.toc.start_addr); 1007 1008 for (id = SOC24_FIRMWARE_ID_RLC_G_UCODE; id < SOC24_FIRMWARE_ID_MAX; id++) 1009 total_size += rlc_autoload_info[id].size; 1010 1011 /* In case the offset in rlc toc ucode is aligned */ 1012 if (total_size < rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset) 1013 total_size = rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].offset + 1014 rlc_autoload_info[SOC24_FIRMWARE_ID_MAX-1].size; 1015 if (total_size < (RLC_TOC_UMF_SIZE_inM << 20)) 1016 total_size = RLC_TOC_UMF_SIZE_inM << 20; 1017 1018 return total_size; 1019 } 1020 1021 static int gfx_v12_1_rlc_autoload_buffer_init(struct amdgpu_device *adev) 1022 { 1023 int r; 1024 uint32_t total_size; 1025 1026 total_size = gfx_v12_1_calc_toc_total_size(adev); 1027 1028 r = amdgpu_bo_create_reserved(adev, total_size, 64 * 1024, 1029 AMDGPU_GEM_DOMAIN_VRAM, 1030 &adev->gfx.rlc.rlc_autoload_bo, 1031 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1032 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1033 1034 if (r) { 1035 dev_err(adev->dev, "(%d) failed to create fw autoload bo\n", r); 1036 return r; 1037 } 1038 1039 return 0; 1040 } 1041 1042 static void gfx_v12_1_rlc_backdoor_autoload_copy_ucode(struct amdgpu_device *adev, 1043 SOC24_FIRMWARE_ID id, 1044 const void *fw_data, 1045 uint32_t fw_size) 1046 { 1047 uint32_t toc_offset; 1048 uint32_t toc_fw_size, toc_fw_inst_size; 1049 char *ptr = adev->gfx.rlc.rlc_autoload_ptr; 1050 int i, num_inst; 1051 1052 if (id <= SOC24_FIRMWARE_ID_INVALID || id >= SOC24_FIRMWARE_ID_MAX) 1053 return; 1054 1055 toc_offset = rlc_autoload_info[id].offset; 1056 toc_fw_size = rlc_autoload_info[id].size; 1057 num_inst = rlc_autoload_info[id].num_inst; 1058 toc_fw_inst_size = toc_fw_size / num_inst; 1059 1060 if (fw_size == 0) 1061 fw_size = toc_fw_inst_size; 1062 1063 if (fw_size > toc_fw_inst_size) 1064 fw_size = toc_fw_inst_size; 1065 1066 for (i = 0; i < num_inst; i++) { 1067 if ((num_inst == RLC_NUM_INS_CODE0) || 1068 ((1 << (i / 2)) & adev->gfx.xcc_mask)) { 1069 memcpy(ptr + toc_offset + i * toc_fw_inst_size, fw_data, fw_size); 1070 1071 if (fw_size < toc_fw_inst_size) 1072 memset(ptr + toc_offset + fw_size + i * toc_fw_inst_size, 1073 0, toc_fw_inst_size - fw_size); 1074 } 1075 } 1076 } 1077 1078 static void 1079 gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(struct amdgpu_device *adev) 1080 { 1081 void *data; 1082 uint32_t size; 1083 uint32_t *toc_ptr; 1084 1085 data = adev->psp.toc.start_addr; 1086 size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_TOC].size; 1087 1088 toc_ptr = (uint32_t *)data + size / 4 - 2; 1089 *toc_ptr = (RLC_TOC_FORMAT_API << 24) | 0x1; 1090 1091 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_TOC, 1092 data, size); 1093 } 1094 1095 static void 1096 gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(struct amdgpu_device *adev) 1097 { 1098 const __le32 *fw_data; 1099 uint32_t fw_size; 1100 const struct gfx_firmware_header_v2_0 *cpv2_hdr; 1101 const struct rlc_firmware_header_v2_0 *rlc_hdr; 1102 const struct rlc_firmware_header_v2_1 *rlcv21_hdr; 1103 const struct rlc_firmware_header_v2_2 *rlcv22_hdr; 1104 uint16_t version_major, version_minor; 1105 1106 /* mec ucode */ 1107 cpv2_hdr = (const struct gfx_firmware_header_v2_0 *) 1108 adev->gfx.mec_fw->data; 1109 /* instruction */ 1110 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1111 le32_to_cpu(cpv2_hdr->ucode_offset_bytes)); 1112 fw_size = le32_to_cpu(cpv2_hdr->ucode_size_bytes); 1113 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC, 1114 fw_data, fw_size); 1115 /* data */ 1116 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 1117 le32_to_cpu(cpv2_hdr->data_offset_bytes)); 1118 fw_size = le32_to_cpu(cpv2_hdr->data_size_bytes); 1119 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P0_STACK, 1120 fw_data, fw_size); 1121 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P1_STACK, 1122 fw_data, fw_size); 1123 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P2_STACK, 1124 fw_data, fw_size); 1125 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RS64_MEC_P3_STACK, 1126 fw_data, fw_size); 1127 1128 /* rlc ucode */ 1129 rlc_hdr = (const struct rlc_firmware_header_v2_0 *) 1130 adev->gfx.rlc_fw->data; 1131 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1132 le32_to_cpu(rlc_hdr->header.ucode_array_offset_bytes)); 1133 fw_size = le32_to_cpu(rlc_hdr->header.ucode_size_bytes); 1134 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_G_UCODE, 1135 fw_data, fw_size); 1136 1137 version_major = le16_to_cpu(rlc_hdr->header.header_version_major); 1138 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor); 1139 if (version_major == 2) { 1140 if (version_minor >= 1) { 1141 rlcv21_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data; 1142 1143 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1144 le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_offset_bytes)); 1145 fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_gpm_size_bytes); 1146 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLCG_SCRATCH, 1147 fw_data, fw_size); 1148 1149 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1150 le32_to_cpu(rlcv21_hdr->save_restore_list_srm_offset_bytes)); 1151 fw_size = le32_to_cpu(rlcv21_hdr->save_restore_list_srm_size_bytes); 1152 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLC_SRM_ARAM, 1153 fw_data, fw_size); 1154 } 1155 if (version_minor >= 2) { 1156 rlcv22_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 1157 1158 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1159 le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_offset_bytes)); 1160 fw_size = le32_to_cpu(rlcv22_hdr->rlc_iram_ucode_size_bytes); 1161 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_UCODE, 1162 fw_data, fw_size); 1163 1164 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1165 le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_offset_bytes)); 1166 fw_size = le32_to_cpu(rlcv22_hdr->rlc_dram_ucode_size_bytes); 1167 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_RLX6_DRAM_BOOT, 1168 fw_data, fw_size); 1169 } 1170 } 1171 } 1172 1173 static void 1174 gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(struct amdgpu_device *adev) 1175 { 1176 const __le32 *fw_data; 1177 uint32_t fw_size; 1178 const struct sdma_firmware_header_v3_0 *sdma_hdr; 1179 1180 if (adev->sdma.instance[0].fw) { 1181 sdma_hdr = (const struct sdma_firmware_header_v3_0 *) 1182 adev->sdma.instance[0].fw->data; 1183 fw_data = (const __le32 *) (adev->sdma.instance[0].fw->data + 1184 le32_to_cpu(sdma_hdr->ucode_offset_bytes)); 1185 fw_size = le32_to_cpu(sdma_hdr->ucode_size_bytes); 1186 1187 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, SOC24_FIRMWARE_ID_SDMA_UCODE_TH0, 1188 fw_data, fw_size); 1189 } 1190 } 1191 1192 static void 1193 gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(struct amdgpu_device *adev) 1194 { 1195 const __le32 *fw_data; 1196 unsigned fw_size; 1197 const struct mes_firmware_header_v1_0 *mes_hdr; 1198 int pipe, ucode_id, data_id; 1199 1200 for (pipe = 0; pipe < 2; pipe++) { 1201 if (pipe == 0) { 1202 ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P0; 1203 data_id = SOC24_FIRMWARE_ID_RS64_MES_P0_STACK; 1204 } else { 1205 ucode_id = SOC24_FIRMWARE_ID_RS64_MES_P1; 1206 data_id = SOC24_FIRMWARE_ID_RS64_MES_P1_STACK; 1207 } 1208 1209 mes_hdr = (const struct mes_firmware_header_v1_0 *) 1210 adev->mes.fw[pipe]->data; 1211 1212 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1213 le32_to_cpu(mes_hdr->mes_ucode_offset_bytes)); 1214 fw_size = le32_to_cpu(mes_hdr->mes_ucode_size_bytes); 1215 1216 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, ucode_id, fw_data, fw_size); 1217 1218 fw_data = (const __le32 *)(adev->mes.fw[pipe]->data + 1219 le32_to_cpu(mes_hdr->mes_ucode_data_offset_bytes)); 1220 fw_size = le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes); 1221 1222 gfx_v12_1_rlc_backdoor_autoload_copy_ucode(adev, data_id, fw_data, fw_size); 1223 } 1224 } 1225 1226 static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev) 1227 { 1228 uint32_t rlc_g_offset, rlc_g_size; 1229 uint64_t gpu_addr; 1230 uint32_t data; 1231 int i, num_xcc; 1232 1233 /* RLC autoload sequence 2: copy ucode */ 1234 gfx_v12_1_rlc_backdoor_autoload_copy_sdma_ucode(adev); 1235 gfx_v12_1_rlc_backdoor_autoload_copy_gfx_ucode(adev); 1236 gfx_v12_1_rlc_backdoor_autoload_copy_mes_ucode(adev); 1237 gfx_v12_1_rlc_backdoor_autoload_copy_toc_ucode(adev); 1238 1239 rlc_g_offset = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].offset; 1240 rlc_g_size = rlc_autoload_info[SOC24_FIRMWARE_ID_RLC_G_UCODE].size; 1241 gpu_addr = adev->gfx.rlc.rlc_autoload_gpu_addr + rlc_g_offset - adev->gmc.vram_start; 1242 1243 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1244 for (i = 0; i < num_xcc; i++) { 1245 WREG32_SOC15(GC, GET_INST(GC, i), 1246 regGFX_IMU_RLC_BOOTLOADER_ADDR_HI, 1247 upper_32_bits(gpu_addr)); 1248 WREG32_SOC15(GC, GET_INST(GC, i), 1249 regGFX_IMU_RLC_BOOTLOADER_ADDR_LO, 1250 lower_32_bits(gpu_addr)); 1251 WREG32_SOC15(GC, GET_INST(GC, i), 1252 regGFX_IMU_RLC_BOOTLOADER_SIZE, 1253 rlc_g_size); 1254 } 1255 1256 if (adev->gfx.imu.funcs) { 1257 /* RLC autoload sequence 3: load IMU fw */ 1258 if (adev->gfx.imu.funcs->load_microcode) 1259 adev->gfx.imu.funcs->load_microcode(adev); 1260 } 1261 1262 /* unhalt rlc to start autoload */ 1263 for (i = 0; i < num_xcc; i++) { 1264 data = RREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE); 1265 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD0_ENABLE, 1); 1266 data = REG_SET_FIELD(data, RLC_GPM_THREAD_ENABLE, THREAD1_ENABLE, 1); 1267 WREG32_SOC15(GC, GET_INST(GC, i), regRLC_GPM_THREAD_ENABLE, data); 1268 WREG32_SOC15(GC, GET_INST(GC, i), regRLC_CNTL, RLC_CNTL__RLC_ENABLE_F32_MASK); 1269 } 1270 1271 return 0; 1272 } 1273 1274 static void gfx_v12_1_alloc_ip_dump(struct amdgpu_device *adev) 1275 { 1276 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1); 1277 uint32_t *ptr, inst, num_xcc; 1278 1279 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1280 1281 ptr = kcalloc(reg_count * num_xcc, sizeof(uint32_t), GFP_KERNEL); 1282 if (!ptr) { 1283 DRM_ERROR("Failed to allocate memory for GFX IP Dump\n"); 1284 adev->gfx.ip_dump_core = NULL; 1285 } else { 1286 adev->gfx.ip_dump_core = ptr; 1287 } 1288 1289 /* Allocate memory for compute queue registers for all the instances */ 1290 reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1); 1291 inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1292 adev->gfx.mec.num_queue_per_pipe; 1293 1294 ptr = kcalloc(reg_count * inst * num_xcc, sizeof(uint32_t), GFP_KERNEL); 1295 if (!ptr) { 1296 DRM_ERROR("Failed to allocate memory for Compute Queues IP Dump\n"); 1297 adev->gfx.ip_dump_compute_queues = NULL; 1298 } else { 1299 adev->gfx.ip_dump_compute_queues = ptr; 1300 } 1301 } 1302 1303 static void gfx_v12_1_ip_print(struct amdgpu_ip_block *ip_block, 1304 struct drm_printer *p) 1305 { 1306 struct amdgpu_device *adev = ip_block->adev; 1307 uint32_t i, j, k; 1308 uint32_t xcc_id, xcc_offset, inst_offset; 1309 uint32_t num_xcc, reg, num_inst; 1310 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1); 1311 1312 if (!adev->gfx.ip_dump_core) 1313 return; 1314 1315 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1316 drm_printf(p, "Number of Instances:%d\n", num_xcc); 1317 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 1318 xcc_offset = xcc_id * reg_count; 1319 drm_printf(p, "\nInstance id:%d\n", xcc_id); 1320 for (i = 0; i < reg_count; i++) 1321 drm_printf(p, "%-50s \t 0x%08x\n", 1322 gc_reg_list_12_1[i].reg_name, 1323 adev->gfx.ip_dump_core[xcc_offset + i]); 1324 } 1325 1326 /* print compute queue registers for all instances */ 1327 if (!adev->gfx.ip_dump_compute_queues) 1328 return; 1329 1330 reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1); 1331 drm_printf(p, "\nnum_xcc: %d num_mec: %d num_pipe: %d num_queue: %d\n", 1332 num_xcc, 1333 adev->gfx.mec.num_mec, 1334 adev->gfx.mec.num_pipe_per_mec, 1335 adev->gfx.mec.num_queue_per_pipe); 1336 1337 num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1338 adev->gfx.mec.num_queue_per_pipe; 1339 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 1340 xcc_offset = xcc_id * reg_count * num_inst; 1341 inst_offset = 0; 1342 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 1343 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 1344 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 1345 drm_printf(p, 1346 "\nxcc:%d mec:%d, pipe:%d, queue:%d\n", 1347 xcc_id, i, j, k); 1348 for (reg = 0; reg < reg_count; reg++) { 1349 drm_printf(p, 1350 "%-50s \t 0x%08x\n", 1351 gc_cp_reg_list_12_1[reg].reg_name, 1352 adev->gfx.ip_dump_compute_queues 1353 [xcc_offset + inst_offset + 1354 reg]); 1355 } 1356 inst_offset += reg_count; 1357 } 1358 } 1359 } 1360 } 1361 } 1362 1363 static void gfx_v12_1_ip_dump(struct amdgpu_ip_block *ip_block) 1364 { 1365 struct amdgpu_device *adev = ip_block->adev; 1366 uint32_t i, j, k; 1367 uint32_t num_xcc, reg, num_inst; 1368 uint32_t xcc_id, xcc_offset, inst_offset; 1369 uint32_t reg_count = ARRAY_SIZE(gc_reg_list_12_1); 1370 1371 if (!adev->gfx.ip_dump_core) 1372 return; 1373 1374 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1375 1376 amdgpu_gfx_off_ctrl(adev, false); 1377 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 1378 xcc_offset = xcc_id * reg_count; 1379 for (i = 0; i < reg_count; i++) 1380 adev->gfx.ip_dump_core[xcc_offset + i] = 1381 RREG32(SOC15_REG_ENTRY_OFFSET_INST(gc_reg_list_12_1[i], 1382 GET_INST(GC, xcc_id))); 1383 } 1384 amdgpu_gfx_off_ctrl(adev, true); 1385 1386 /* dump compute queue registers for all instances */ 1387 if (!adev->gfx.ip_dump_compute_queues) 1388 return; 1389 1390 num_inst = adev->gfx.mec.num_mec * adev->gfx.mec.num_pipe_per_mec * 1391 adev->gfx.mec.num_queue_per_pipe; 1392 reg_count = ARRAY_SIZE(gc_cp_reg_list_12_1); 1393 amdgpu_gfx_off_ctrl(adev, false); 1394 mutex_lock(&adev->srbm_mutex); 1395 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 1396 xcc_offset = xcc_id * reg_count * num_inst; 1397 inst_offset = 0; 1398 for (i = 0; i < adev->gfx.mec.num_mec; i++) { 1399 for (j = 0; j < adev->gfx.mec.num_pipe_per_mec; j++) { 1400 for (k = 0; k < adev->gfx.mec.num_queue_per_pipe; k++) { 1401 /* ME0 is for GFX so start from 1 for CP */ 1402 soc_v1_0_grbm_select(adev, 1 + i, j, k, 0, 1403 GET_INST(GC, xcc_id)); 1404 1405 for (reg = 0; reg < reg_count; reg++) { 1406 adev->gfx.ip_dump_compute_queues 1407 [xcc_offset + 1408 inst_offset + reg] = 1409 RREG32(SOC15_REG_ENTRY_OFFSET_INST( 1410 gc_cp_reg_list_12_1[reg], 1411 GET_INST(GC, xcc_id))); 1412 } 1413 inst_offset += reg_count; 1414 } 1415 } 1416 } 1417 } 1418 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, 0); 1419 mutex_unlock(&adev->srbm_mutex); 1420 amdgpu_gfx_off_ctrl(adev, true); 1421 } 1422 1423 static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block) 1424 { 1425 uint16_t major_ver, minor_ver; 1426 int i, j, k, r, ring_id = 0; 1427 unsigned num_compute_rings; 1428 int xcc_id, num_xcc; 1429 struct amdgpu_device *adev = ip_block->adev; 1430 1431 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 1432 case IP_VERSION(12, 1, 0): 1433 adev->gfx.mec.num_mec = 1; 1434 adev->gfx.mec.num_pipe_per_mec = 4; 1435 adev->gfx.mec.num_queue_per_pipe = 8; 1436 1437 if (!amdgpu_discovery_get_gc_major_minor_version( 1438 adev, &major_ver, &minor_ver)) { 1439 if (major_ver == 1 && minor_ver == 3) { 1440 adev->gfx.config.max_cu_per_sh /= 2; 1441 dev_dbg(adev->dev, "Halving max_cu_per_sh for GC Discovery table v1:3 %d\n", 1442 adev->gfx.config.max_cu_per_sh); 1443 } 1444 } 1445 break; 1446 default: 1447 adev->gfx.mec.num_mec = 2; 1448 adev->gfx.mec.num_pipe_per_mec = 2; 1449 adev->gfx.mec.num_queue_per_pipe = 4; 1450 break; 1451 } 1452 1453 if (adev->gfx.num_compute_rings) { 1454 /* recalculate compute rings to use based on hardware configuration */ 1455 num_compute_rings = (adev->gfx.mec.num_pipe_per_mec * 1456 adev->gfx.mec.num_queue_per_pipe) / 2; 1457 adev->gfx.num_compute_rings = min(adev->gfx.num_compute_rings, 1458 num_compute_rings); 1459 } 1460 1461 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1462 1463 /* EOP Event */ 1464 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP, 1465 GFX_12_1_0__SRCID__CP_EOP_INTERRUPT, 1466 &adev->gfx.eop_irq); 1467 if (r) 1468 return r; 1469 1470 /* Privileged reg */ 1471 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP, 1472 GFX_12_1_0__SRCID__CP_PRIV_REG_FAULT, 1473 &adev->gfx.priv_reg_irq); 1474 if (r) 1475 return r; 1476 1477 /* Privileged inst */ 1478 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_GRBM_CP, 1479 GFX_12_1_0__SRCID__CP_PRIV_INSTR_FAULT, 1480 &adev->gfx.priv_inst_irq); 1481 if (r) 1482 return r; 1483 1484 /* RLC POISON Error */ 1485 r = amdgpu_irq_add_id(adev, SOC_V1_0_IH_CLIENTID_RLC, 1486 GFX_12_1_0__SRCID__RLC_POISON_INTERRUPT, 1487 &adev->gfx.rlc_poison_irq); 1488 if (r) 1489 return r; 1490 1491 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE; 1492 1493 r = gfx_v12_1_rlc_init(adev); 1494 if (r) { 1495 dev_err(adev->dev, "Failed to init rlc BOs!\n"); 1496 return r; 1497 } 1498 1499 r = gfx_v12_1_mec_init(adev); 1500 if (r) { 1501 dev_err(adev->dev, "Failed to init MEC BOs!\n"); 1502 return r; 1503 } 1504 1505 /* set up the compute queues - allocate horizontally across pipes */ 1506 for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { 1507 ring_id = 0; 1508 for (i = 0; i < adev->gfx.mec.num_mec; ++i) { 1509 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { 1510 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) { 1511 if (!amdgpu_gfx_is_mec_queue_enabled(adev, 1512 xcc_id, i, k, j)) 1513 continue; 1514 1515 r = gfx_v12_1_compute_ring_init(adev, ring_id, 1516 xcc_id, i, k, j); 1517 if (r) 1518 return r; 1519 1520 ring_id++; 1521 } 1522 } 1523 } 1524 1525 if (!adev->enable_mes_kiq) { 1526 r = amdgpu_gfx_kiq_init(adev, GFX12_MEC_HPD_SIZE, xcc_id); 1527 if (r) { 1528 dev_err(adev->dev, "Failed to init KIQ BOs!\n"); 1529 return r; 1530 } 1531 1532 r = amdgpu_gfx_kiq_init_ring(adev, xcc_id); 1533 if (r) 1534 return r; 1535 } 1536 1537 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v12_1_compute_mqd), xcc_id); 1538 if (r) 1539 return r; 1540 } 1541 1542 /* allocate visible FB for rlc auto-loading fw */ 1543 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 1544 r = gfx_v12_1_rlc_autoload_buffer_init(adev); 1545 if (r) 1546 return r; 1547 } else if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 1548 r = gfx_v12_1_init_cp_compute_microcode_bo(adev); 1549 if (r) 1550 return r; 1551 } 1552 1553 r = gfx_v12_1_gpu_early_init(adev); 1554 if (r) 1555 return r; 1556 1557 r = amdgpu_gfx_sysfs_init(adev); 1558 if (r) 1559 return r; 1560 1561 gfx_v12_1_alloc_ip_dump(adev); 1562 1563 mutex_init(&adev->gfx.mec.reset_mutex); 1564 1565 return 0; 1566 } 1567 1568 static void gfx_v12_1_rlc_autoload_buffer_fini(struct amdgpu_device *adev) 1569 { 1570 amdgpu_bo_free_kernel(&adev->gfx.rlc.rlc_autoload_bo, 1571 &adev->gfx.rlc.rlc_autoload_gpu_addr, 1572 (void **)&adev->gfx.rlc.rlc_autoload_ptr); 1573 } 1574 1575 static int gfx_v12_1_sw_fini(struct amdgpu_ip_block *ip_block) 1576 { 1577 int i, num_xcc; 1578 struct amdgpu_device *adev = ip_block->adev; 1579 1580 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1581 for (i = 0; i < adev->gfx.num_compute_rings * num_xcc; i++) 1582 amdgpu_ring_fini(&adev->gfx.compute_ring[i]); 1583 1584 for (i = 0; i < num_xcc; i++) { 1585 amdgpu_gfx_mqd_sw_fini(adev, i); 1586 1587 if (!adev->enable_mes_kiq) { 1588 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq[i].ring); 1589 amdgpu_gfx_kiq_fini(adev, i); 1590 } 1591 } 1592 1593 gfx_v12_1_rlc_fini(adev); 1594 gfx_v12_1_mec_fini(adev); 1595 1596 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) 1597 gfx_v12_1_rlc_autoload_buffer_fini(adev); 1598 1599 gfx_v12_1_free_microcode(adev); 1600 amdgpu_gfx_sysfs_fini(adev); 1601 1602 kfree(adev->gfx.ip_dump_core); 1603 kfree(adev->gfx.ip_dump_compute_queues); 1604 1605 return 0; 1606 } 1607 1608 static void gfx_v12_1_xcc_select_se_sh(struct amdgpu_device *adev, u32 se_num, 1609 u32 sh_num, u32 instance, int xcc_id) 1610 { 1611 u32 data; 1612 1613 if (instance == 0xffffffff) 1614 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, 1615 INSTANCE_BROADCAST_WRITES, 1); 1616 else 1617 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, 1618 instance); 1619 1620 if (se_num == 0xffffffff) 1621 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1622 1); 1623 else 1624 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num); 1625 1626 if (sh_num == 0xffffffff) 1627 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_BROADCAST_WRITES, 1628 1); 1629 else 1630 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SA_INDEX, sh_num); 1631 1632 WREG32_SOC15_RLC_SHADOW_EX(reg, GC, GET_INST(GC, xcc_id), regGRBM_GFX_INDEX, data); 1633 } 1634 1635 static u32 gfx_v12_1_get_sa_active_bitmap(struct amdgpu_device *adev, 1636 int xcc_id) 1637 { 1638 u32 gc_disabled_sa_mask, gc_user_disabled_sa_mask, sa_mask; 1639 1640 gc_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SA_UNIT_DISABLE); 1641 gc_disabled_sa_mask = REG_GET_FIELD(gc_disabled_sa_mask, 1642 CC_GC_SA_UNIT_DISABLE, 1643 SA_DISABLE); 1644 gc_user_disabled_sa_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SA_UNIT_DISABLE); 1645 gc_user_disabled_sa_mask = REG_GET_FIELD(gc_user_disabled_sa_mask, 1646 GC_USER_SA_UNIT_DISABLE, 1647 SA_DISABLE); 1648 sa_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_sh_per_se * 1649 adev->gfx.config.max_shader_engines); 1650 1651 return sa_mask & (~(gc_disabled_sa_mask | gc_user_disabled_sa_mask)); 1652 } 1653 1654 static u32 gfx_v12_1_get_rb_active_bitmap(struct amdgpu_device *adev, 1655 int xcc_id) 1656 { 1657 u32 gc_disabled_rb_mask, gc_user_disabled_rb_mask; 1658 u32 rb_mask; 1659 1660 gc_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), 1661 regCC_RB_BACKEND_DISABLE); 1662 gc_disabled_rb_mask = REG_GET_FIELD(gc_disabled_rb_mask, 1663 CC_RB_BACKEND_DISABLE, 1664 BACKEND_DISABLE); 1665 gc_user_disabled_rb_mask = RREG32_SOC15(GC, GET_INST(GC, xcc_id), 1666 regGC_USER_RB_BACKEND_DISABLE); 1667 gc_user_disabled_rb_mask = REG_GET_FIELD(gc_user_disabled_rb_mask, 1668 GC_USER_RB_BACKEND_DISABLE, 1669 BACKEND_DISABLE); 1670 rb_mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se * 1671 adev->gfx.config.max_shader_engines); 1672 1673 return rb_mask & (~(gc_disabled_rb_mask | gc_user_disabled_rb_mask)); 1674 } 1675 1676 static void gfx_v12_1_setup_rb(struct amdgpu_device *adev) 1677 { 1678 u32 rb_bitmap_width_per_sa; 1679 u32 max_sa; 1680 u32 active_sa_bitmap; 1681 u32 global_active_rb_bitmap; 1682 u32 active_rb_bitmap = 0; 1683 u32 i; 1684 int xcc_id; 1685 1686 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) { 1687 /* query sa bitmap from SA_UNIT_DISABLE registers */ 1688 active_sa_bitmap = gfx_v12_1_get_sa_active_bitmap(adev, xcc_id); 1689 /* query rb bitmap from RB_BACKEND_DISABLE registers */ 1690 global_active_rb_bitmap = gfx_v12_1_get_rb_active_bitmap(adev, xcc_id); 1691 1692 /* generate active rb bitmap according to active sa bitmap */ 1693 max_sa = adev->gfx.config.max_shader_engines * 1694 adev->gfx.config.max_sh_per_se; 1695 rb_bitmap_width_per_sa = adev->gfx.config.max_backends_per_se / 1696 adev->gfx.config.max_sh_per_se; 1697 for (i = 0; i < max_sa; i++) { 1698 if (active_sa_bitmap & (1 << i)) 1699 active_rb_bitmap |= (0x3 << (i * rb_bitmap_width_per_sa)); 1700 } 1701 1702 active_rb_bitmap |= global_active_rb_bitmap; 1703 } 1704 1705 adev->gfx.config.backend_enable_mask = active_rb_bitmap; 1706 adev->gfx.config.num_rbs = hweight32(active_rb_bitmap); 1707 } 1708 1709 static void gfx_v12_1_xcc_init_compute_vmid(struct amdgpu_device *adev, 1710 int xcc_id) 1711 { 1712 int i; 1713 uint32_t sh_mem_bases; 1714 uint32_t data; 1715 1716 /* 1717 * Configure apertures: 1718 * LDS: 0x20000000'00000000 - 0x20000001'00000000 (4GB) 1719 * Scratch: 0x10000000'00000000 - 0x11ffffff'ffffffff (128PB 57-bit) 1720 */ 1721 sh_mem_bases = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 1722 (adev->gmc.private_aperture_start >> 58)); 1723 sh_mem_bases = REG_SET_FIELD(sh_mem_bases, SH_MEM_BASES, SHARED_BASE, 1724 (adev->gmc.shared_aperture_start >> 48)); 1725 1726 mutex_lock(&adev->srbm_mutex); 1727 for (i = adev->vm_manager.first_kfd_vmid; i < AMDGPU_NUM_VMID; i++) { 1728 soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); 1729 /* CP and shaders */ 1730 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 1731 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, sh_mem_bases); 1732 1733 /* Enable trap for each kfd vmid. */ 1734 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL); 1735 data = REG_SET_FIELD(data, SPI_GDBG_PER_VMID_CNTL, TRAP_EN, 1); 1736 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSPI_GDBG_PER_VMID_CNTL, data); 1737 1738 /* Disable VGPR deallocation instruction for each KFD vmid. */ 1739 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_DEBUG); 1740 data = REG_SET_FIELD(data, SQ_DEBUG, DISABLE_VGPR_DEALLOC, 1); 1741 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSQ_DEBUG, data); 1742 } 1743 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 1744 mutex_unlock(&adev->srbm_mutex); 1745 } 1746 1747 static void gfx_v12_1_tcp_harvest(struct amdgpu_device *adev) 1748 { 1749 /* TODO: harvest feature to be added later. */ 1750 } 1751 1752 static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev) 1753 { 1754 } 1755 1756 static void gfx_v12_1_xcc_xnack_set_chicken_bits(struct amdgpu_device *adev, int xcc_id) 1757 { 1758 /* NOTE: COMPRESSION_ENABLE is used a chicken bit to enable/disable xcc xnack */ 1759 mutex_lock(&adev->srbm_mutex); 1760 if (!adev->gmc.noretry) { 1761 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), 1762 TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x1); 1763 } else 1764 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), 1765 TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x0); 1766 mutex_unlock(&adev->srbm_mutex); 1767 } 1768 1769 static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev, 1770 int xcc_id) 1771 { 1772 u32 tmp; 1773 int i; 1774 1775 /* XXX SH_MEM regs */ 1776 /* where to put LDS, scratch, GPUVM in FSA64 space */ 1777 mutex_lock(&adev->srbm_mutex); 1778 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB(0)].num_ids; i++) { 1779 soc_v1_0_grbm_select(adev, 0, 0, 0, i, GET_INST(GC, xcc_id)); 1780 /* CP and shaders */ 1781 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 1782 regSH_MEM_CONFIG, DEFAULT_SH_MEM_CONFIG); 1783 if (i != 0) { 1784 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE, 1785 (adev->gmc.private_aperture_start >> 58)); 1786 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE, 1787 (adev->gmc.shared_aperture_start >> 48)); 1788 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regSH_MEM_BASES, tmp); 1789 } 1790 } 1791 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 1792 1793 mutex_unlock(&adev->srbm_mutex); 1794 1795 gfx_v12_1_xcc_init_compute_vmid(adev, xcc_id); 1796 gfx_v12_1_xcc_xnack_set_chicken_bits(adev, xcc_id); 1797 1798 } 1799 1800 static void gfx_v12_1_constants_init(struct amdgpu_device *adev) 1801 { 1802 int i, num_xcc; 1803 1804 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1805 1806 gfx_v12_1_setup_rb(adev); 1807 gfx_v12_1_get_cu_info(adev, &adev->gfx.cu_info); 1808 gfx_v12_1_get_tcc_info(adev); 1809 adev->gfx.config.pa_sc_tile_steering_override = 0; 1810 1811 for (i = 0; i < num_xcc; i++) 1812 gfx_v12_1_xcc_constants_init(adev, i); 1813 } 1814 1815 static void gfx_v12_1_xcc_enable_gui_idle_interrupt(struct amdgpu_device *adev, 1816 bool enable, int xcc_id) 1817 { 1818 u32 tmp; 1819 1820 if (amdgpu_sriov_vf(adev)) 1821 return; 1822 1823 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0); 1824 1825 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, 1826 enable ? 1 : 0); 1827 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, 1828 enable ? 1 : 0); 1829 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, 1830 enable ? 1 : 0); 1831 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, 1832 enable ? 1 : 0); 1833 1834 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL_RING0, tmp); 1835 } 1836 1837 static int gfx_v12_1_xcc_init_csb(struct amdgpu_device *adev, 1838 int xcc_id) 1839 { 1840 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr); 1841 1842 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_HI, 1843 adev->gfx.rlc.clear_state_gpu_addr >> 32); 1844 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CSIB_ADDR_LO, 1845 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc); 1846 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 1847 regRLC_CSIB_LENGTH, adev->gfx.rlc.clear_state_size); 1848 1849 return 0; 1850 } 1851 1852 static void gfx_v12_1_xcc_rlc_stop(struct amdgpu_device *adev, 1853 int xcc_id) 1854 { 1855 u32 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL); 1856 1857 tmp = REG_SET_FIELD(tmp, RLC_CNTL, RLC_ENABLE_F32, 0); 1858 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CNTL, tmp); 1859 } 1860 1861 static void gfx_v12_1_rlc_stop(struct amdgpu_device *adev) 1862 { 1863 int i, num_xcc; 1864 1865 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1866 for (i = 0; i < num_xcc; i++) 1867 gfx_v12_1_xcc_rlc_stop(adev, i); 1868 } 1869 1870 static void gfx_v12_1_xcc_rlc_reset(struct amdgpu_device *adev, 1871 int xcc_id) 1872 { 1873 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), 1874 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); 1875 udelay(50); 1876 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), 1877 GRBM_SOFT_RESET, SOFT_RESET_RLC, 0); 1878 udelay(50); 1879 } 1880 1881 static void gfx_v12_1_rlc_reset(struct amdgpu_device *adev) 1882 { 1883 int i, num_xcc; 1884 1885 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1886 for (i = 0; i < num_xcc; i++) 1887 gfx_v12_1_xcc_rlc_reset(adev, i); 1888 } 1889 1890 static void gfx_v12_1_xcc_rlc_smu_handshake_cntl(struct amdgpu_device *adev, 1891 bool enable, int xcc_id) 1892 { 1893 uint32_t rlc_pg_cntl; 1894 1895 rlc_pg_cntl = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL); 1896 1897 if (!enable) { 1898 /* RLC_PG_CNTL[23] = 0 (default) 1899 * RLC will wait for handshake acks with SMU 1900 * GFXOFF will be enabled 1901 * RLC_PG_CNTL[23] = 1 1902 * RLC will not issue any message to SMU 1903 * hence no handshake between SMU & RLC 1904 * GFXOFF will be disabled 1905 */ 1906 rlc_pg_cntl |= RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 1907 } else 1908 rlc_pg_cntl &= ~RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK; 1909 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, rlc_pg_cntl); 1910 } 1911 1912 static void gfx_v12_1_xcc_rlc_start(struct amdgpu_device *adev, 1913 int xcc_id) 1914 { 1915 /* TODO: enable rlc & smu handshake until smu 1916 * and gfxoff feature works as expected */ 1917 if (!(amdgpu_pp_feature_mask & PP_GFXOFF_MASK)) 1918 gfx_v12_1_xcc_rlc_smu_handshake_cntl(adev, false, xcc_id); 1919 1920 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), RLC_CNTL, RLC_ENABLE_F32, 1); 1921 udelay(50); 1922 } 1923 1924 static void gfx_v12_1_rlc_start(struct amdgpu_device *adev) 1925 { 1926 int i, num_xcc; 1927 1928 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1929 for (i = 0; i < num_xcc; i++) { 1930 gfx_v12_1_xcc_rlc_start(adev, i); 1931 } 1932 } 1933 1934 static void gfx_v12_1_xcc_rlc_enable_srm(struct amdgpu_device *adev, 1935 int xcc_id) 1936 { 1937 uint32_t tmp; 1938 1939 /* enable Save Restore Machine */ 1940 tmp = RREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL)); 1941 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK; 1942 tmp |= RLC_SRM_CNTL__SRM_ENABLE_MASK; 1943 WREG32(SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SRM_CNTL), tmp); 1944 } 1945 1946 static void gfx_v12_1_xcc_load_rlcg_microcode(struct amdgpu_device *adev, 1947 int xcc_id) 1948 { 1949 const struct rlc_firmware_header_v2_0 *hdr; 1950 const __le32 *fw_data; 1951 unsigned i, fw_size; 1952 1953 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 1954 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1955 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1956 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4; 1957 1958 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_GPM_UCODE_ADDR, 1959 RLCG_UCODE_LOADING_START_ADDRESS); 1960 1961 for (i = 0; i < fw_size; i++) 1962 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 1963 regRLC_GPM_UCODE_DATA, 1964 le32_to_cpup(fw_data++)); 1965 1966 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 1967 regRLC_GPM_UCODE_ADDR, 1968 adev->gfx.rlc_fw_version); 1969 } 1970 1971 static void gfx_v12_1_xcc_load_rlc_iram_dram_microcode(struct amdgpu_device *adev, 1972 int xcc_id) 1973 { 1974 const struct rlc_firmware_header_v2_2 *hdr; 1975 const __le32 *fw_data; 1976 unsigned i, fw_size; 1977 u32 tmp; 1978 1979 hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data; 1980 1981 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1982 le32_to_cpu(hdr->rlc_iram_ucode_offset_bytes)); 1983 fw_size = le32_to_cpu(hdr->rlc_iram_ucode_size_bytes) / 4; 1984 1985 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_IRAM_ADDR, 0); 1986 1987 for (i = 0; i < fw_size; i++) { 1988 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 1989 msleep(1); 1990 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 1991 regRLC_LX6_IRAM_DATA, 1992 le32_to_cpup(fw_data++)); 1993 } 1994 1995 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 1996 regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 1997 1998 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data + 1999 le32_to_cpu(hdr->rlc_dram_ucode_offset_bytes)); 2000 fw_size = le32_to_cpu(hdr->rlc_dram_ucode_size_bytes) / 4; 2001 2002 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 2003 regRLC_LX6_DRAM_ADDR, 0); 2004 for (i = 0; i < fw_size; i++) { 2005 if ((amdgpu_emu_mode == 1) && (i % 100 == 99)) 2006 msleep(1); 2007 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 2008 regRLC_LX6_DRAM_DATA, 2009 le32_to_cpup(fw_data++)); 2010 } 2011 2012 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 2013 regRLC_LX6_IRAM_ADDR, adev->gfx.rlc_fw_version); 2014 2015 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL); 2016 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, PDEBUG_ENABLE, 1); 2017 tmp = REG_SET_FIELD(tmp, RLC_LX6_CNTL, BRESET, 0); 2018 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_LX6_CNTL, tmp); 2019 } 2020 2021 static int gfx_v12_1_xcc_rlc_load_microcode(struct amdgpu_device *adev, 2022 int xcc_id) 2023 { 2024 const struct rlc_firmware_header_v2_0 *hdr; 2025 uint16_t version_major; 2026 uint16_t version_minor; 2027 2028 if (!adev->gfx.rlc_fw) 2029 return -EINVAL; 2030 2031 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data; 2032 amdgpu_ucode_print_rlc_hdr(&hdr->header); 2033 2034 version_major = le16_to_cpu(hdr->header.header_version_major); 2035 version_minor = le16_to_cpu(hdr->header.header_version_minor); 2036 2037 if (version_major == 2) { 2038 gfx_v12_1_xcc_load_rlcg_microcode(adev, xcc_id); 2039 if (amdgpu_dpm == 1) { 2040 if (version_minor >= 2) 2041 gfx_v12_1_xcc_load_rlc_iram_dram_microcode(adev, xcc_id); 2042 } 2043 2044 return 0; 2045 } 2046 2047 return -EINVAL; 2048 } 2049 2050 static int gfx_v12_1_xcc_rlc_resume(struct amdgpu_device *adev, 2051 int xcc_id) 2052 { 2053 int r; 2054 2055 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 2056 gfx_v12_1_xcc_init_csb(adev, xcc_id); 2057 2058 if (!amdgpu_sriov_vf(adev)) /* enable RLC SRM */ 2059 gfx_v12_1_xcc_rlc_enable_srm(adev, xcc_id); 2060 } else { 2061 if (amdgpu_sriov_vf(adev)) { 2062 gfx_v12_1_xcc_init_csb(adev, xcc_id); 2063 return 0; 2064 } 2065 2066 gfx_v12_1_xcc_rlc_stop(adev, xcc_id); 2067 2068 /* disable CG */ 2069 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, 0); 2070 2071 /* disable PG */ 2072 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_PG_CNTL, 0); 2073 2074 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 2075 /* legacy rlc firmware loading */ 2076 r = gfx_v12_1_xcc_rlc_load_microcode(adev, xcc_id); 2077 if (r) 2078 return r; 2079 } 2080 2081 gfx_v12_1_xcc_init_csb(adev, xcc_id); 2082 2083 gfx_v12_1_xcc_rlc_start(adev, xcc_id); 2084 } 2085 2086 return 0; 2087 } 2088 2089 static int gfx_v12_1_rlc_resume(struct amdgpu_device *adev) 2090 { 2091 int r, i, num_xcc; 2092 2093 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 2094 for (i = 0; i < num_xcc; i++) { 2095 r = gfx_v12_1_xcc_rlc_resume(adev, i); 2096 if (r) 2097 return r; 2098 } 2099 2100 return 0; 2101 } 2102 2103 static void gfx_v12_1_xcc_config_gfx_rs64(struct amdgpu_device *adev, 2104 int xcc_id) 2105 { 2106 const struct gfx_firmware_header_v2_0 *mec_hdr; 2107 uint32_t pipe_id, tmp; 2108 2109 mec_hdr = (const struct gfx_firmware_header_v2_0 *) 2110 adev->gfx.mec_fw->data; 2111 2112 /* config mec program start addr */ 2113 for (pipe_id = 0; pipe_id < 4; pipe_id++) { 2114 soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id)); 2115 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START, 2116 mec_hdr->ucode_start_addr_lo >> 2 | 2117 mec_hdr->ucode_start_addr_hi << 30); 2118 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2119 mec_hdr->ucode_start_addr_hi >> 2); 2120 } 2121 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 2122 2123 /* reset mec pipe */ 2124 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL); 2125 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1); 2126 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1); 2127 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1); 2128 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1); 2129 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp); 2130 2131 /* clear mec pipe reset */ 2132 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0); 2133 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0); 2134 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0); 2135 tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0); 2136 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, tmp); 2137 } 2138 2139 static void gfx_v12_1_config_gfx_rs64(struct amdgpu_device *adev) 2140 { 2141 int i, num_xcc; 2142 2143 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 2144 2145 for (i = 0; i < num_xcc; i++) 2146 gfx_v12_1_xcc_config_gfx_rs64(adev, i); 2147 } 2148 2149 static void gfx_v12_1_xcc_set_mec_ucode_start_addr(struct amdgpu_device *adev, 2150 int xcc_id) 2151 { 2152 const struct gfx_firmware_header_v2_0 *cp_hdr; 2153 unsigned pipe_id; 2154 2155 cp_hdr = (const struct gfx_firmware_header_v2_0 *) 2156 adev->gfx.mec_fw->data; 2157 mutex_lock(&adev->srbm_mutex); 2158 for (pipe_id = 0; pipe_id < adev->gfx.mec.num_pipe_per_mec; pipe_id++) { 2159 soc_v1_0_grbm_select(adev, 1, pipe_id, 0, 0, GET_INST(GC, xcc_id)); 2160 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START, 2161 cp_hdr->ucode_start_addr_lo >> 2 | 2162 cp_hdr->ucode_start_addr_hi << 30); 2163 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_PRGRM_CNTR_START_HI, 2164 cp_hdr->ucode_start_addr_hi >> 2); 2165 } 2166 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 2167 mutex_unlock(&adev->srbm_mutex); 2168 } 2169 2170 static int gfx_v12_1_xcc_wait_for_rlc_autoload_complete(struct amdgpu_device *adev, 2171 int xcc_id) 2172 { 2173 uint32_t cp_status; 2174 uint32_t bootload_status; 2175 int i; 2176 2177 for (i = 0; i < adev->usec_timeout; i++) { 2178 cp_status = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_STAT); 2179 bootload_status = RREG32_SOC15(GC, GET_INST(GC, xcc_id), 2180 regRLC_RLCS_BOOTLOAD_STATUS); 2181 2182 if ((cp_status == 0) && 2183 (REG_GET_FIELD(bootload_status, 2184 RLC_RLCS_BOOTLOAD_STATUS, BOOTLOAD_COMPLETE) == 1)) { 2185 break; 2186 } 2187 udelay(1); 2188 if (amdgpu_emu_mode) 2189 msleep(10); 2190 } 2191 2192 if (i >= adev->usec_timeout) { 2193 dev_err(adev->dev, 2194 "rlc autoload: xcc%d gc ucode autoload timeout\n", xcc_id); 2195 return -ETIMEDOUT; 2196 } 2197 2198 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 2199 gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id); 2200 } 2201 2202 return 0; 2203 } 2204 2205 static int gfx_v12_1_wait_for_rlc_autoload_complete(struct amdgpu_device *adev) 2206 { 2207 int xcc_id; 2208 2209 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) 2210 gfx_v12_1_xcc_wait_for_rlc_autoload_complete(adev, xcc_id); 2211 2212 return 0; 2213 } 2214 2215 static void gfx_v12_1_xcc_cp_compute_enable(struct amdgpu_device *adev, 2216 bool enable, int xcc_id) 2217 { 2218 u32 data; 2219 2220 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL); 2221 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_INVALIDATE_ICACHE, 2222 enable ? 0 : 1); 2223 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 2224 enable ? 0 : 1); 2225 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 2226 enable ? 0 : 1); 2227 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 2228 enable ? 0 : 1); 2229 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 2230 enable ? 0 : 1); 2231 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE0_ACTIVE, 2232 enable ? 1 : 0); 2233 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE1_ACTIVE, 2234 enable ? 1 : 0); 2235 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE2_ACTIVE, 2236 enable ? 1 : 0); 2237 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_PIPE3_ACTIVE, 2238 enable ? 1 : 0); 2239 data = REG_SET_FIELD(data, CP_MEC_RS64_CNTL, MEC_HALT, 2240 enable ? 0 : 1); 2241 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_RS64_CNTL, data); 2242 2243 adev->gfx.kiq[xcc_id].ring.sched.ready = enable; 2244 2245 udelay(50); 2246 } 2247 2248 static int gfx_v12_1_init_cp_compute_microcode_bo(struct amdgpu_device *adev) 2249 { 2250 const struct gfx_firmware_header_v2_0 *mec_hdr; 2251 const __le32 *fw_ucode, *fw_data; 2252 u32 fw_ucode_size, fw_data_size; 2253 u32 *fw_ucode_ptr, *fw_data_ptr; 2254 int i, r, xcc_id; 2255 2256 if (!adev->gfx.mec_fw) 2257 return -EINVAL; 2258 2259 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; 2260 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header); 2261 2262 fw_ucode = (const __le32 *) (adev->gfx.mec_fw->data + 2263 le32_to_cpu(mec_hdr->ucode_offset_bytes)); 2264 fw_ucode_size = le32_to_cpu(mec_hdr->ucode_size_bytes); 2265 2266 fw_data = (const __le32 *) (adev->gfx.mec_fw->data + 2267 le32_to_cpu(mec_hdr->data_offset_bytes)); 2268 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes); 2269 2270 if (adev->gfx.mec.mec_fw_obj == NULL) { 2271 r = amdgpu_bo_create_reserved(adev, fw_ucode_size, 2272 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM, 2273 &adev->gfx.mec.mec_fw_obj, 2274 &adev->gfx.mec.mec_fw_gpu_addr, 2275 (void **)&fw_ucode_ptr); 2276 if (r) { 2277 dev_err(adev->dev, "(%d) failed to create mec fw ucode bo\n", r); 2278 gfx_v12_1_mec_fini(adev); 2279 return r; 2280 } 2281 2282 memcpy(fw_ucode_ptr, fw_ucode, fw_ucode_size); 2283 2284 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj); 2285 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj); 2286 } 2287 2288 if (adev->gfx.mec.mec_fw_data_obj == NULL) { 2289 r = amdgpu_bo_create_reserved(adev, 2290 ALIGN(fw_data_size, 64 * 1024) * 2291 adev->gfx.mec.num_pipe_per_mec * NUM_XCC(adev->gfx.xcc_mask), 2292 64 * 1024, AMDGPU_GEM_DOMAIN_VRAM, 2293 &adev->gfx.mec.mec_fw_data_obj, 2294 &adev->gfx.mec.mec_fw_data_gpu_addr, 2295 (void **)&fw_data_ptr); 2296 if (r) { 2297 dev_err(adev->dev, "(%d) failed to create mec fw data bo\n", r); 2298 gfx_v12_1_mec_fini(adev); 2299 return r; 2300 } 2301 2302 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) { 2303 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 2304 u32 offset = (xcc_id * adev->gfx.mec.num_pipe_per_mec + i) * 2305 ALIGN(fw_data_size, 64 * 1024) / 4; 2306 memcpy(fw_data_ptr + offset, fw_data, fw_data_size); 2307 } 2308 } 2309 2310 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_data_obj); 2311 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_data_obj); 2312 } 2313 2314 return 0; 2315 } 2316 2317 static int gfx_v12_1_xcc_cp_compute_load_microcode_rs64(struct amdgpu_device *adev, 2318 int xcc_id) 2319 { 2320 const struct gfx_firmware_header_v2_0 *mec_hdr; 2321 u32 fw_data_size; 2322 u32 tmp, i, usec_timeout = 50000; /* Wait for 50 ms */ 2323 2324 if (!adev->gfx.mec_fw) 2325 return -EINVAL; 2326 2327 mec_hdr = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data; 2328 fw_data_size = le32_to_cpu(mec_hdr->data_size_bytes); 2329 2330 gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id); 2331 2332 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL); 2333 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0); 2334 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, EXE_DISABLE, 0); 2335 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0); 2336 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_CNTL, tmp); 2337 2338 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL); 2339 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, VMID, 0); 2340 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_BASE_CNTL, CACHE_POLICY, 0); 2341 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_BASE_CNTL, tmp); 2342 2343 mutex_lock(&adev->srbm_mutex); 2344 for (i = 0; i < adev->gfx.mec.num_pipe_per_mec; i++) { 2345 soc_v1_0_grbm_select(adev, 1, i, 0, 0, GET_INST(GC, xcc_id)); 2346 2347 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_LO, 2348 lower_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr + 2349 (xcc_id * adev->gfx.mec.num_pipe_per_mec + i) * 2350 ALIGN(fw_data_size, 64 * 1024))); 2351 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_MDBASE_HI, 2352 upper_32_bits(adev->gfx.mec.mec_fw_data_gpu_addr + 2353 (xcc_id * adev->gfx.mec.num_pipe_per_mec + i) * 2354 ALIGN(fw_data_size, 64 * 1024))); 2355 2356 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_LO, 2357 lower_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 2358 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_BASE_HI, 2359 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr)); 2360 } 2361 mutex_unlock(&adev->srbm_mutex); 2362 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 2363 2364 /* Trigger an invalidation of the L1 instruction caches */ 2365 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL); 2366 tmp = REG_SET_FIELD(tmp, CP_MEC_DC_OP_CNTL, INVALIDATE_DCACHE, 1); 2367 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL, tmp); 2368 2369 /* Wait for invalidation complete */ 2370 for (i = 0; i < usec_timeout; i++) { 2371 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DC_OP_CNTL); 2372 if (1 == REG_GET_FIELD(tmp, CP_MEC_DC_OP_CNTL, 2373 INVALIDATE_DCACHE_COMPLETE)) 2374 break; 2375 udelay(1); 2376 } 2377 2378 if (i >= usec_timeout) { 2379 dev_err(adev->dev, "failed to invalidate data cache\n"); 2380 return -EINVAL; 2381 } 2382 2383 /* Trigger an invalidation of the L1 instruction caches */ 2384 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL); 2385 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_OP_CNTL, INVALIDATE_CACHE, 1); 2386 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL, tmp); 2387 2388 /* Wait for invalidation complete */ 2389 for (i = 0; i < usec_timeout; i++) { 2390 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_CPC_IC_OP_CNTL); 2391 if (1 == REG_GET_FIELD(tmp, CP_CPC_IC_OP_CNTL, 2392 INVALIDATE_CACHE_COMPLETE)) 2393 break; 2394 udelay(1); 2395 } 2396 2397 if (i >= usec_timeout) { 2398 dev_err(adev->dev, "failed to invalidate instruction cache\n"); 2399 return -EINVAL; 2400 } 2401 2402 gfx_v12_1_xcc_set_mec_ucode_start_addr(adev, xcc_id); 2403 2404 return 0; 2405 } 2406 2407 static void gfx_v12_1_xcc_kiq_setting(struct amdgpu_ring *ring, 2408 int xcc_id) 2409 { 2410 uint32_t tmp; 2411 struct amdgpu_device *adev = ring->adev; 2412 2413 /* tell RLC which is KIQ queue */ 2414 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS); 2415 tmp &= 0xffffff00; 2416 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue); 2417 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp); 2418 tmp |= 0x80; 2419 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp); 2420 } 2421 2422 static void gfx_v12_1_xcc_cp_set_doorbell_range(struct amdgpu_device *adev, 2423 int xcc_id) 2424 { 2425 /* disable gfx engine doorbell range */ 2426 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_LOWER, 0); 2427 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_DOORBELL_RANGE_UPPER, 0); 2428 2429 /* set compute engine doorbell range */ 2430 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_LOWER, 2431 ((adev->doorbell_index.kiq + 2432 xcc_id * adev->doorbell_index.xcc_doorbell_range) * 2433 2) << 2); 2434 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MEC_DOORBELL_RANGE_UPPER, 2435 ((adev->doorbell_index.userqueue_end + 2436 xcc_id * adev->doorbell_index.xcc_doorbell_range) * 2437 2) << 2); 2438 } 2439 2440 static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m, 2441 struct amdgpu_mqd_prop *prop) 2442 { 2443 struct v12_1_compute_mqd *mqd = m; 2444 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr; 2445 uint32_t tmp; 2446 2447 mqd->header = 0xC0310800; 2448 mqd->compute_pipelinestat_enable = 0x00000001; 2449 mqd->compute_static_thread_mgmt_se0 = 0xffffffff; 2450 mqd->compute_static_thread_mgmt_se1 = 0xffffffff; 2451 mqd->compute_static_thread_mgmt_se2 = 0xffffffff; 2452 mqd->compute_static_thread_mgmt_se3 = 0xffffffff; 2453 mqd->compute_misc_reserved = 0x00000007; 2454 2455 eop_base_addr = prop->eop_gpu_addr >> 8; 2456 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr; 2457 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr); 2458 2459 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 2460 tmp = regCP_HQD_EOP_CONTROL_DEFAULT; 2461 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE, 2462 (order_base_2(GFX12_MEC_HPD_SIZE / 4) - 1)); 2463 2464 mqd->cp_hqd_eop_control = tmp; 2465 2466 /* enable doorbell? */ 2467 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 2468 2469 if (prop->use_doorbell) { 2470 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2471 DOORBELL_OFFSET, prop->doorbell_index); 2472 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2473 DOORBELL_EN, 1); 2474 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2475 DOORBELL_SOURCE, 0); 2476 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2477 DOORBELL_HIT, 0); 2478 } else { 2479 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2480 DOORBELL_EN, 0); 2481 } 2482 2483 mqd->cp_hqd_pq_doorbell_control = tmp; 2484 2485 /* disable the queue if it's active */ 2486 mqd->cp_hqd_dequeue_request = 0; 2487 mqd->cp_hqd_pq_rptr = 0; 2488 mqd->cp_hqd_pq_wptr_lo = 0; 2489 mqd->cp_hqd_pq_wptr_hi = 0; 2490 2491 /* set the pointer to the MQD */ 2492 mqd->cp_mqd_base_addr_lo = prop->mqd_gpu_addr & 0xfffffffc; 2493 mqd->cp_mqd_base_addr_hi = upper_32_bits(prop->mqd_gpu_addr); 2494 2495 /* set MQD vmid to 0 */ 2496 tmp = regCP_MQD_CONTROL_DEFAULT; 2497 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0); 2498 mqd->cp_mqd_control = tmp; 2499 2500 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 2501 hqd_gpu_addr = prop->hqd_base_gpu_addr >> 8; 2502 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr; 2503 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr); 2504 2505 /* set up the HQD, this is similar to CP_RB0_CNTL */ 2506 tmp = regCP_HQD_PQ_CONTROL_DEFAULT; 2507 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE, 2508 (order_base_2(prop->queue_size / 4) - 1)); 2509 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE, 2510 (order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1)); 2511 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0); 2512 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, TUNNEL_DISPATCH, 0); 2513 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1); 2514 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1); 2515 mqd->cp_hqd_pq_control = tmp; 2516 2517 /* set the wb address whether it's enabled or not */ 2518 wb_gpu_addr = prop->rptr_gpu_addr; 2519 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc; 2520 mqd->cp_hqd_pq_rptr_report_addr_hi = 2521 upper_32_bits(wb_gpu_addr) & 0xffff; 2522 2523 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 2524 wb_gpu_addr = prop->wptr_gpu_addr; 2525 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc; 2526 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff; 2527 2528 tmp = 0; 2529 /* enable the doorbell if requested */ 2530 if (prop->use_doorbell) { 2531 tmp = regCP_HQD_PQ_DOORBELL_CONTROL_DEFAULT; 2532 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2533 DOORBELL_OFFSET, prop->doorbell_index); 2534 2535 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2536 DOORBELL_EN, 1); 2537 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2538 DOORBELL_SOURCE, 0); 2539 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL, 2540 DOORBELL_HIT, 0); 2541 } 2542 2543 mqd->cp_hqd_pq_doorbell_control = tmp; 2544 2545 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 2546 mqd->cp_hqd_pq_rptr = regCP_HQD_PQ_RPTR_DEFAULT; 2547 2548 /* set the vmid for the queue */ 2549 mqd->cp_hqd_vmid = 0; 2550 2551 tmp = regCP_HQD_PERSISTENT_STATE_DEFAULT; 2552 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x63); 2553 mqd->cp_hqd_persistent_state = tmp; 2554 2555 /* set MIN_IB_AVAIL_SIZE */ 2556 tmp = regCP_HQD_IB_CONTROL_DEFAULT; 2557 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 1); 2558 mqd->cp_hqd_ib_control = tmp; 2559 2560 /* set static priority for a compute queue/ring */ 2561 mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority; 2562 mqd->cp_hqd_queue_priority = prop->hqd_queue_priority; 2563 2564 mqd->cp_mqd_stride_size = prop->mqd_stride_size ? prop->mqd_stride_size : 2565 AMDGPU_MQD_SIZE_ALIGN(adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size); 2566 2567 tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1); 2568 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1); 2569 tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1); 2570 mqd->cp_hqd_quantum = tmp; 2571 2572 mqd->cp_hqd_active = prop->hqd_active; 2573 2574 return 0; 2575 } 2576 2577 static int gfx_v12_1_xcc_kiq_init_register(struct amdgpu_ring *ring, 2578 int xcc_id) 2579 { 2580 struct amdgpu_device *adev = ring->adev; 2581 struct v12_1_compute_mqd *mqd = ring->mqd_ptr; 2582 int j; 2583 2584 /* inactivate the queue */ 2585 if (amdgpu_sriov_vf(adev)) 2586 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 0); 2587 2588 /* disable wptr polling */ 2589 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_WPTR_POLL_CNTL, EN, 0); 2590 2591 /* write the EOP addr */ 2592 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR, 2593 mqd->cp_hqd_eop_base_addr_lo); 2594 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_BASE_ADDR_HI, 2595 mqd->cp_hqd_eop_base_addr_hi); 2596 2597 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */ 2598 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_EOP_CONTROL, 2599 mqd->cp_hqd_eop_control); 2600 2601 /* enable doorbell? */ 2602 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 2603 mqd->cp_hqd_pq_doorbell_control); 2604 2605 /* disable the queue if it's active */ 2606 if (RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1) { 2607 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 1); 2608 for (j = 0; j < adev->usec_timeout; j++) { 2609 if (!(RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE) & 1)) 2610 break; 2611 udelay(1); 2612 } 2613 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_DEQUEUE_REQUEST, 2614 mqd->cp_hqd_dequeue_request); 2615 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR, 2616 mqd->cp_hqd_pq_rptr); 2617 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO, 2618 mqd->cp_hqd_pq_wptr_lo); 2619 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI, 2620 mqd->cp_hqd_pq_wptr_hi); 2621 } 2622 2623 /* set the pointer to the MQD */ 2624 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR, 2625 mqd->cp_mqd_base_addr_lo); 2626 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_BASE_ADDR_HI, 2627 mqd->cp_mqd_base_addr_hi); 2628 2629 /* set MQD vmid to 0 */ 2630 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MQD_CONTROL, 2631 mqd->cp_mqd_control); 2632 2633 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */ 2634 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE, 2635 mqd->cp_hqd_pq_base_lo); 2636 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_BASE_HI, 2637 mqd->cp_hqd_pq_base_hi); 2638 2639 /* set up the HQD, this is similar to CP_RB0_CNTL */ 2640 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_CONTROL, 2641 mqd->cp_hqd_pq_control); 2642 2643 /* set the wb address whether it's enabled or not */ 2644 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR, 2645 mqd->cp_hqd_pq_rptr_report_addr_lo); 2646 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_RPTR_REPORT_ADDR_HI, 2647 mqd->cp_hqd_pq_rptr_report_addr_hi); 2648 2649 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */ 2650 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR, 2651 mqd->cp_hqd_pq_wptr_poll_addr_lo); 2652 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_POLL_ADDR_HI, 2653 mqd->cp_hqd_pq_wptr_poll_addr_hi); 2654 2655 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_DOORBELL_CONTROL, 2656 mqd->cp_hqd_pq_doorbell_control); 2657 2658 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */ 2659 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_LO, 2660 mqd->cp_hqd_pq_wptr_lo); 2661 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PQ_WPTR_HI, 2662 mqd->cp_hqd_pq_wptr_hi); 2663 2664 /* set the vmid for the queue */ 2665 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_VMID, mqd->cp_hqd_vmid); 2666 2667 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_PERSISTENT_STATE, 2668 mqd->cp_hqd_persistent_state); 2669 2670 /* activate the queue */ 2671 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_ACTIVE, 2672 mqd->cp_hqd_active); 2673 2674 if (ring->use_doorbell) 2675 WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id), CP_PQ_STATUS, DOORBELL_ENABLE, 1); 2676 2677 return 0; 2678 } 2679 2680 static int gfx_v12_1_xcc_kiq_init_queue(struct amdgpu_ring *ring, 2681 int xcc_id) 2682 { 2683 struct amdgpu_device *adev = ring->adev; 2684 struct v12_1_compute_mqd *mqd = ring->mqd_ptr; 2685 2686 gfx_v12_1_xcc_kiq_setting(ring, xcc_id); 2687 2688 if (amdgpu_in_reset(adev)) { /* for GPU_RESET case */ 2689 /* reset MQD to a clean status */ 2690 if (adev->gfx.kiq[xcc_id].mqd_backup) 2691 memcpy(mqd, adev->gfx.kiq[xcc_id].mqd_backup, sizeof(*mqd)); 2692 2693 /* reset ring buffer */ 2694 ring->wptr = 0; 2695 amdgpu_ring_clear_ring(ring); 2696 2697 mutex_lock(&adev->srbm_mutex); 2698 soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id)); 2699 gfx_v12_1_xcc_kiq_init_register(ring, xcc_id); 2700 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 2701 mutex_unlock(&adev->srbm_mutex); 2702 } else { 2703 memset((void *)mqd, 0, sizeof(*mqd)); 2704 if (amdgpu_sriov_vf(adev) && adev->in_suspend) 2705 amdgpu_ring_clear_ring(ring); 2706 mutex_lock(&adev->srbm_mutex); 2707 soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id)); 2708 amdgpu_ring_init_mqd(ring); 2709 gfx_v12_1_xcc_kiq_init_register(ring, xcc_id); 2710 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 2711 mutex_unlock(&adev->srbm_mutex); 2712 2713 if (adev->gfx.kiq[xcc_id].mqd_backup) 2714 memcpy(adev->gfx.kiq[xcc_id].mqd_backup, mqd, sizeof(*mqd)); 2715 } 2716 2717 return 0; 2718 } 2719 2720 static int gfx_v12_1_xcc_kcq_init_queue(struct amdgpu_ring *ring, 2721 int xcc_id) 2722 { 2723 struct amdgpu_device *adev = ring->adev; 2724 struct v12_1_compute_mqd *mqd = ring->mqd_ptr; 2725 int mqd_idx = ring - &adev->gfx.compute_ring[0]; 2726 2727 if (!amdgpu_in_reset(adev) && !adev->in_suspend) { 2728 memset((void *)mqd, 0, sizeof(*mqd)); 2729 mutex_lock(&adev->srbm_mutex); 2730 soc_v1_0_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0, GET_INST(GC, xcc_id)); 2731 amdgpu_ring_init_mqd(ring); 2732 soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id)); 2733 mutex_unlock(&adev->srbm_mutex); 2734 2735 if (adev->gfx.mec.mqd_backup[mqd_idx]) 2736 memcpy_fromio(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(*mqd)); 2737 } else { 2738 /* restore MQD to a clean status */ 2739 if (adev->gfx.mec.mqd_backup[mqd_idx]) 2740 memcpy_toio(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(*mqd)); 2741 /* reset ring buffer */ 2742 ring->wptr = 0; 2743 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); 2744 amdgpu_ring_clear_ring(ring); 2745 } 2746 2747 return 0; 2748 } 2749 2750 static int gfx_v12_1_xcc_kiq_resume(struct amdgpu_device *adev, 2751 int xcc_id) 2752 { 2753 struct amdgpu_ring *ring; 2754 int r; 2755 2756 ring = &adev->gfx.kiq[xcc_id].ring; 2757 2758 r = amdgpu_bo_reserve(ring->mqd_obj, false); 2759 if (unlikely(r != 0)) 2760 return r; 2761 2762 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 2763 if (unlikely(r != 0)) { 2764 amdgpu_bo_unreserve(ring->mqd_obj); 2765 return r; 2766 } 2767 2768 gfx_v12_1_xcc_kiq_init_queue(ring, xcc_id); 2769 amdgpu_bo_kunmap(ring->mqd_obj); 2770 ring->mqd_ptr = NULL; 2771 amdgpu_bo_unreserve(ring->mqd_obj); 2772 ring->sched.ready = true; 2773 return 0; 2774 } 2775 2776 static int gfx_v12_1_xcc_kcq_resume(struct amdgpu_device *adev, 2777 int xcc_id) 2778 { 2779 struct amdgpu_ring *ring = NULL; 2780 int r = 0, i; 2781 2782 if (!amdgpu_async_gfx_ring) 2783 gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id); 2784 2785 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 2786 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings]; 2787 2788 r = amdgpu_bo_reserve(ring->mqd_obj, false); 2789 if (unlikely(r != 0)) 2790 goto done; 2791 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr); 2792 if (!r) { 2793 r = gfx_v12_1_xcc_kcq_init_queue(ring, xcc_id); 2794 amdgpu_bo_kunmap(ring->mqd_obj); 2795 ring->mqd_ptr = NULL; 2796 } 2797 amdgpu_bo_unreserve(ring->mqd_obj); 2798 if (r) 2799 goto done; 2800 } 2801 2802 r = amdgpu_gfx_enable_kcq(adev, xcc_id); 2803 done: 2804 return r; 2805 } 2806 2807 static int gfx_v12_1_xcc_cp_resume(struct amdgpu_device *adev, uint16_t xcc_mask) 2808 { 2809 int r, i, xcc_id; 2810 struct amdgpu_ring *ring; 2811 2812 for_each_inst(xcc_id, xcc_mask) { 2813 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 2814 /* legacy firmware loading */ 2815 r = gfx_v12_1_xcc_cp_compute_load_microcode_rs64(adev, xcc_id); 2816 if (r) 2817 return r; 2818 } 2819 2820 /* GFX CGCG and LS is set by default */ 2821 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) 2822 gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, true, xcc_id); 2823 2824 gfx_v12_1_xcc_cp_set_doorbell_range(adev, xcc_id); 2825 2826 gfx_v12_1_xcc_cp_compute_enable(adev, true, xcc_id); 2827 2828 if (adev->enable_mes_kiq && adev->mes.kiq_hw_init) { 2829 r = amdgpu_mes_kiq_hw_init(adev, xcc_id); 2830 /* 2831 * With MES, GFX KIQ ring is owned by the MES and is never 2832 * initialized/used directly by the driver, so it must 2833 * not be left flagged as ready. mes_v12_0_hw_init() clears 2834 * but clear here if MES init fails 2835 */ 2836 if (r) 2837 adev->gfx.kiq[xcc_id].ring.sched.ready = false; 2838 } else { 2839 r = gfx_v12_1_xcc_kiq_resume(adev, xcc_id); 2840 } 2841 if (r) 2842 return r; 2843 2844 r = gfx_v12_1_xcc_kcq_resume(adev, xcc_id); 2845 if (r) 2846 return r; 2847 2848 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 2849 ring = &adev->gfx.compute_ring[i + xcc_id * adev->gfx.num_compute_rings]; 2850 r = amdgpu_ring_test_helper(ring); 2851 if (r) 2852 return r; 2853 } 2854 } 2855 2856 return 0; 2857 } 2858 2859 static int gfx_v12_1_cp_resume(struct amdgpu_device *adev) 2860 { 2861 int num_xcc, num_xcp, num_xcc_per_xcp; 2862 uint16_t xcc_mask; 2863 int r = 0; 2864 2865 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 2866 if (amdgpu_sriov_vf(adev)) { 2867 enum amdgpu_gfx_partition mode; 2868 2869 mode = amdgpu_xcp_query_partition_mode(adev->xcp_mgr, 2870 AMDGPU_XCP_FL_NONE); 2871 if (mode == AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) 2872 return -EINVAL; 2873 if (adev->gfx.funcs && 2874 adev->gfx.funcs->get_xccs_per_xcp) { 2875 num_xcc_per_xcp = adev->gfx.funcs->get_xccs_per_xcp(adev); 2876 adev->gfx.num_xcc_per_xcp = num_xcc_per_xcp; 2877 num_xcp = num_xcc / num_xcc_per_xcp; 2878 } else { 2879 return -EINVAL; 2880 } 2881 r = amdgpu_xcp_init(adev->xcp_mgr, num_xcp, mode); 2882 2883 } else { 2884 if (amdgpu_xcp_query_partition_mode(adev->xcp_mgr, 2885 AMDGPU_XCP_FL_NONE) == 2886 AMDGPU_UNKNOWN_COMPUTE_PARTITION_MODE) 2887 r = amdgpu_xcp_switch_partition_mode(adev->xcp_mgr, 2888 amdgpu_user_partt_mode); 2889 } 2890 2891 if (r) 2892 return r; 2893 2894 xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0); 2895 2896 return gfx_v12_1_xcc_cp_resume(adev, xcc_mask); 2897 } 2898 2899 static int gfx_v12_1_gfxhub_enable(struct amdgpu_device *adev) 2900 { 2901 int r, i; 2902 bool value; 2903 2904 r = adev->gfxhub.funcs->gart_enable(adev); 2905 if (r) 2906 return r; 2907 2908 value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ? 2909 false : true; 2910 2911 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 2912 /* TODO investigate why TLB flush is needed, 2913 * are we missing a flush somewhere else? */ 2914 for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 2915 if (AMDGPU_IS_GFXHUB(i)) 2916 adev->gmc.gmc_funcs->flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(i), 0); 2917 } 2918 2919 return 0; 2920 } 2921 2922 static int get_gb_addr_config(struct amdgpu_device *adev) 2923 { 2924 u32 gb_addr_config; 2925 2926 gb_addr_config = RREG32_SOC15(GC, GET_INST(GC, 0), regGB_ADDR_CONFIG_READ); 2927 if (gb_addr_config == 0) 2928 return -EINVAL; 2929 2930 adev->gfx.config.gb_addr_config_fields.num_pkrs = 2931 1 << REG_GET_FIELD(gb_addr_config, GB_ADDR_CONFIG_READ, NUM_PKRS); 2932 2933 adev->gfx.config.gb_addr_config = gb_addr_config; 2934 2935 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 << 2936 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 2937 GB_ADDR_CONFIG_READ, NUM_PIPES); 2938 2939 adev->gfx.config.max_tile_pipes = 2940 adev->gfx.config.gb_addr_config_fields.num_pipes; 2941 2942 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 << 2943 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 2944 GB_ADDR_CONFIG_READ, MAX_COMPRESSED_FRAGS); 2945 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 << 2946 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 2947 GB_ADDR_CONFIG_READ, NUM_RB_PER_SE); 2948 adev->gfx.config.gb_addr_config_fields.num_se = 1 << 2949 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 2950 GB_ADDR_CONFIG_READ, NUM_SHADER_ENGINES); 2951 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 + 2952 REG_GET_FIELD(adev->gfx.config.gb_addr_config, 2953 GB_ADDR_CONFIG_READ, PIPE_INTERLEAVE_SIZE)); 2954 2955 return 0; 2956 } 2957 2958 static void gfx_v12_1_xcc_disable_gpa_mode(struct amdgpu_device *adev, 2959 int xcc_id) 2960 { 2961 uint32_t data; 2962 2963 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG); 2964 data |= CPC_PSP_DEBUG__GPA_OVERRIDE_MASK; 2965 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPC_PSP_DEBUG, data); 2966 2967 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG); 2968 data |= CPG_PSP_DEBUG__GPA_OVERRIDE_MASK; 2969 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCPG_PSP_DEBUG, data); 2970 } 2971 2972 static void gfx_v12_1_xcc_enable_atomics(struct amdgpu_device *adev, 2973 int xcc_id) 2974 { 2975 uint32_t data; 2976 2977 /* Set the TCP UTCL0 register to enable atomics */ 2978 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_UTCL0_CNTL1); 2979 data = REG_SET_FIELD(data, TCP_UTCL0_CNTL1, ATOMIC_REQUESTER_EN, 0x1); 2980 2981 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_UTCL0_CNTL1, data); 2982 } 2983 2984 static void gfx_v12_1_xcc_disable_burst(struct amdgpu_device *adev, 2985 int xcc_id) 2986 { 2987 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGL1_DRAM_BURST_CTRL, 0xf); 2988 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGLARB_DRAM_BURST_CTRL, 0xf); 2989 } 2990 2991 static void gfx_v12_1_xcc_disable_early_write_ack(struct amdgpu_device *adev, 2992 int xcc_id) 2993 { 2994 uint32_t data; 2995 2996 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL3); 2997 data = REG_SET_FIELD(data, TCP_CNTL3, DISABLE_EARLY_WRITE_ACK, 0x1); 2998 2999 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL3, data); 3000 } 3001 3002 static void gfx_v12_1_xcc_disable_tcp_spill_cache(struct amdgpu_device *adev, 3003 int xcc_id) 3004 { 3005 uint32_t data; 3006 3007 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL); 3008 data = REG_SET_FIELD(data, TCP_CNTL, TCP_SPILL_CACHE_DISABLE, 0x1); 3009 3010 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regTCP_CNTL, data); 3011 } 3012 3013 static void gfx_v12_1_init_golden_registers(struct amdgpu_device *adev) 3014 { 3015 int i; 3016 3017 for (i = 0; i < NUM_XCC(adev->gfx.xcc_mask); i++) { 3018 gfx_v12_1_xcc_disable_burst(adev, i); 3019 gfx_v12_1_xcc_enable_atomics(adev, i); 3020 gfx_v12_1_xcc_disable_early_write_ack(adev, i); 3021 gfx_v12_1_xcc_disable_tcp_spill_cache(adev, i); 3022 } 3023 } 3024 3025 static int gfx_v12_1_set_userq_eop_interrupts(struct amdgpu_device *adev, 3026 bool enable) 3027 { 3028 unsigned int irq_type; 3029 int m, p, r; 3030 3031 if (!adev->gfx.disable_kq) 3032 return 0; 3033 3034 for (m = 0; m < adev->gfx.mec.num_mec; ++m) { 3035 for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { 3036 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 3037 + (m * adev->gfx.mec.num_pipe_per_mec) 3038 + p; 3039 if (enable) 3040 r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, irq_type); 3041 else 3042 r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 3043 if (r) { 3044 if (!enable) 3045 return r; 3046 goto err_unwind; 3047 } 3048 } 3049 } 3050 3051 return 0; 3052 3053 err_unwind: 3054 for (p--; p >= 0; p--) { 3055 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 3056 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 3057 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 3058 } 3059 for (m--; m >= 0; m--) { 3060 for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { 3061 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP 3062 + (m * adev->gfx.mec.num_pipe_per_mec) + p; 3063 amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); 3064 } 3065 } 3066 return r; 3067 } 3068 3069 static int gfx_v12_1_hw_init(struct amdgpu_ip_block *ip_block) 3070 { 3071 int r, i, num_xcc; 3072 struct amdgpu_device *adev = ip_block->adev; 3073 3074 if (adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) { 3075 /* rlc autoload firmware */ 3076 r = gfx_v12_1_rlc_backdoor_autoload_enable(adev); 3077 if (r) 3078 return r; 3079 } else { 3080 if (adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) { 3081 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 3082 3083 if (adev->gfx.imu.funcs) { 3084 if (adev->gfx.imu.funcs->load_microcode) 3085 adev->gfx.imu.funcs->load_microcode(adev); 3086 } 3087 3088 for (i = 0; i < num_xcc; i++) { 3089 /* disable gpa mode in backdoor loading */ 3090 gfx_v12_1_xcc_disable_gpa_mode(adev, i); 3091 } 3092 } 3093 } 3094 3095 if ((adev->firmware.load_type == AMDGPU_FW_LOAD_RLC_BACKDOOR_AUTO) || 3096 (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP)) { 3097 r = gfx_v12_1_wait_for_rlc_autoload_complete(adev); 3098 if (r) { 3099 dev_err(adev->dev, "(%d) failed to wait rlc autoload complete\n", r); 3100 return r; 3101 } 3102 } 3103 3104 adev->gfx.is_poweron = true; 3105 3106 if (get_gb_addr_config(adev)) 3107 DRM_WARN("Invalid gb_addr_config !\n"); 3108 3109 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) 3110 gfx_v12_1_config_gfx_rs64(adev); 3111 3112 r = gfx_v12_1_gfxhub_enable(adev); 3113 if (r) 3114 return r; 3115 3116 gfx_v12_1_init_golden_registers(adev); 3117 3118 gfx_v12_1_constants_init(adev); 3119 3120 if (adev->nbio.funcs->gc_doorbell_init) 3121 adev->nbio.funcs->gc_doorbell_init(adev); 3122 3123 r = gfx_v12_1_rlc_resume(adev); 3124 if (r) 3125 return r; 3126 3127 /* 3128 * init golden registers and rlc resume may override some registers, 3129 * reconfig them here 3130 */ 3131 gfx_v12_1_tcp_harvest(adev); 3132 3133 r = gfx_v12_1_cp_resume(adev); 3134 if (r) 3135 return r; 3136 3137 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0); 3138 if (r) 3139 return r; 3140 3141 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0); 3142 if (r) 3143 goto err_priv_inst; 3144 3145 r = gfx_v12_1_set_userq_eop_interrupts(adev, true); 3146 if (r) 3147 goto err_userq_eop; 3148 3149 return 0; 3150 3151 err_userq_eop: 3152 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3153 err_priv_inst: 3154 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3155 return r; 3156 } 3157 3158 static void gfx_v12_1_xcc_fini(struct amdgpu_device *adev, 3159 int xcc_id) 3160 { 3161 uint32_t tmp; 3162 3163 if (!adev->no_hw_access) { 3164 if (amdgpu_gfx_disable_kcq(adev, xcc_id)) 3165 DRM_ERROR("KCQ disable failed\n"); 3166 3167 amdgpu_mes_kiq_hw_fini(adev, xcc_id); 3168 } 3169 3170 if (amdgpu_sriov_vf(adev)) { 3171 /* Program KIQ position of RLC_CP_SCHEDULERS during destroy */ 3172 tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS); 3173 tmp &= 0xffffff00; 3174 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CP_SCHEDULERS, tmp); 3175 } 3176 gfx_v12_1_xcc_cp_compute_enable(adev, false, xcc_id); 3177 gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, false, xcc_id); 3178 } 3179 3180 static int gfx_v12_1_hw_fini(struct amdgpu_ip_block *ip_block) 3181 { 3182 struct amdgpu_device *adev = ip_block->adev; 3183 int i, num_xcc; 3184 3185 gfx_v12_1_set_userq_eop_interrupts(adev, false); 3186 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); 3187 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); 3188 3189 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 3190 for (i = 0; i < num_xcc; i++) { 3191 gfx_v12_1_xcc_fini(adev, i); 3192 } 3193 3194 adev->gfxhub.funcs->gart_disable(adev); 3195 3196 adev->gfx.is_poweron = false; 3197 3198 return 0; 3199 } 3200 3201 static int gfx_v12_1_suspend(struct amdgpu_ip_block *ip_block) 3202 { 3203 return gfx_v12_1_hw_fini(ip_block); 3204 } 3205 3206 static int gfx_v12_1_resume(struct amdgpu_ip_block *ip_block) 3207 { 3208 return gfx_v12_1_hw_init(ip_block); 3209 } 3210 3211 static bool gfx_v12_1_is_idle(struct amdgpu_ip_block *ip_block) 3212 { 3213 struct amdgpu_device *adev = ip_block->adev; 3214 int i, num_xcc; 3215 3216 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 3217 for (i = 0; i < num_xcc; i++) { 3218 if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), 3219 regGRBM_STATUS), GRBM_STATUS, GUI_ACTIVE)) 3220 return false; 3221 } 3222 return true; 3223 } 3224 3225 static int gfx_v12_1_wait_for_idle(struct amdgpu_ip_block *ip_block) 3226 { 3227 unsigned i; 3228 struct amdgpu_device *adev = ip_block->adev; 3229 3230 for (i = 0; i < adev->usec_timeout; i++) { 3231 if (gfx_v12_1_is_idle(ip_block)) 3232 return 0; 3233 udelay(1); 3234 } 3235 return -ETIMEDOUT; 3236 } 3237 3238 static uint64_t gfx_v12_1_get_gpu_clock_counter(struct amdgpu_device *adev) 3239 { 3240 uint64_t clock = 0; 3241 3242 if (adev->smuio.funcs && 3243 adev->smuio.funcs->get_gpu_clock_counter) 3244 clock = adev->smuio.funcs->get_gpu_clock_counter(adev); 3245 else 3246 dev_warn(adev->dev, "query gpu clock counter is not supported\n"); 3247 3248 return clock; 3249 } 3250 3251 static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block) 3252 { 3253 struct amdgpu_device *adev = ip_block->adev; 3254 3255 3256 switch (amdgpu_user_queue) { 3257 case -1: 3258 default: 3259 adev->gfx.disable_kq = true; 3260 adev->gfx.disable_uq = true; 3261 break; 3262 case 0: 3263 adev->gfx.disable_kq = false; 3264 adev->gfx.disable_uq = true; 3265 break; 3266 } 3267 3268 adev->gfx.funcs = &gfx_v12_1_gfx_funcs; 3269 3270 if (adev->gfx.disable_kq) 3271 adev->gfx.num_compute_rings = 0; 3272 else 3273 adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), 3274 AMDGPU_MAX_COMPUTE_RINGS); 3275 3276 gfx_v12_1_set_kiq_pm4_funcs(adev); 3277 gfx_v12_1_set_ring_funcs(adev); 3278 gfx_v12_1_set_irq_funcs(adev); 3279 gfx_v12_1_set_rlc_funcs(adev); 3280 gfx_v12_1_set_mqd_funcs(adev); 3281 gfx_v12_1_set_imu_funcs(adev); 3282 3283 gfx_v12_1_init_rlcg_reg_access_ctrl(adev); 3284 3285 amdgpu_init_rlc_reg_funcs(adev); 3286 3287 return gfx_v12_1_init_microcode(adev); 3288 } 3289 3290 static bool gfx_v12_1_is_rlc_enabled(struct amdgpu_device *adev) 3291 { 3292 uint32_t rlc_cntl; 3293 3294 /* if RLC is not enabled, do nothing */ 3295 rlc_cntl = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CNTL); 3296 return (REG_GET_FIELD(rlc_cntl, RLC_CNTL, RLC_ENABLE_F32)) ? true : false; 3297 } 3298 3299 static void gfx_v12_1_xcc_set_safe_mode(struct amdgpu_device *adev, 3300 int xcc_id) 3301 { 3302 uint32_t data; 3303 unsigned i; 3304 3305 data = RLC_SAFE_MODE__CMD_MASK; 3306 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT); 3307 3308 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SAFE_MODE, data); 3309 3310 /* wait for RLC_SAFE_MODE */ 3311 for (i = 0; i < adev->usec_timeout; i++) { 3312 if (!REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, xcc_id), 3313 regRLC_SAFE_MODE), RLC_SAFE_MODE, CMD)) 3314 break; 3315 udelay(1); 3316 } 3317 } 3318 3319 static void gfx_v12_1_xcc_unset_safe_mode(struct amdgpu_device *adev, 3320 int xcc_id) 3321 { 3322 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 3323 regRLC_SAFE_MODE, RLC_SAFE_MODE__CMD_MASK); 3324 } 3325 3326 static void gfx_v12_1_update_perf_clk(struct amdgpu_device *adev, 3327 bool enable) 3328 { 3329 int i, num_xcc; 3330 3331 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 3332 for (i = 0; i < num_xcc; i++) 3333 gfx_v12_1_xcc_update_perf_clk(adev, enable, i); 3334 } 3335 3336 static void gfx_v12_1_update_spm_vmid(struct amdgpu_device *adev, 3337 int xcc_id, 3338 struct amdgpu_ring *ring, 3339 unsigned vmid) 3340 { 3341 u32 reg, data; 3342 3343 reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL); 3344 if (amdgpu_sriov_is_pp_one_vf(adev)) 3345 data = RREG32_NO_KIQ(reg); 3346 else 3347 data = RREG32(reg); 3348 3349 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK; 3350 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT; 3351 3352 if (amdgpu_sriov_is_pp_one_vf(adev)) 3353 WREG32_SOC15_NO_KIQ(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data); 3354 else 3355 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL, data); 3356 3357 if (ring 3358 && amdgpu_sriov_is_pp_one_vf(adev) 3359 && ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) 3360 || (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE))) { 3361 uint32_t reg = SOC15_REG_OFFSET(GC, GET_INST(GC, xcc_id), regRLC_SPM_MC_CNTL); 3362 amdgpu_ring_emit_wreg(ring, reg, data); 3363 } 3364 } 3365 3366 static const struct amdgpu_rlc_funcs gfx_v12_1_rlc_funcs = { 3367 .is_rlc_enabled = gfx_v12_1_is_rlc_enabled, 3368 .set_safe_mode = gfx_v12_1_xcc_set_safe_mode, 3369 .unset_safe_mode = gfx_v12_1_xcc_unset_safe_mode, 3370 .init = gfx_v12_1_rlc_init, 3371 .get_csb_size = gfx_v12_1_get_csb_size, 3372 .get_csb_buffer = gfx_v12_1_get_csb_buffer, 3373 .resume = gfx_v12_1_rlc_resume, 3374 .stop = gfx_v12_1_rlc_stop, 3375 .reset = gfx_v12_1_rlc_reset, 3376 .start = gfx_v12_1_rlc_start, 3377 .update_spm_vmid = gfx_v12_1_update_spm_vmid, 3378 }; 3379 3380 #if 0 3381 static void gfx_v12_cntl_power_gating(struct amdgpu_device *adev, bool enable) 3382 { 3383 /* TODO */ 3384 } 3385 3386 static void gfx_v12_cntl_pg(struct amdgpu_device *adev, bool enable) 3387 { 3388 /* TODO */ 3389 } 3390 #endif 3391 3392 static int gfx_v12_1_set_powergating_state(struct amdgpu_ip_block *ip_block, 3393 enum amd_powergating_state state) 3394 { 3395 struct amdgpu_device *adev = ip_block->adev; 3396 bool enable = (state == AMD_PG_STATE_GATE); 3397 3398 if (amdgpu_sriov_vf(adev)) 3399 return 0; 3400 3401 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { 3402 case IP_VERSION(12, 1, 0): 3403 amdgpu_gfx_off_ctrl(adev, enable); 3404 break; 3405 default: 3406 break; 3407 } 3408 3409 return 0; 3410 } 3411 3412 static void gfx_v12_1_xcc_update_coarse_grain_clock_gating(struct amdgpu_device *adev, 3413 bool enable, int xcc_id) 3414 { 3415 uint32_t def, data; 3416 3417 if (!(adev->cg_flags & 3418 (AMD_CG_SUPPORT_GFX_CGCG | 3419 AMD_CG_SUPPORT_GFX_CGLS | 3420 AMD_CG_SUPPORT_GFX_3D_CGCG | 3421 AMD_CG_SUPPORT_GFX_3D_CGLS))) 3422 return; 3423 3424 if (enable) { 3425 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), 3426 regRLC_CGTT_MGCG_OVERRIDE); 3427 3428 /* unset CGCG override */ 3429 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 3430 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK; 3431 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 3432 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK; 3433 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG || 3434 adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS) 3435 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK; 3436 3437 /* update CGCG override bits */ 3438 if (def != data) 3439 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 3440 regRLC_CGTT_MGCG_OVERRIDE, data); 3441 3442 /* enable cgcg FSM(0x0000363F) */ 3443 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL); 3444 3445 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) { 3446 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK; 3447 data |= (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) | 3448 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 3449 } 3450 3451 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) { 3452 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK; 3453 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) | 3454 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 3455 } 3456 3457 if (def != data) 3458 WREG32_SOC15(GC, GET_INST(GC, xcc_id), 3459 regRLC_CGCG_CGLS_CTRL, data); 3460 3461 /* set IDLE_POLL_COUNT(0x00900100) */ 3462 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL); 3463 3464 data &= ~CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK; 3465 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK; 3466 data |= (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) | 3467 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT); 3468 3469 if (def != data) 3470 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_RB_WPTR_POLL_CNTL, data); 3471 3472 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL); 3473 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_BUSY_INT_ENABLE, 1); 3474 data = REG_SET_FIELD(data, CP_INT_CNTL, CNTX_EMPTY_INT_ENABLE, 1); 3475 data = REG_SET_FIELD(data, CP_INT_CNTL, CMP_BUSY_INT_ENABLE, 1); 3476 data = REG_SET_FIELD(data, CP_INT_CNTL, GFX_IDLE_INT_ENABLE, 1); 3477 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_INT_CNTL, data); 3478 } else { 3479 /* Program RLC_CGCG_CGLS_CTRL */ 3480 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL); 3481 3482 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG) 3483 data &= ~RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK; 3484 3485 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS) 3486 data &= ~RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK; 3487 3488 if (def != data) 3489 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGCG_CGLS_CTRL, data); 3490 } 3491 } 3492 3493 static void gfx_v12_1_xcc_update_medium_grain_clock_gating(struct amdgpu_device *adev, 3494 bool enable, int xcc_id) 3495 { 3496 uint32_t data, def; 3497 if (!(adev->cg_flags & (AMD_CG_SUPPORT_GFX_MGCG | AMD_CG_SUPPORT_GFX_MGLS))) 3498 return; 3499 3500 /* It is disabled by HW by default */ 3501 if (enable) { 3502 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 3503 /* 1 - RLC_CGTT_MGCG_OVERRIDE */ 3504 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE); 3505 3506 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 3507 RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 3508 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 3509 3510 if (def != data) 3511 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data); 3512 } 3513 } else { 3514 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG) { 3515 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE); 3516 3517 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK | 3518 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK | 3519 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK); 3520 3521 if (def != data) 3522 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data); 3523 } 3524 } 3525 } 3526 3527 static void gfx_v12_1_xcc_update_repeater_fgcg(struct amdgpu_device *adev, 3528 bool enable, int xcc_id) 3529 { 3530 uint32_t def, data; 3531 3532 if (!(adev->cg_flags & AMD_CG_SUPPORT_REPEATER_FGCG)) 3533 return; 3534 3535 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE); 3536 3537 if (enable) 3538 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK | 3539 RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK); 3540 else 3541 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK | 3542 RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK; 3543 3544 if (def != data) 3545 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data); 3546 } 3547 3548 static void gfx_v12_1_xcc_update_sram_fgcg(struct amdgpu_device *adev, 3549 bool enable, int xcc_id) 3550 { 3551 uint32_t def, data; 3552 3553 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_FGCG)) 3554 return; 3555 3556 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE); 3557 3558 if (enable) 3559 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 3560 else 3561 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK; 3562 3563 if (def != data) 3564 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data); 3565 } 3566 3567 static void gfx_v12_1_xcc_update_perf_clk(struct amdgpu_device *adev, 3568 bool enable, int xcc_id) 3569 { 3570 uint32_t def, data; 3571 3572 if (!(adev->cg_flags & AMD_CG_SUPPORT_GFX_PERF_CLK)) 3573 return; 3574 3575 def = data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE); 3576 3577 if (enable) 3578 data &= ~RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 3579 else 3580 data |= RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK; 3581 3582 if (def != data) 3583 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regRLC_CGTT_MGCG_OVERRIDE, data); 3584 } 3585 3586 static int gfx_v12_1_xcc_update_gfx_clock_gating(struct amdgpu_device *adev, 3587 bool enable, int xcc_id) 3588 { 3589 amdgpu_gfx_rlc_enter_safe_mode(adev, xcc_id); 3590 3591 gfx_v12_1_xcc_update_coarse_grain_clock_gating(adev, enable, xcc_id); 3592 3593 gfx_v12_1_xcc_update_medium_grain_clock_gating(adev, enable, xcc_id); 3594 3595 gfx_v12_1_xcc_update_repeater_fgcg(adev, enable, xcc_id); 3596 3597 gfx_v12_1_xcc_update_sram_fgcg(adev, enable, xcc_id); 3598 3599 gfx_v12_1_xcc_update_perf_clk(adev, enable, xcc_id); 3600 3601 if (adev->cg_flags & 3602 (AMD_CG_SUPPORT_GFX_MGCG | 3603 AMD_CG_SUPPORT_GFX_CGLS | 3604 AMD_CG_SUPPORT_GFX_CGCG | 3605 AMD_CG_SUPPORT_GFX_3D_CGCG | 3606 AMD_CG_SUPPORT_GFX_3D_CGLS)) 3607 gfx_v12_1_xcc_enable_gui_idle_interrupt(adev, enable, xcc_id); 3608 3609 amdgpu_gfx_rlc_exit_safe_mode(adev, xcc_id); 3610 3611 return 0; 3612 } 3613 3614 static int gfx_v12_1_set_clockgating_state(struct amdgpu_ip_block *ip_block, 3615 enum amd_clockgating_state state) 3616 { 3617 struct amdgpu_device *adev = ip_block->adev; 3618 int i, num_xcc; 3619 3620 if (amdgpu_sriov_vf(adev)) 3621 return 0; 3622 3623 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 3624 switch (adev->ip_versions[GC_HWIP][0]) { 3625 case IP_VERSION(12, 1, 0): 3626 for (i = 0; i < num_xcc; i++) 3627 gfx_v12_1_xcc_update_gfx_clock_gating(adev, 3628 state == AMD_CG_STATE_GATE, i); 3629 break; 3630 default: 3631 break; 3632 } 3633 3634 return 0; 3635 } 3636 3637 static void gfx_v12_1_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64 *flags) 3638 { 3639 struct amdgpu_device *adev = ip_block->adev; 3640 int data; 3641 3642 /* AMD_CG_SUPPORT_GFX_MGCG */ 3643 data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGTT_MGCG_OVERRIDE); 3644 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK)) 3645 *flags |= AMD_CG_SUPPORT_GFX_MGCG; 3646 3647 /* AMD_CG_SUPPORT_REPEATER_FGCG */ 3648 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK)) 3649 *flags |= AMD_CG_SUPPORT_REPEATER_FGCG; 3650 3651 /* AMD_CG_SUPPORT_GFX_FGCG */ 3652 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK)) 3653 *flags |= AMD_CG_SUPPORT_GFX_FGCG; 3654 3655 /* AMD_CG_SUPPORT_GFX_PERF_CLK */ 3656 if (!(data & RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK)) 3657 *flags |= AMD_CG_SUPPORT_GFX_PERF_CLK; 3658 3659 /* AMD_CG_SUPPORT_GFX_CGCG */ 3660 data = RREG32_SOC15(GC, GET_INST(GC, 0), regRLC_CGCG_CGLS_CTRL); 3661 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK) 3662 *flags |= AMD_CG_SUPPORT_GFX_CGCG; 3663 3664 /* AMD_CG_SUPPORT_GFX_CGLS */ 3665 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK) 3666 *flags |= AMD_CG_SUPPORT_GFX_CGLS; 3667 } 3668 3669 static u64 gfx_v12_1_ring_get_rptr_compute(struct amdgpu_ring *ring) 3670 { 3671 /* gfx12 hardware is 32bit rptr */ 3672 return *(uint32_t *)ring->rptr_cpu_addr; 3673 } 3674 3675 static u64 gfx_v12_1_ring_get_wptr_compute(struct amdgpu_ring *ring) 3676 { 3677 struct amdgpu_device *adev = ring->adev; 3678 u64 wptr; 3679 3680 /* XXX check if swapping is necessary on BE */ 3681 if (ring->use_doorbell) { 3682 wptr = atomic64_read((atomic64_t *)ring->wptr_cpu_addr); 3683 } else { 3684 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__); 3685 wptr = 0; 3686 } 3687 return wptr; 3688 } 3689 3690 static void gfx_v12_1_ring_set_wptr_compute(struct amdgpu_ring *ring) 3691 { 3692 struct amdgpu_device *adev = ring->adev; 3693 3694 /* XXX check if swapping is necessary on BE */ 3695 if (ring->use_doorbell) { 3696 atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 3697 ring->wptr); 3698 WDOORBELL64(ring->doorbell_index, ring->wptr); 3699 } else { 3700 dev_warn_once(adev->dev, "%s requires doorbell!\n", __func__); 3701 } 3702 } 3703 3704 static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring, 3705 struct amdgpu_job *job, 3706 struct amdgpu_ib *ib, 3707 uint32_t flags) 3708 { 3709 unsigned vmid = AMDGPU_JOB_GET_VMID(job); 3710 u32 control = PACKET3_INDIRECT_BUFFER__VALID(1) | ib->length_dw | (vmid << 24); 3711 3712 /* Currently, there is a high possibility to get wave ID mismatch 3713 * between ME and GDS, leading to a hw deadlock, because ME generates 3714 * different wave IDs than the GDS expects. This situation happens 3715 * randomly when at least 5 compute pipes use GDS ordered append. 3716 * The wave IDs generated by ME are also wrong after suspend/resume. 3717 * Those are probably bugs somewhere else in the kernel driver. 3718 * 3719 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and 3720 * GDS to 0 for this ring (me/pipe). 3721 */ 3722 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) { 3723 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1)); 3724 amdgpu_ring_write(ring, regGDS_COMPUTE_MAX_WAVE_ID); 3725 } 3726 3727 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2)); 3728 WARN_ON(ib->gpu_addr & 0x3); /* Dword align */ 3729 amdgpu_ring_write(ring, 3730 #ifdef __BIG_ENDIAN 3731 (2 << 0) | 3732 #endif 3733 lower_32_bits(ib->gpu_addr)); 3734 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr)); 3735 amdgpu_ring_write(ring, control); 3736 } 3737 3738 static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, 3739 u64 seq, unsigned flags) 3740 { 3741 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT; 3742 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT; 3743 3744 /* RELEASE_MEM - flush caches, send int */ 3745 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6)); 3746 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__GCR_SEQ(1) | 3747 PACKET3_RELEASE_MEM__GCR_GLV_WB | 3748 PACKET3_RELEASE_MEM__GCR_GL2_WB | 3749 PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(2) | 3750 PACKET3_RELEASE_MEM__TEMPORAL(3) | 3751 PACKET3_RELEASE_MEM__EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | 3752 PACKET3_RELEASE_MEM__EVENT_INDEX(5))); 3753 amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__DATA_SEL(write64bit ? 2 : 1) | 3754 PACKET3_RELEASE_MEM__INT_SEL(int_sel ? 2 : 0))); 3755 3756 /* 3757 * the address should be Qword aligned if 64bit write, Dword 3758 * aligned if only send 32bit data low (discard data high) 3759 */ 3760 if (write64bit) 3761 WARN_ON(addr & 0x7); 3762 else 3763 WARN_ON(addr & 0x3); 3764 amdgpu_ring_write(ring, lower_32_bits(addr)); 3765 amdgpu_ring_write(ring, upper_32_bits(addr)); 3766 amdgpu_ring_write(ring, lower_32_bits(seq)); 3767 amdgpu_ring_write(ring, upper_32_bits(seq)); 3768 amdgpu_ring_write(ring, 0); 3769 } 3770 3771 static void gfx_v12_1_ring_emit_pipeline_sync(struct amdgpu_ring *ring) 3772 { 3773 uint32_t seq = ring->fence_drv.sync_seq; 3774 uint64_t addr = ring->fence_drv.gpu_addr; 3775 3776 gfx_v12_1_wait_reg_mem(ring, 0, 1, 0, lower_32_bits(addr), 3777 upper_32_bits(addr), seq, 0xffffffff, 4); 3778 } 3779 3780 static void gfx_v12_1_ring_invalidate_tlbs(struct amdgpu_ring *ring, 3781 uint16_t pasid, uint32_t flush_type, 3782 bool all_hub, uint8_t dst_sel) 3783 { 3784 amdgpu_ring_write(ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0)); 3785 amdgpu_ring_write(ring, 3786 PACKET3_INVALIDATE_TLBS_DST_SEL(dst_sel) | 3787 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) | 3788 PACKET3_INVALIDATE_TLBS_PASID(pasid) | 3789 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type)); 3790 } 3791 3792 static void gfx_v12_1_ring_emit_vm_flush(struct amdgpu_ring *ring, 3793 unsigned vmid, uint64_t pd_addr) 3794 { 3795 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 3796 3797 /* compute doesn't have PFP */ 3798 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { 3799 /* sync PFP to ME, otherwise we might get invalid PFP reads */ 3800 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0)); 3801 amdgpu_ring_write(ring, 0x0); 3802 } 3803 } 3804 3805 static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr, 3806 u64 seq, unsigned int flags) 3807 { 3808 struct amdgpu_device *adev = ring->adev; 3809 3810 /* write fence seq to the "addr" */ 3811 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3812 amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1))); 3813 amdgpu_ring_write(ring, lower_32_bits(addr)); 3814 amdgpu_ring_write(ring, upper_32_bits(addr)); 3815 amdgpu_ring_write(ring, lower_32_bits(seq)); 3816 3817 if (flags & AMDGPU_FENCE_FLAG_INT) { 3818 /* set register to trigger INT */ 3819 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3820 amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(0) | PACKET3_WRITE_DATA__WR_CONFIRM(1))); 3821 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS)); 3822 amdgpu_ring_write(ring, 0); 3823 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */ 3824 } 3825 } 3826 3827 static void gfx_v12_1_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg, 3828 uint32_t reg_val_offs) 3829 { 3830 struct amdgpu_device *adev = ring->adev; 3831 3832 reg = soc_v1_0_normalize_xcc_reg_offset(reg); 3833 3834 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4)); 3835 amdgpu_ring_write(ring, 0 | /* src: register*/ 3836 (5 << 8) | /* dst: memory */ 3837 (1 << 20)); /* write confirm */ 3838 amdgpu_ring_write(ring, reg); 3839 amdgpu_ring_write(ring, 0); 3840 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr + 3841 reg_val_offs * 4)); 3842 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr + 3843 reg_val_offs * 4)); 3844 } 3845 3846 static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring, 3847 uint32_t reg, 3848 uint32_t val) 3849 { 3850 uint32_t cmd = 0; 3851 3852 reg = soc_v1_0_normalize_xcc_reg_offset(reg); 3853 3854 switch (ring->funcs->type) { 3855 case AMDGPU_RING_TYPE_KIQ: 3856 cmd = (1 << 16); /* no inc addr */ 3857 break; 3858 default: 3859 cmd = PACKET3_WRITE_DATA__WR_CONFIRM(1); 3860 break; 3861 } 3862 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3)); 3863 amdgpu_ring_write(ring, cmd); 3864 amdgpu_ring_write(ring, reg); 3865 amdgpu_ring_write(ring, 0); 3866 amdgpu_ring_write(ring, val); 3867 } 3868 3869 static void gfx_v12_1_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 3870 uint32_t val, uint32_t mask) 3871 { 3872 gfx_v12_1_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20); 3873 } 3874 3875 static void gfx_v12_1_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring, 3876 uint32_t reg0, uint32_t reg1, 3877 uint32_t ref, uint32_t mask) 3878 { 3879 gfx_v12_1_wait_reg_mem(ring, 0, 0, 1, reg0, reg1, 3880 ref, mask, 0x20); 3881 } 3882 3883 static void gfx_v12_1_xcc_set_compute_eop_interrupt_state(struct amdgpu_device *adev, 3884 int me, int pipe, 3885 enum amdgpu_interrupt_state state, 3886 int xcc_id) 3887 { 3888 u32 mec_int_cntl, mec_int_cntl_reg; 3889 3890 /* 3891 * amdgpu controls only the first MEC. That's why this function only 3892 * handles the setting of interrupts for this specific MEC. All other 3893 * pipes' interrupts are set by amdkfd. 3894 */ 3895 3896 if (me == 1) { 3897 switch (pipe) { 3898 case 0: 3899 mec_int_cntl_reg = SOC15_REG_OFFSET( 3900 GC, GET_INST(GC, xcc_id), 3901 regCP_ME1_PIPE0_INT_CNTL); 3902 break; 3903 case 1: 3904 mec_int_cntl_reg = SOC15_REG_OFFSET( 3905 GC, GET_INST(GC, xcc_id), 3906 regCP_ME1_PIPE1_INT_CNTL); 3907 break; 3908 case 2: 3909 mec_int_cntl_reg = SOC15_REG_OFFSET( 3910 GC, GET_INST(GC, xcc_id), 3911 regCP_ME1_PIPE2_INT_CNTL); 3912 break; 3913 case 3: 3914 mec_int_cntl_reg = SOC15_REG_OFFSET( 3915 GC, GET_INST(GC, xcc_id), 3916 regCP_ME1_PIPE3_INT_CNTL); 3917 break; 3918 default: 3919 DRM_DEBUG("invalid pipe %d\n", pipe); 3920 return; 3921 } 3922 } else { 3923 DRM_DEBUG("invalid me %d\n", me); 3924 return; 3925 } 3926 3927 switch (state) { 3928 case AMDGPU_IRQ_STATE_DISABLE: 3929 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id); 3930 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 3931 TIME_STAMP_INT_ENABLE, 0); 3932 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 3933 GENERIC0_INT_ENABLE, 0); 3934 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id); 3935 break; 3936 case AMDGPU_IRQ_STATE_ENABLE: 3937 mec_int_cntl = RREG32_XCC(mec_int_cntl_reg, xcc_id); 3938 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 3939 TIME_STAMP_INT_ENABLE, 1); 3940 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL, 3941 GENERIC0_INT_ENABLE, 1); 3942 WREG32_XCC(mec_int_cntl_reg, mec_int_cntl, xcc_id); 3943 break; 3944 default: 3945 break; 3946 } 3947 } 3948 3949 static int gfx_v12_1_set_eop_interrupt_state(struct amdgpu_device *adev, 3950 struct amdgpu_irq_src *src, 3951 unsigned type, 3952 enum amdgpu_interrupt_state state) 3953 { 3954 int i, num_xcc; 3955 3956 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 3957 for (i = 0; i < num_xcc; i++) { 3958 switch (type) { 3959 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP: 3960 gfx_v12_1_xcc_set_compute_eop_interrupt_state( 3961 adev, 1, 0, state, i); 3962 break; 3963 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP: 3964 gfx_v12_1_xcc_set_compute_eop_interrupt_state( 3965 adev, 1, 1, state, i); 3966 break; 3967 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP: 3968 gfx_v12_1_xcc_set_compute_eop_interrupt_state( 3969 adev, 1, 2, state, i); 3970 break; 3971 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP: 3972 gfx_v12_1_xcc_set_compute_eop_interrupt_state( 3973 adev, 1, 3, state, i); 3974 break; 3975 default: 3976 break; 3977 } 3978 } 3979 3980 return 0; 3981 } 3982 3983 static int gfx_v12_1_eop_irq(struct amdgpu_device *adev, 3984 struct amdgpu_irq_src *source, 3985 struct amdgpu_iv_entry *entry) 3986 { 3987 u32 doorbell_offset = entry->src_data[0]; 3988 u8 me_id, pipe_id, queue_id; 3989 struct amdgpu_ring *ring; 3990 int i, xcc_id; 3991 3992 DRM_DEBUG("IH: CP EOP\n"); 3993 3994 if (adev->enable_mes && doorbell_offset) { 3995 amdgpu_userq_process_fence_irq(adev, doorbell_offset); 3996 } else { 3997 me_id = (entry->ring_id & 0x0c) >> 2; 3998 pipe_id = (entry->ring_id & 0x03) >> 0; 3999 queue_id = (entry->ring_id & 0x70) >> 4; 4000 xcc_id = gfx_v12_1_ih_to_xcc_inst(adev, entry->node_id); 4001 4002 if (xcc_id == -EINVAL) 4003 return -EINVAL; 4004 4005 switch (me_id) { 4006 case 1: 4007 case 2: 4008 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4009 ring = &adev->gfx.compute_ring 4010 [i + 4011 xcc_id * adev->gfx.num_compute_rings]; 4012 /* Per-queue interrupt is supported for MEC starting from VI. 4013 * The interrupt can only be enabled/disabled per pipe instead 4014 * of per queue. 4015 */ 4016 if ((ring->me == me_id) && 4017 (ring->pipe == pipe_id) && 4018 (ring->queue == queue_id)) 4019 amdgpu_fence_process(ring); 4020 } 4021 break; 4022 default: 4023 dev_dbg(adev->dev, "Unexpected me %d in eop_irq\n", me_id); 4024 break; 4025 } 4026 } 4027 4028 return 0; 4029 } 4030 4031 static int gfx_v12_1_set_priv_reg_fault_state(struct amdgpu_device *adev, 4032 struct amdgpu_irq_src *source, 4033 unsigned type, 4034 enum amdgpu_interrupt_state state) 4035 { 4036 int i, num_xcc; 4037 4038 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 4039 switch (state) { 4040 case AMDGPU_IRQ_STATE_DISABLE: 4041 case AMDGPU_IRQ_STATE_ENABLE: 4042 for (i = 0; i < num_xcc; i++) 4043 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0, 4044 PRIV_REG_INT_ENABLE, 4045 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 4046 break; 4047 default: 4048 break; 4049 } 4050 4051 return 0; 4052 } 4053 4054 static int gfx_v12_1_set_priv_inst_fault_state(struct amdgpu_device *adev, 4055 struct amdgpu_irq_src *source, 4056 unsigned type, 4057 enum amdgpu_interrupt_state state) 4058 { 4059 int i, num_xcc; 4060 4061 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 4062 switch (state) { 4063 case AMDGPU_IRQ_STATE_DISABLE: 4064 case AMDGPU_IRQ_STATE_ENABLE: 4065 for (i = 0; i < num_xcc; i++) 4066 WREG32_FIELD15_PREREG(GC, GET_INST(GC, i), CP_INT_CNTL_RING0, 4067 PRIV_INSTR_INT_ENABLE, 4068 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0); 4069 break; 4070 default: 4071 break; 4072 } 4073 4074 return 0; 4075 } 4076 4077 static void gfx_v12_1_handle_priv_fault(struct amdgpu_device *adev, 4078 struct amdgpu_iv_entry *entry) 4079 { 4080 u8 me_id, pipe_id, queue_id; 4081 struct amdgpu_ring *ring; 4082 int i, xcc_id; 4083 4084 me_id = (entry->ring_id & 0x0c) >> 2; 4085 pipe_id = (entry->ring_id & 0x03) >> 0; 4086 queue_id = (entry->ring_id & 0x70) >> 4; 4087 xcc_id = gfx_v12_1_ih_to_xcc_inst(adev, entry->node_id); 4088 4089 if (xcc_id == -EINVAL) 4090 return; 4091 4092 if (!adev->gfx.disable_kq) { 4093 switch (me_id) { 4094 case 1: 4095 case 2: 4096 for (i = 0; i < adev->gfx.num_compute_rings; i++) { 4097 ring = &adev->gfx.compute_ring 4098 [i + 4099 xcc_id * adev->gfx.num_compute_rings]; 4100 if (ring->me == me_id && ring->pipe == pipe_id && 4101 ring->queue == queue_id) 4102 drm_sched_fault(&ring->sched); 4103 } 4104 break; 4105 default: 4106 dev_dbg(adev->dev, "Unexpected me %d in priv_fault\n", me_id); 4107 break; 4108 } 4109 } 4110 } 4111 4112 static int gfx_v12_1_priv_reg_irq(struct amdgpu_device *adev, 4113 struct amdgpu_irq_src *source, 4114 struct amdgpu_iv_entry *entry) 4115 { 4116 DRM_ERROR("Illegal register access in command stream\n"); 4117 gfx_v12_1_handle_priv_fault(adev, entry); 4118 return 0; 4119 } 4120 4121 static int gfx_v12_1_priv_inst_irq(struct amdgpu_device *adev, 4122 struct amdgpu_irq_src *source, 4123 struct amdgpu_iv_entry *entry) 4124 { 4125 DRM_ERROR("Illegal instruction in command stream\n"); 4126 gfx_v12_1_handle_priv_fault(adev, entry); 4127 return 0; 4128 } 4129 4130 static int gfx_v12_1_rlc_poison_irq(struct amdgpu_device *adev, 4131 struct amdgpu_irq_src *source, 4132 struct amdgpu_iv_entry *entry) 4133 { 4134 uint32_t rlc_fed_status = 0; 4135 uint32_t ras_blk = RAS_BLOCK_ID__GFX; 4136 struct ras_ih_info ih_info = {0}; 4137 int i, num_xcc; 4138 4139 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 4140 for (i = 0; i < num_xcc; i++) 4141 rlc_fed_status |= RREG32(SOC15_REG_OFFSET(GC, 4142 GET_INST(GC, i), regRLC_RLCS_FED_STATUS)); 4143 4144 if (!rlc_fed_status) 4145 return 0; 4146 4147 if (REG_GET_FIELD(rlc_fed_status, RLC_RLCS_FED_STATUS, SDMA0_FED_ERR) || 4148 REG_GET_FIELD(rlc_fed_status, RLC_RLCS_FED_STATUS, SDMA1_FED_ERR)) 4149 ras_blk = RAS_BLOCK_ID__SDMA; 4150 4151 dev_warn(adev->dev, "RLC %d FED IRQ\n", ras_blk); 4152 4153 ih_info.block = ras_blk; 4154 ih_info.reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; 4155 amdgpu_ras_mgr_dispatch_interrupt(adev, &ih_info); 4156 return 0; 4157 } 4158 4159 static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring) 4160 { 4161 const unsigned int gcr_cntl = 4162 PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(1) | 4163 PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(1) | 4164 PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(1) | 4165 PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(1) | 4166 PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(1) | 4167 PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(2); 4168 4169 /* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */ 4170 amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6)); 4171 amdgpu_ring_write(ring, 0); /* CP_COHER_CNTL */ 4172 amdgpu_ring_write(ring, 0xffffffff); /* CP_COHER_SIZE */ 4173 amdgpu_ring_write(ring, 0xffffff); /* CP_COHER_SIZE_HI */ 4174 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE */ 4175 amdgpu_ring_write(ring, 0); /* CP_COHER_BASE_HI */ 4176 amdgpu_ring_write(ring, 0x0000000A); /* POLL_INTERVAL */ 4177 amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ 4178 } 4179 4180 static const struct amd_ip_funcs gfx_v12_1_ip_funcs = { 4181 .name = "gfx_v12_1", 4182 .early_init = gfx_v12_1_early_init, 4183 .sw_init = gfx_v12_1_sw_init, 4184 .sw_fini = gfx_v12_1_sw_fini, 4185 .hw_init = gfx_v12_1_hw_init, 4186 .hw_fini = gfx_v12_1_hw_fini, 4187 .suspend = gfx_v12_1_suspend, 4188 .resume = gfx_v12_1_resume, 4189 .is_idle = gfx_v12_1_is_idle, 4190 .wait_for_idle = gfx_v12_1_wait_for_idle, 4191 .set_clockgating_state = gfx_v12_1_set_clockgating_state, 4192 .set_powergating_state = gfx_v12_1_set_powergating_state, 4193 .get_clockgating_state = gfx_v12_1_get_clockgating_state, 4194 .dump_ip_state = gfx_v12_1_ip_dump, 4195 .print_ip_state = gfx_v12_1_ip_print, 4196 }; 4197 4198 static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_compute = { 4199 .type = AMDGPU_RING_TYPE_COMPUTE, 4200 .align_mask = 0xff, 4201 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 4202 .support_64bit_ptrs = true, 4203 .get_rptr = gfx_v12_1_ring_get_rptr_compute, 4204 .get_wptr = gfx_v12_1_ring_get_wptr_compute, 4205 .set_wptr = gfx_v12_1_ring_set_wptr_compute, 4206 .emit_frame_size = 4207 7 + /* gfx_v12_1_ring_emit_pipeline_sync */ 4208 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 4209 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 4210 2 + /* gfx_v12_1_ring_emit_vm_flush */ 4211 8 + 8 + 8 + /* gfx_v12_1_ring_emit_fence x3 for user fence, vm fence */ 4212 8, /* gfx_v12_1_emit_mem_sync */ 4213 .emit_ib_size = 7, /* gfx_v12_1_ring_emit_ib_compute */ 4214 .emit_ib = gfx_v12_1_ring_emit_ib_compute, 4215 .emit_fence = gfx_v12_1_ring_emit_fence, 4216 .emit_pipeline_sync = gfx_v12_1_ring_emit_pipeline_sync, 4217 .emit_vm_flush = gfx_v12_1_ring_emit_vm_flush, 4218 .test_ring = gfx_v12_1_ring_test_ring, 4219 .test_ib = gfx_v12_1_ring_test_ib, 4220 .insert_nop = amdgpu_ring_insert_nop, 4221 .pad_ib = amdgpu_ring_generic_pad_ib, 4222 .emit_wreg = gfx_v12_1_ring_emit_wreg, 4223 .emit_reg_wait = gfx_v12_1_ring_emit_reg_wait, 4224 .emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait, 4225 .emit_mem_sync = gfx_v12_1_emit_mem_sync, 4226 }; 4227 4228 static const struct amdgpu_ring_funcs gfx_v12_1_ring_funcs_kiq = { 4229 .type = AMDGPU_RING_TYPE_KIQ, 4230 .align_mask = 0xff, 4231 .nop = PACKET3(PACKET3_NOP, 0x3FFF), 4232 .support_64bit_ptrs = true, 4233 .get_rptr = gfx_v12_1_ring_get_rptr_compute, 4234 .get_wptr = gfx_v12_1_ring_get_wptr_compute, 4235 .set_wptr = gfx_v12_1_ring_set_wptr_compute, 4236 .emit_frame_size = 4237 7 + /* gfx_v12_1_ring_emit_pipeline_sync */ 4238 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 + 4239 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 + 4240 2 + /* gfx_v12_1_ring_emit_vm_flush */ 4241 8 + 8 + 8, /* gfx_v12_1_ring_emit_fence_kiq x3 for user fence, vm fence */ 4242 .emit_ib_size = 7, /* gfx_v12_1_ring_emit_ib_compute */ 4243 .emit_ib = gfx_v12_1_ring_emit_ib_compute, 4244 .emit_fence = gfx_v12_1_ring_emit_fence_kiq, 4245 .test_ring = gfx_v12_1_ring_test_ring, 4246 .test_ib = gfx_v12_1_ring_test_ib, 4247 .insert_nop = amdgpu_ring_insert_nop, 4248 .pad_ib = amdgpu_ring_generic_pad_ib, 4249 .emit_rreg = gfx_v12_1_ring_emit_rreg, 4250 .emit_wreg = gfx_v12_1_ring_emit_wreg, 4251 .emit_reg_wait = gfx_v12_1_ring_emit_reg_wait, 4252 .emit_reg_write_reg_wait = gfx_v12_1_ring_emit_reg_write_reg_wait, 4253 }; 4254 4255 static void gfx_v12_1_set_ring_funcs(struct amdgpu_device *adev) 4256 { 4257 int i, j, num_xcc; 4258 4259 num_xcc = NUM_XCC(adev->gfx.xcc_mask); 4260 for (i = 0; i < num_xcc; i++) { 4261 adev->gfx.kiq[i].ring.funcs = &gfx_v12_1_ring_funcs_kiq; 4262 4263 for (j = 0; j < adev->gfx.num_compute_rings; j++) 4264 adev->gfx.compute_ring[j + i * adev->gfx.num_compute_rings].funcs = 4265 &gfx_v12_1_ring_funcs_compute; 4266 } 4267 } 4268 4269 static const struct amdgpu_irq_src_funcs gfx_v12_1_eop_irq_funcs = { 4270 .set = gfx_v12_1_set_eop_interrupt_state, 4271 .process = gfx_v12_1_eop_irq, 4272 }; 4273 4274 static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_reg_irq_funcs = { 4275 .set = gfx_v12_1_set_priv_reg_fault_state, 4276 .process = gfx_v12_1_priv_reg_irq, 4277 }; 4278 4279 static const struct amdgpu_irq_src_funcs gfx_v12_1_priv_inst_irq_funcs = { 4280 .set = gfx_v12_1_set_priv_inst_fault_state, 4281 .process = gfx_v12_1_priv_inst_irq, 4282 }; 4283 4284 static const struct amdgpu_irq_src_funcs gfx_v12_1_rlc_poison_irq_funcs = { 4285 .process = gfx_v12_1_rlc_poison_irq, 4286 }; 4287 4288 static void gfx_v12_1_set_irq_funcs(struct amdgpu_device *adev) 4289 { 4290 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST; 4291 adev->gfx.eop_irq.funcs = &gfx_v12_1_eop_irq_funcs; 4292 4293 adev->gfx.priv_reg_irq.num_types = 1; 4294 adev->gfx.priv_reg_irq.funcs = &gfx_v12_1_priv_reg_irq_funcs; 4295 4296 adev->gfx.priv_inst_irq.num_types = 1; 4297 adev->gfx.priv_inst_irq.funcs = &gfx_v12_1_priv_inst_irq_funcs; 4298 4299 adev->gfx.rlc_poison_irq.num_types = 1; 4300 adev->gfx.rlc_poison_irq.funcs = &gfx_v12_1_rlc_poison_irq_funcs; 4301 } 4302 4303 static void gfx_v12_1_set_imu_funcs(struct amdgpu_device *adev) 4304 { 4305 if (adev->flags & AMD_IS_APU) 4306 adev->gfx.imu.mode = MISSION_MODE; 4307 else 4308 adev->gfx.imu.mode = DEBUG_MODE; 4309 if (!amdgpu_sriov_vf(adev)) 4310 adev->gfx.imu.funcs = &gfx_v12_1_imu_funcs; 4311 } 4312 4313 static void gfx_v12_1_set_rlc_funcs(struct amdgpu_device *adev) 4314 { 4315 adev->gfx.rlc.funcs = &gfx_v12_1_rlc_funcs; 4316 } 4317 4318 static void gfx_v12_1_set_mqd_funcs(struct amdgpu_device *adev) 4319 { 4320 /* set compute eng mqd */ 4321 adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size = 4322 sizeof(struct v12_1_compute_mqd); 4323 adev->mqds[AMDGPU_HW_IP_COMPUTE].init_mqd = 4324 gfx_v12_1_compute_mqd_init; 4325 } 4326 4327 static void gfx_v12_1_set_user_cu_inactive_bitmap_per_sh(struct amdgpu_device *adev, 4328 u32 bitmap, int xcc_id) 4329 { 4330 u32 data; 4331 4332 if (!bitmap) 4333 return; 4334 4335 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 4336 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 4337 4338 WREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG, data); 4339 } 4340 4341 static u32 gfx_v12_1_get_cu_active_bitmap_per_sh(struct amdgpu_device *adev, 4342 int xcc_id) 4343 { 4344 u32 data, mask; 4345 4346 data = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCC_GC_SHADER_ARRAY_CONFIG); 4347 data |= RREG32_SOC15(GC, GET_INST(GC, xcc_id), regGC_USER_SHADER_ARRAY_CONFIG); 4348 4349 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK; 4350 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT; 4351 4352 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh); 4353 4354 return (~data) & mask; 4355 } 4356 4357 static int gfx_v12_1_get_cu_info(struct amdgpu_device *adev, 4358 struct amdgpu_cu_info *cu_info) 4359 { 4360 int i, j, k, counter, xcc_id, active_cu_number = 0; 4361 u32 mask, bitmap; 4362 unsigned int disable_masks[2 * 2]; 4363 4364 if (!adev || !cu_info) 4365 return -EINVAL; 4366 4367 if (adev->gfx.config.max_shader_engines > 2 || 4368 adev->gfx.config.max_sh_per_se > 2) { 4369 dev_err(adev->dev, 4370 "Max SE (%d) and Max SA per SE (%d) is greater than expected\n", 4371 adev->gfx.config.max_shader_engines, 4372 adev->gfx.config.max_sh_per_se); 4373 return -EINVAL; 4374 } 4375 4376 amdgpu_gfx_parse_disable_cu(adev, disable_masks, 4377 adev->gfx.config.max_shader_engines, 4378 adev->gfx.config.max_sh_per_se); 4379 4380 mutex_lock(&adev->grbm_idx_mutex); 4381 for (xcc_id = 0; xcc_id < NUM_XCC(adev->gfx.xcc_mask); xcc_id++) { 4382 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) { 4383 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) { 4384 bitmap = i * adev->gfx.config.max_sh_per_se + j; 4385 if (!((gfx_v12_1_get_sa_active_bitmap(adev, xcc_id) >> bitmap) & 1)) 4386 continue; 4387 mask = 1; 4388 counter = 0; 4389 gfx_v12_1_xcc_select_se_sh(adev, i, j, 0xffffffff, xcc_id); 4390 gfx_v12_1_set_user_cu_inactive_bitmap_per_sh( 4391 adev, 4392 disable_masks[i * adev->gfx.config.max_sh_per_se + j], 4393 xcc_id); 4394 bitmap = gfx_v12_1_get_cu_active_bitmap_per_sh(adev, xcc_id); 4395 4396 cu_info->bitmap[xcc_id][i][j] = bitmap; 4397 4398 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k++) { 4399 if (bitmap & mask) 4400 counter++; 4401 4402 mask <<= 1; 4403 } 4404 active_cu_number += counter; 4405 } 4406 } 4407 gfx_v12_1_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, xcc_id); 4408 } 4409 mutex_unlock(&adev->grbm_idx_mutex); 4410 4411 cu_info->number = active_cu_number; 4412 cu_info->simd_per_cu = NUM_SIMD_PER_CU_GFX12_1; 4413 cu_info->lds_size = 320; 4414 4415 return 0; 4416 } 4417 4418 const struct amdgpu_ip_block_version gfx_v12_1_ip_block = { 4419 .type = AMD_IP_BLOCK_TYPE_GFX, 4420 .major = 12, 4421 .minor = 1, 4422 .rev = 0, 4423 .funcs = &gfx_v12_1_ip_funcs, 4424 }; 4425 4426 static int gfx_v12_1_xcp_resume(void *handle, uint32_t inst_mask) 4427 { 4428 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4429 uint32_t tmp_mask; 4430 int i, r; 4431 4432 /* TODO : Initialize golden regs */ 4433 /* gfx_v12_1_init_golden_registers(adev); */ 4434 4435 tmp_mask = inst_mask; 4436 for_each_inst(i, tmp_mask) 4437 gfx_v12_1_xcc_constants_init(adev, i); 4438 4439 if (!amdgpu_sriov_vf(adev)) { 4440 tmp_mask = inst_mask; 4441 for_each_inst(i, tmp_mask) { 4442 r = gfx_v12_1_xcc_rlc_resume(adev, i); 4443 if (r) 4444 return r; 4445 } 4446 } 4447 4448 r = gfx_v12_1_xcc_cp_resume(adev, inst_mask); 4449 4450 return r; 4451 } 4452 4453 static int gfx_v12_1_xcp_suspend(void *handle, uint32_t inst_mask) 4454 { 4455 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 4456 int i; 4457 4458 for_each_inst(i, inst_mask) 4459 gfx_v12_1_xcc_fini(adev, i); 4460 4461 return 0; 4462 } 4463 4464 struct amdgpu_xcp_ip_funcs gfx_v12_1_xcp_funcs = { 4465 .suspend = &gfx_v12_1_xcp_suspend, 4466 .resume = &gfx_v12_1_xcp_resume 4467 }; 4468