1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_drv.h> 26 27 #include "amdgpu.h" 28 #include "amdgpu_vcn.h" 29 #include "amdgpu_pm.h" 30 #include "soc15.h" 31 #include "soc15d.h" 32 #include "soc15_hw_ip.h" 33 #include "vcn_v2_0.h" 34 #include "vcn_v4_0_3.h" 35 #include "mmsch_v4_0_3.h" 36 37 #include "vcn/vcn_4_0_3_offset.h" 38 #include "vcn/vcn_4_0_3_sh_mask.h" 39 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" 40 41 #define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL 42 #define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX 43 #define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA 44 #define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX 45 46 #define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 47 #define VCN1_VID_SOC_ADDRESS_3_0 0x48300 48 #define VCN1_AON_SOC_ADDRESS_3_0 0x48000 49 50 static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { 51 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), 52 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), 53 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), 54 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), 55 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), 56 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), 57 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), 58 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), 59 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), 60 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), 61 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), 62 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), 63 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), 64 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), 65 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), 66 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), 67 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), 68 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), 69 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), 70 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), 71 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), 72 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), 73 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), 74 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), 75 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), 76 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), 77 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), 78 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), 79 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), 80 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), 81 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), 82 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), 83 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) 84 }; 85 86 #define NORMALIZE_VCN_REG_OFFSET(offset) \ 87 (offset & 0x1FFFF) 88 89 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); 90 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); 91 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); 92 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 93 enum amd_powergating_state state); 94 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 95 struct dpg_pause_state *new_state); 96 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); 97 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); 98 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 99 int inst_idx, bool indirect); 100 101 static inline bool vcn_v4_0_3_normalizn_reqd(struct amdgpu_device *adev) 102 { 103 return (adev->vcn.caps & AMDGPU_VCN_CAPS(RRMT_ENABLED)) == 0; 104 } 105 106 /** 107 * vcn_v4_0_3_early_init - set function pointers 108 * 109 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 110 * 111 * Set ring and irq function pointers 112 */ 113 static int vcn_v4_0_3_early_init(struct amdgpu_ip_block *ip_block) 114 { 115 struct amdgpu_device *adev = ip_block->adev; 116 int i, r; 117 118 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) 119 /* re-use enc ring as unified ring */ 120 adev->vcn.inst[i].num_enc_rings = 1; 121 122 vcn_v4_0_3_set_unified_ring_funcs(adev); 123 vcn_v4_0_3_set_irq_funcs(adev); 124 vcn_v4_0_3_set_ras_funcs(adev); 125 126 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 127 adev->vcn.inst[i].set_pg_state = vcn_v4_0_3_set_pg_state; 128 129 r = amdgpu_vcn_early_init(adev, i); 130 if (r) 131 return r; 132 } 133 134 return 0; 135 } 136 137 static int vcn_v4_0_3_fw_shared_init(struct amdgpu_device *adev, int inst_idx) 138 { 139 struct amdgpu_vcn4_fw_shared *fw_shared; 140 141 fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 142 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); 143 fw_shared->sq.is_enabled = 1; 144 145 if (amdgpu_vcnfw_log) 146 amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]); 147 148 return 0; 149 } 150 151 /** 152 * vcn_v4_0_3_sw_init - sw init for VCN block 153 * 154 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 155 * 156 * Load firmware and sw initialization 157 */ 158 static int vcn_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block) 159 { 160 struct amdgpu_device *adev = ip_block->adev; 161 struct amdgpu_ring *ring; 162 int i, r, vcn_inst; 163 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); 164 uint32_t *ptr; 165 166 /* VCN DEC TRAP */ 167 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 168 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); 169 if (r) 170 return r; 171 172 /* VCN POISON TRAP */ 173 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 174 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst->ras_poison_irq); 175 176 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 177 178 r = amdgpu_vcn_sw_init(adev, i); 179 if (r) 180 return r; 181 182 amdgpu_vcn_setup_ucode(adev, i); 183 184 r = amdgpu_vcn_resume(adev, i); 185 if (r) 186 return r; 187 188 vcn_inst = GET_INST(VCN, i); 189 190 ring = &adev->vcn.inst[i].ring_enc[0]; 191 ring->use_doorbell = true; 192 193 if (!amdgpu_sriov_vf(adev)) 194 ring->doorbell_index = 195 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 196 9 * vcn_inst; 197 else 198 ring->doorbell_index = 199 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 200 32 * vcn_inst; 201 202 ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); 203 sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); 204 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, 205 AMDGPU_RING_PRIO_DEFAULT, 206 &adev->vcn.inst[i].sched_score); 207 if (r) 208 return r; 209 210 vcn_v4_0_3_fw_shared_init(adev, i); 211 212 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 213 adev->vcn.inst[i].pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; 214 } 215 216 /* TODO: Add queue reset mask when FW fully supports it */ 217 adev->vcn.supported_reset = 218 amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]); 219 220 if (amdgpu_sriov_vf(adev)) { 221 r = amdgpu_virt_alloc_mm_table(adev); 222 if (r) 223 return r; 224 } 225 226 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 227 r = amdgpu_vcn_ras_sw_init(adev); 228 if (r) { 229 dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); 230 return r; 231 } 232 } 233 234 /* Allocate memory for VCN IP Dump buffer */ 235 ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL); 236 if (!ptr) { 237 DRM_ERROR("Failed to allocate memory for VCN IP Dump\n"); 238 adev->vcn.ip_dump = NULL; 239 } else { 240 adev->vcn.ip_dump = ptr; 241 } 242 243 r = amdgpu_vcn_sysfs_reset_mask_init(adev); 244 if (r) 245 return r; 246 247 return 0; 248 } 249 250 /** 251 * vcn_v4_0_3_sw_fini - sw fini for VCN block 252 * 253 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 254 * 255 * VCN suspend and free up sw allocation 256 */ 257 static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block) 258 { 259 struct amdgpu_device *adev = ip_block->adev; 260 int i, r, idx; 261 262 if (drm_dev_enter(&adev->ddev, &idx)) { 263 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 264 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 265 266 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 267 fw_shared->present_flag_0 = 0; 268 fw_shared->sq.is_enabled = cpu_to_le32(false); 269 } 270 drm_dev_exit(idx); 271 } 272 273 if (amdgpu_sriov_vf(adev)) 274 amdgpu_virt_free_mm_table(adev); 275 276 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 277 r = amdgpu_vcn_suspend(adev, i); 278 if (r) 279 return r; 280 } 281 282 amdgpu_vcn_sysfs_reset_mask_fini(adev); 283 284 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 285 r = amdgpu_vcn_sw_fini(adev, i); 286 if (r) 287 return r; 288 } 289 290 kfree(adev->vcn.ip_dump); 291 292 return 0; 293 } 294 295 static int vcn_v4_0_3_hw_init_inst(struct amdgpu_vcn_inst *vinst) 296 { 297 int vcn_inst; 298 struct amdgpu_device *adev = vinst->adev; 299 struct amdgpu_ring *ring; 300 int inst_idx = vinst->inst; 301 302 vcn_inst = GET_INST(VCN, inst_idx); 303 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 304 if (ring->use_doorbell) { 305 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, 306 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 9 * vcn_inst, 307 adev->vcn.inst[inst_idx].aid_id); 308 309 WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL, 310 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT | 311 VCN_RB1_DB_CTRL__EN_MASK); 312 313 /* Read DB_CTRL to flush the write DB_CTRL command. */ 314 RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL); 315 } 316 317 return 0; 318 } 319 320 /** 321 * vcn_v4_0_3_hw_init - start and test VCN block 322 * 323 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 324 * 325 * Initialize the hardware, boot up the VCPU and do some testing 326 */ 327 static int vcn_v4_0_3_hw_init(struct amdgpu_ip_block *ip_block) 328 { 329 struct amdgpu_device *adev = ip_block->adev; 330 struct amdgpu_ring *ring; 331 struct amdgpu_vcn_inst *vinst; 332 int i, r; 333 334 if (amdgpu_sriov_vf(adev)) { 335 r = vcn_v4_0_3_start_sriov(adev); 336 if (r) 337 return r; 338 339 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 340 ring = &adev->vcn.inst[i].ring_enc[0]; 341 ring->wptr = 0; 342 ring->wptr_old = 0; 343 vcn_v4_0_3_unified_ring_set_wptr(ring); 344 ring->sched.ready = true; 345 } 346 } else { 347 /* This flag is not set for VF, assumed to be disabled always */ 348 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 349 0x100) 350 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 351 352 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 353 struct amdgpu_vcn4_fw_shared *fw_shared; 354 355 ring = &adev->vcn.inst[i].ring_enc[0]; 356 vinst = &adev->vcn.inst[i]; 357 vcn_v4_0_3_hw_init_inst(vinst); 358 359 /* Re-init fw_shared when RAS fatal error occurred */ 360 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 361 if (!fw_shared->sq.is_enabled) 362 vcn_v4_0_3_fw_shared_init(adev, i); 363 364 r = amdgpu_ring_test_helper(ring); 365 if (r) 366 return r; 367 } 368 } 369 370 return r; 371 } 372 373 /** 374 * vcn_v4_0_3_hw_fini - stop the hardware block 375 * 376 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 377 * 378 * Stop the VCN block, mark ring as not ready any more 379 */ 380 static int vcn_v4_0_3_hw_fini(struct amdgpu_ip_block *ip_block) 381 { 382 struct amdgpu_device *adev = ip_block->adev; 383 int i; 384 385 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 386 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 387 388 cancel_delayed_work_sync(&vinst->idle_work); 389 390 if (vinst->cur_state != AMD_PG_STATE_GATE) 391 vinst->set_pg_state(vinst, AMD_PG_STATE_GATE); 392 } 393 394 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) 395 amdgpu_irq_put(adev, &adev->vcn.inst->ras_poison_irq, 0); 396 397 return 0; 398 } 399 400 /** 401 * vcn_v4_0_3_suspend - suspend VCN block 402 * 403 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 404 * 405 * HW fini and suspend VCN block 406 */ 407 static int vcn_v4_0_3_suspend(struct amdgpu_ip_block *ip_block) 408 { 409 struct amdgpu_device *adev = ip_block->adev; 410 int r, i; 411 412 r = vcn_v4_0_3_hw_fini(ip_block); 413 if (r) 414 return r; 415 416 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 417 r = amdgpu_vcn_suspend(adev, i); 418 if (r) 419 return r; 420 } 421 422 return 0; 423 } 424 425 /** 426 * vcn_v4_0_3_resume - resume VCN block 427 * 428 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 429 * 430 * Resume firmware and hw init VCN block 431 */ 432 static int vcn_v4_0_3_resume(struct amdgpu_ip_block *ip_block) 433 { 434 struct amdgpu_device *adev = ip_block->adev; 435 int r, i; 436 437 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 438 r = amdgpu_vcn_resume(ip_block->adev, i); 439 if (r) 440 return r; 441 } 442 443 r = vcn_v4_0_3_hw_init(ip_block); 444 445 return r; 446 } 447 448 /** 449 * vcn_v4_0_3_mc_resume - memory controller programming 450 * 451 * @vinst: VCN instance 452 * 453 * Let the VCN memory controller know it's offsets 454 */ 455 static void vcn_v4_0_3_mc_resume(struct amdgpu_vcn_inst *vinst) 456 { 457 struct amdgpu_device *adev = vinst->adev; 458 int inst_idx = vinst->inst; 459 uint32_t offset, size, vcn_inst; 460 const struct common_firmware_header *hdr; 461 462 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 463 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 464 465 vcn_inst = GET_INST(VCN, inst_idx); 466 /* cache window 0: fw */ 467 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 468 WREG32_SOC15( 469 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 470 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 471 .tmr_mc_addr_lo)); 472 WREG32_SOC15( 473 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 474 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 475 .tmr_mc_addr_hi)); 476 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); 477 offset = 0; 478 } else { 479 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 480 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 481 WREG32_SOC15(VCN, vcn_inst, 482 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 483 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 484 offset = size; 485 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 486 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 487 } 488 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); 489 490 /* cache window 1: stack */ 491 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, 492 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 493 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, 494 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 495 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); 496 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, 497 AMDGPU_VCN_STACK_SIZE); 498 499 /* cache window 2: context */ 500 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, 501 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 502 AMDGPU_VCN_STACK_SIZE)); 503 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, 504 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 505 AMDGPU_VCN_STACK_SIZE)); 506 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); 507 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, 508 AMDGPU_VCN_CONTEXT_SIZE); 509 510 /* non-cache window */ 511 WREG32_SOC15( 512 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, 513 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 514 WREG32_SOC15( 515 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, 516 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 517 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); 518 WREG32_SOC15( 519 VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, 520 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 521 } 522 523 /** 524 * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode 525 * 526 * @vinst: VCN instance 527 * @indirect: indirectly write sram 528 * 529 * Let the VCN memory controller know it's offsets with dpg mode 530 */ 531 static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst, 532 bool indirect) 533 { 534 struct amdgpu_device *adev = vinst->adev; 535 int inst_idx = vinst->inst; 536 uint32_t offset, size; 537 const struct common_firmware_header *hdr; 538 539 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 540 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 541 542 /* cache window 0: fw */ 543 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 544 if (!indirect) { 545 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 546 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 547 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 548 inst_idx].tmr_mc_addr_lo), 0, indirect); 549 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 550 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 551 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 552 inst_idx].tmr_mc_addr_hi), 0, indirect); 553 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 554 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 555 } else { 556 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 557 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); 558 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 559 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); 560 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 561 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 562 } 563 offset = 0; 564 } else { 565 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 566 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 567 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 568 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 569 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 570 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 571 offset = size; 572 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 573 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 574 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); 575 } 576 577 if (!indirect) 578 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 579 VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); 580 else 581 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 582 VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); 583 584 /* cache window 1: stack */ 585 if (!indirect) { 586 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 587 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 588 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 589 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 590 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 591 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 592 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 593 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 594 } else { 595 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 596 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); 597 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 598 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); 599 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 600 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 601 } 602 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 603 VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); 604 605 /* cache window 2: context */ 606 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 607 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 608 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 609 AMDGPU_VCN_STACK_SIZE), 0, indirect); 610 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 611 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 612 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 613 AMDGPU_VCN_STACK_SIZE), 0, indirect); 614 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 615 VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); 616 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 617 VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); 618 619 /* non-cache window */ 620 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 621 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 622 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 623 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 624 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 625 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 626 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 627 VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); 628 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 629 VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), 630 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); 631 632 /* VCN global tiling registers */ 633 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 634 VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 635 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 636 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 637 } 638 639 /** 640 * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating 641 * 642 * @vinst: VCN instance 643 * 644 * Disable clock gating for VCN block 645 */ 646 static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_vcn_inst *vinst) 647 { 648 struct amdgpu_device *adev = vinst->adev; 649 int inst_idx = vinst->inst; 650 uint32_t data; 651 int vcn_inst; 652 653 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 654 return; 655 656 vcn_inst = GET_INST(VCN, inst_idx); 657 658 /* VCN disable CGC */ 659 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 660 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; 661 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 662 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 663 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 664 665 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); 666 data &= ~(UVD_CGC_GATE__SYS_MASK 667 | UVD_CGC_GATE__MPEG2_MASK 668 | UVD_CGC_GATE__REGS_MASK 669 | UVD_CGC_GATE__RBC_MASK 670 | UVD_CGC_GATE__LMI_MC_MASK 671 | UVD_CGC_GATE__LMI_UMC_MASK 672 | UVD_CGC_GATE__MPC_MASK 673 | UVD_CGC_GATE__LBSI_MASK 674 | UVD_CGC_GATE__LRBBM_MASK 675 | UVD_CGC_GATE__WCB_MASK 676 | UVD_CGC_GATE__VCPU_MASK 677 | UVD_CGC_GATE__MMSCH_MASK); 678 679 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); 680 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); 681 682 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 683 data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK 684 | UVD_CGC_CTRL__MPEG2_MODE_MASK 685 | UVD_CGC_CTRL__REGS_MODE_MASK 686 | UVD_CGC_CTRL__RBC_MODE_MASK 687 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 688 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 689 | UVD_CGC_CTRL__MPC_MODE_MASK 690 | UVD_CGC_CTRL__LBSI_MODE_MASK 691 | UVD_CGC_CTRL__LRBBM_MODE_MASK 692 | UVD_CGC_CTRL__WCB_MODE_MASK 693 | UVD_CGC_CTRL__VCPU_MODE_MASK 694 | UVD_CGC_CTRL__MMSCH_MODE_MASK); 695 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 696 697 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); 698 data |= (UVD_SUVD_CGC_GATE__SRE_MASK 699 | UVD_SUVD_CGC_GATE__SIT_MASK 700 | UVD_SUVD_CGC_GATE__SMP_MASK 701 | UVD_SUVD_CGC_GATE__SCM_MASK 702 | UVD_SUVD_CGC_GATE__SDB_MASK 703 | UVD_SUVD_CGC_GATE__SRE_H264_MASK 704 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK 705 | UVD_SUVD_CGC_GATE__SIT_H264_MASK 706 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK 707 | UVD_SUVD_CGC_GATE__SCM_H264_MASK 708 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK 709 | UVD_SUVD_CGC_GATE__SDB_H264_MASK 710 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK 711 | UVD_SUVD_CGC_GATE__ENT_MASK 712 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK 713 | UVD_SUVD_CGC_GATE__SITE_MASK 714 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK 715 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK 716 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK 717 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK 718 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); 719 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); 720 721 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 722 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 723 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 724 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 725 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 726 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 727 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 728 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 729 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 730 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 731 } 732 733 /** 734 * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode 735 * 736 * @vinst: VCN instance 737 * @sram_sel: sram select 738 * @indirect: indirectly write sram 739 * 740 * Disable clock gating for VCN block with dpg mode 741 */ 742 static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst, 743 uint8_t sram_sel, 744 uint8_t indirect) 745 { 746 struct amdgpu_device *adev = vinst->adev; 747 int inst_idx = vinst->inst; 748 uint32_t reg_data = 0; 749 750 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 751 return; 752 753 /* enable sw clock gating control */ 754 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 755 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 756 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 757 reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | 758 UVD_CGC_CTRL__MPEG2_MODE_MASK | 759 UVD_CGC_CTRL__REGS_MODE_MASK | 760 UVD_CGC_CTRL__RBC_MODE_MASK | 761 UVD_CGC_CTRL__LMI_MC_MODE_MASK | 762 UVD_CGC_CTRL__LMI_UMC_MODE_MASK | 763 UVD_CGC_CTRL__IDCT_MODE_MASK | 764 UVD_CGC_CTRL__MPRD_MODE_MASK | 765 UVD_CGC_CTRL__MPC_MODE_MASK | 766 UVD_CGC_CTRL__LBSI_MODE_MASK | 767 UVD_CGC_CTRL__LRBBM_MODE_MASK | 768 UVD_CGC_CTRL__WCB_MODE_MASK | 769 UVD_CGC_CTRL__VCPU_MODE_MASK); 770 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 771 VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); 772 773 /* turn off clock gating */ 774 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 775 VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); 776 777 /* turn on SUVD clock gating */ 778 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 779 VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); 780 781 /* turn on sw mode in UVD_SUVD_CGC_CTRL */ 782 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 783 VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); 784 } 785 786 /** 787 * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating 788 * 789 * @vinst: VCN instance 790 * 791 * Enable clock gating for VCN block 792 */ 793 static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_vcn_inst *vinst) 794 { 795 struct amdgpu_device *adev = vinst->adev; 796 int inst_idx = vinst->inst; 797 uint32_t data; 798 int vcn_inst; 799 800 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 801 return; 802 803 vcn_inst = GET_INST(VCN, inst_idx); 804 805 /* enable VCN CGC */ 806 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 807 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 808 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 809 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 810 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 811 812 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 813 data |= (UVD_CGC_CTRL__SYS_MODE_MASK 814 | UVD_CGC_CTRL__MPEG2_MODE_MASK 815 | UVD_CGC_CTRL__REGS_MODE_MASK 816 | UVD_CGC_CTRL__RBC_MODE_MASK 817 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 818 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 819 | UVD_CGC_CTRL__MPC_MODE_MASK 820 | UVD_CGC_CTRL__LBSI_MODE_MASK 821 | UVD_CGC_CTRL__LRBBM_MODE_MASK 822 | UVD_CGC_CTRL__WCB_MODE_MASK 823 | UVD_CGC_CTRL__VCPU_MODE_MASK); 824 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 825 826 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 827 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 828 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 829 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 830 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 831 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 832 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 833 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 834 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 835 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 836 } 837 838 /** 839 * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode 840 * 841 * @vinst: VCN instance 842 * @indirect: indirectly write sram 843 * 844 * Start VCN block with dpg mode 845 */ 846 static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_vcn_inst *vinst, 847 bool indirect) 848 { 849 struct amdgpu_device *adev = vinst->adev; 850 int inst_idx = vinst->inst; 851 volatile struct amdgpu_vcn4_fw_shared *fw_shared = 852 adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 853 struct amdgpu_ring *ring; 854 int vcn_inst; 855 uint32_t tmp; 856 857 vcn_inst = GET_INST(VCN, inst_idx); 858 /* disable register anti-hang mechanism */ 859 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, 860 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 861 /* enable dynamic power gating mode */ 862 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); 863 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; 864 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; 865 WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); 866 867 if (indirect) { 868 DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", 869 inst_idx, adev->vcn.inst[inst_idx].aid_id); 870 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = 871 (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; 872 /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ 873 WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, 874 adev->vcn.inst[inst_idx].aid_id, 0, true); 875 } 876 877 /* enable clock gating */ 878 vcn_v4_0_3_disable_clock_gating_dpg_mode(vinst, 0, indirect); 879 880 /* enable VCPU clock */ 881 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 882 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 883 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; 884 885 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 886 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 887 888 /* disable master interrupt */ 889 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 890 VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); 891 892 /* setup regUVD_LMI_CTRL */ 893 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 894 UVD_LMI_CTRL__REQ_MODE_MASK | 895 UVD_LMI_CTRL__CRC_RESET_MASK | 896 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 897 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 898 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | 899 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | 900 0x00100000L); 901 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 902 VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); 903 904 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 905 VCN, 0, regUVD_MPC_CNTL), 906 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); 907 908 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 909 VCN, 0, regUVD_MPC_SET_MUXA0), 910 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 911 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 912 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 913 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); 914 915 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 916 VCN, 0, regUVD_MPC_SET_MUXB0), 917 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 918 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 919 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 920 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); 921 922 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 923 VCN, 0, regUVD_MPC_SET_MUX), 924 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 925 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 926 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); 927 928 vcn_v4_0_3_mc_resume_dpg_mode(vinst, indirect); 929 930 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 931 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 932 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 933 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 934 935 /* enable LMI MC and UMC channels */ 936 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; 937 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 938 VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); 939 940 vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); 941 942 /* enable master interrupt */ 943 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 944 VCN, 0, regUVD_MASTINT_EN), 945 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); 946 947 if (indirect) 948 amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); 949 950 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 951 952 /* program the RB_BASE for ring buffer */ 953 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 954 lower_32_bits(ring->gpu_addr)); 955 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 956 upper_32_bits(ring->gpu_addr)); 957 958 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 959 ring->ring_size / sizeof(uint32_t)); 960 961 /* resetting ring, fw should not check RB ring */ 962 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 963 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 964 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 965 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; 966 967 /* Initialize the ring buffer's read and write pointers */ 968 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 969 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 970 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 971 972 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 973 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 974 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 975 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); 976 977 /*resetting done, fw can check RB ring */ 978 fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); 979 980 /* Keeping one read-back to ensure all register writes are done, 981 * otherwise it may introduce race conditions. 982 */ 983 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 984 985 return 0; 986 } 987 988 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) 989 { 990 int i, vcn_inst; 991 struct amdgpu_ring *ring_enc; 992 uint64_t cache_addr; 993 uint64_t rb_enc_addr; 994 uint64_t ctx_addr; 995 uint32_t param, resp, expected; 996 uint32_t offset, cache_size; 997 uint32_t tmp, timeout; 998 999 struct amdgpu_mm_table *table = &adev->virt.mm_table; 1000 uint32_t *table_loc; 1001 uint32_t table_size; 1002 uint32_t size, size_dw; 1003 uint32_t init_status; 1004 uint32_t enabled_vcn; 1005 1006 struct mmsch_v4_0_cmd_direct_write 1007 direct_wt = { {0} }; 1008 struct mmsch_v4_0_cmd_direct_read_modify_write 1009 direct_rd_mod_wt = { {0} }; 1010 struct mmsch_v4_0_cmd_end end = { {0} }; 1011 struct mmsch_v4_0_3_init_header header; 1012 1013 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 1014 volatile struct amdgpu_fw_shared_rb_setup *rb_setup; 1015 1016 direct_wt.cmd_header.command_type = 1017 MMSCH_COMMAND__DIRECT_REG_WRITE; 1018 direct_rd_mod_wt.cmd_header.command_type = 1019 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; 1020 end.cmd_header.command_type = MMSCH_COMMAND__END; 1021 1022 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1023 vcn_inst = GET_INST(VCN, i); 1024 1025 vcn_v4_0_3_fw_shared_init(adev, vcn_inst); 1026 1027 memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); 1028 header.version = MMSCH_VERSION; 1029 header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; 1030 1031 table_loc = (uint32_t *)table->cpu_addr; 1032 table_loc += header.total_size; 1033 1034 table_size = 0; 1035 1036 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), 1037 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); 1038 1039 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4); 1040 1041 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1042 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1043 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1044 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); 1045 1046 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1047 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1048 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); 1049 1050 offset = 0; 1051 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1052 regUVD_VCPU_CACHE_OFFSET0), 0); 1053 } else { 1054 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1055 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1056 lower_32_bits(adev->vcn.inst[i].gpu_addr)); 1057 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1058 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1059 upper_32_bits(adev->vcn.inst[i].gpu_addr)); 1060 offset = cache_size; 1061 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1062 regUVD_VCPU_CACHE_OFFSET0), 1063 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 1064 } 1065 1066 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1067 regUVD_VCPU_CACHE_SIZE0), 1068 cache_size); 1069 1070 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; 1071 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1072 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1073 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1074 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1075 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1076 regUVD_VCPU_CACHE_OFFSET1), 0); 1077 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1078 regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); 1079 1080 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + 1081 AMDGPU_VCN_STACK_SIZE; 1082 1083 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1084 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1085 1086 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1087 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1088 1089 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1090 regUVD_VCPU_CACHE_OFFSET2), 0); 1091 1092 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1093 regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); 1094 1095 fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; 1096 rb_setup = &fw_shared->rb_setup; 1097 1098 ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; 1099 ring_enc->wptr = 0; 1100 rb_enc_addr = ring_enc->gpu_addr; 1101 1102 rb_setup->is_rb_enabled_flags |= RB_ENABLED; 1103 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); 1104 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); 1105 rb_setup->rb_size = ring_enc->ring_size / 4; 1106 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); 1107 1108 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1109 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 1110 lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1111 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1112 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 1113 upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1114 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1115 regUVD_VCPU_NONCACHE_SIZE0), 1116 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 1117 MMSCH_V4_0_INSERT_END(); 1118 1119 header.vcn0.init_status = 0; 1120 header.vcn0.table_offset = header.total_size; 1121 header.vcn0.table_size = table_size; 1122 header.total_size += table_size; 1123 1124 /* Send init table to mmsch */ 1125 size = sizeof(struct mmsch_v4_0_3_init_header); 1126 table_loc = (uint32_t *)table->cpu_addr; 1127 memcpy((void *)table_loc, &header, size); 1128 1129 ctx_addr = table->gpu_addr; 1130 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); 1131 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); 1132 1133 tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); 1134 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; 1135 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); 1136 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); 1137 1138 size = header.total_size; 1139 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); 1140 1141 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); 1142 1143 param = 0x00000001; 1144 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); 1145 tmp = 0; 1146 timeout = 1000; 1147 resp = 0; 1148 expected = MMSCH_VF_MAILBOX_RESP__OK; 1149 while (resp != expected) { 1150 resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); 1151 if (resp != 0) 1152 break; 1153 1154 udelay(10); 1155 tmp = tmp + 10; 1156 if (tmp >= timeout) { 1157 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ 1158 " waiting for regMMSCH_VF_MAILBOX_RESP "\ 1159 "(expected=0x%08x, readback=0x%08x)\n", 1160 tmp, expected, resp); 1161 return -EBUSY; 1162 } 1163 } 1164 1165 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; 1166 init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; 1167 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE 1168 && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { 1169 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ 1170 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); 1171 } 1172 } 1173 1174 return 0; 1175 } 1176 1177 /** 1178 * vcn_v4_0_3_start - VCN start 1179 * 1180 * @vinst: VCN instance 1181 * 1182 * Start VCN block 1183 */ 1184 static int vcn_v4_0_3_start(struct amdgpu_vcn_inst *vinst) 1185 { 1186 struct amdgpu_device *adev = vinst->adev; 1187 int i = vinst->inst; 1188 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 1189 struct amdgpu_ring *ring; 1190 int j, k, r, vcn_inst; 1191 uint32_t tmp; 1192 1193 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 1194 return vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram); 1195 1196 vcn_inst = GET_INST(VCN, i); 1197 /* set VCN status busy */ 1198 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | 1199 UVD_STATUS__UVD_BUSY; 1200 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); 1201 1202 /* SW clock gating */ 1203 vcn_v4_0_3_disable_clock_gating(vinst); 1204 1205 /* enable VCPU clock */ 1206 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1207 UVD_VCPU_CNTL__CLK_EN_MASK, 1208 ~UVD_VCPU_CNTL__CLK_EN_MASK); 1209 1210 /* disable master interrupt */ 1211 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, 1212 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1213 1214 /* enable LMI MC and UMC channels */ 1215 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, 1216 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); 1217 1218 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1219 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1220 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1221 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1222 1223 /* setup regUVD_LMI_CTRL */ 1224 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); 1225 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, 1226 tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 1227 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 1228 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 1229 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); 1230 1231 /* setup regUVD_MPC_CNTL */ 1232 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); 1233 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; 1234 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; 1235 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); 1236 1237 /* setup UVD_MPC_SET_MUXA0 */ 1238 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, 1239 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 1240 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 1241 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 1242 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); 1243 1244 /* setup UVD_MPC_SET_MUXB0 */ 1245 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, 1246 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 1247 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 1248 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 1249 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); 1250 1251 /* setup UVD_MPC_SET_MUX */ 1252 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, 1253 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 1254 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 1255 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); 1256 1257 vcn_v4_0_3_mc_resume(vinst); 1258 1259 /* VCN global tiling registers */ 1260 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, 1261 adev->gfx.config.gb_addr_config); 1262 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, 1263 adev->gfx.config.gb_addr_config); 1264 1265 /* unblock VCPU register access */ 1266 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, 1267 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1268 1269 /* release VCPU reset to boot */ 1270 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1271 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1272 1273 for (j = 0; j < 10; ++j) { 1274 uint32_t status; 1275 1276 for (k = 0; k < 100; ++k) { 1277 status = RREG32_SOC15(VCN, vcn_inst, 1278 regUVD_STATUS); 1279 if (status & 2) 1280 break; 1281 mdelay(10); 1282 } 1283 r = 0; 1284 if (status & 2) 1285 break; 1286 1287 DRM_DEV_ERROR(adev->dev, 1288 "VCN decode not responding, trying to reset the VCPU!!!\n"); 1289 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1290 regUVD_VCPU_CNTL), 1291 UVD_VCPU_CNTL__BLK_RST_MASK, 1292 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1293 mdelay(10); 1294 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1295 regUVD_VCPU_CNTL), 1296 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); 1297 1298 mdelay(10); 1299 r = -1; 1300 } 1301 1302 if (r) { 1303 DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); 1304 return r; 1305 } 1306 1307 /* enable master interrupt */ 1308 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 1309 UVD_MASTINT_EN__VCPU_EN_MASK, 1310 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1311 1312 /* clear the busy bit of VCN_STATUS */ 1313 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, 1314 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); 1315 1316 ring = &adev->vcn.inst[i].ring_enc[0]; 1317 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1318 1319 /* program the RB_BASE for ring buffer */ 1320 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 1321 lower_32_bits(ring->gpu_addr)); 1322 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 1323 upper_32_bits(ring->gpu_addr)); 1324 1325 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 1326 ring->ring_size / sizeof(uint32_t)); 1327 1328 /* resetting ring, fw should not check RB ring */ 1329 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1330 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 1331 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1332 1333 /* Initialize the ring buffer's read and write pointers */ 1334 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 1335 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 1336 1337 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1338 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 1339 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1340 1341 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1342 fw_shared->sq.queue_mode &= 1343 cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); 1344 1345 return 0; 1346 } 1347 1348 /** 1349 * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode 1350 * 1351 * @vinst: VCN instance 1352 * 1353 * Stop VCN block with dpg mode 1354 */ 1355 static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_vcn_inst *vinst) 1356 { 1357 struct amdgpu_device *adev = vinst->adev; 1358 int inst_idx = vinst->inst; 1359 uint32_t tmp; 1360 int vcn_inst; 1361 1362 vcn_inst = GET_INST(VCN, inst_idx); 1363 1364 /* Wait for power status to be 1 */ 1365 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1366 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1367 1368 /* wait for read ptr to be equal to write ptr */ 1369 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1370 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); 1371 1372 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1373 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1374 1375 /* disable dynamic power gating mode */ 1376 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, 1377 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); 1378 1379 /* Keeping one read-back to ensure all register writes are done, 1380 * otherwise it may introduce race conditions. 1381 */ 1382 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1383 1384 return 0; 1385 } 1386 1387 /** 1388 * vcn_v4_0_3_stop - VCN stop 1389 * 1390 * @vinst: VCN instance 1391 * 1392 * Stop VCN block 1393 */ 1394 static int vcn_v4_0_3_stop(struct amdgpu_vcn_inst *vinst) 1395 { 1396 struct amdgpu_device *adev = vinst->adev; 1397 int i = vinst->inst; 1398 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 1399 int r = 0, vcn_inst; 1400 uint32_t tmp; 1401 1402 vcn_inst = GET_INST(VCN, i); 1403 1404 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1405 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; 1406 1407 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { 1408 vcn_v4_0_3_stop_dpg_mode(vinst); 1409 goto Done; 1410 } 1411 1412 /* wait for vcn idle */ 1413 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, 1414 UVD_STATUS__IDLE, 0x7); 1415 if (r) 1416 goto Done; 1417 1418 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | 1419 UVD_LMI_STATUS__READ_CLEAN_MASK | 1420 UVD_LMI_STATUS__WRITE_CLEAN_MASK | 1421 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; 1422 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1423 tmp); 1424 if (r) 1425 goto Done; 1426 1427 /* stall UMC channel */ 1428 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); 1429 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; 1430 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); 1431 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | 1432 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; 1433 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1434 tmp); 1435 if (r) 1436 goto Done; 1437 1438 /* Unblock VCPU Register access */ 1439 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 1440 UVD_RB_ARB_CTRL__VCPU_DIS_MASK, 1441 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1442 1443 /* release VCPU reset to boot */ 1444 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1445 UVD_VCPU_CNTL__BLK_RST_MASK, 1446 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1447 1448 /* disable VCPU clock */ 1449 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1450 ~(UVD_VCPU_CNTL__CLK_EN_MASK)); 1451 1452 /* reset LMI UMC/LMI/VCPU */ 1453 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1454 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1455 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1456 1457 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1458 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1459 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1460 1461 /* clear VCN status */ 1462 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); 1463 1464 /* apply HW clock gating */ 1465 vcn_v4_0_3_enable_clock_gating(vinst); 1466 1467 /* Keeping one read-back to ensure all register writes are done, 1468 * otherwise it may introduce race conditions. 1469 */ 1470 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1471 1472 Done: 1473 return 0; 1474 } 1475 1476 /** 1477 * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode 1478 * 1479 * @vinst: VCN instance 1480 * @new_state: pause state 1481 * 1482 * Pause dpg mode for VCN block 1483 */ 1484 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 1485 struct dpg_pause_state *new_state) 1486 { 1487 1488 return 0; 1489 } 1490 1491 /** 1492 * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer 1493 * 1494 * @ring: amdgpu_ring pointer 1495 * 1496 * Returns the current hardware unified read pointer 1497 */ 1498 static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) 1499 { 1500 struct amdgpu_device *adev = ring->adev; 1501 1502 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1503 DRM_ERROR("wrong ring id is identified in %s", __func__); 1504 1505 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); 1506 } 1507 1508 /** 1509 * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer 1510 * 1511 * @ring: amdgpu_ring pointer 1512 * 1513 * Returns the current hardware unified write pointer 1514 */ 1515 static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) 1516 { 1517 struct amdgpu_device *adev = ring->adev; 1518 1519 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1520 DRM_ERROR("wrong ring id is identified in %s", __func__); 1521 1522 if (ring->use_doorbell) 1523 return *ring->wptr_cpu_addr; 1524 else 1525 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), 1526 regUVD_RB_WPTR); 1527 } 1528 1529 void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 1530 uint32_t val, uint32_t mask) 1531 { 1532 /* Use normalized offsets when required */ 1533 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1534 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1535 1536 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); 1537 amdgpu_ring_write(ring, reg << 2); 1538 amdgpu_ring_write(ring, mask); 1539 amdgpu_ring_write(ring, val); 1540 } 1541 1542 void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 1543 uint32_t val) 1544 { 1545 /* Use normalized offsets when required */ 1546 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1547 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1548 1549 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); 1550 amdgpu_ring_write(ring, reg << 2); 1551 amdgpu_ring_write(ring, val); 1552 } 1553 1554 void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, 1555 unsigned int vmid, uint64_t pd_addr) 1556 { 1557 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; 1558 1559 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 1560 1561 /* wait for reg writes */ 1562 vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + 1563 vmid * hub->ctx_addr_distance, 1564 lower_32_bits(pd_addr), 0xffffffff); 1565 } 1566 1567 void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) 1568 { 1569 /* VCN engine access for HDP flush doesn't work when RRMT is enabled. 1570 * This is a workaround to avoid any HDP flush through VCN ring. 1571 */ 1572 } 1573 1574 /** 1575 * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer 1576 * 1577 * @ring: amdgpu_ring pointer 1578 * 1579 * Commits the enc write pointer to the hardware 1580 */ 1581 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) 1582 { 1583 struct amdgpu_device *adev = ring->adev; 1584 1585 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1586 DRM_ERROR("wrong ring id is identified in %s", __func__); 1587 1588 if (ring->use_doorbell) { 1589 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 1590 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1591 } else { 1592 WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, 1593 lower_32_bits(ring->wptr)); 1594 } 1595 } 1596 1597 static int vcn_v4_0_3_ring_reset(struct amdgpu_ring *ring, unsigned int vmid) 1598 { 1599 int r = 0; 1600 int vcn_inst; 1601 struct amdgpu_device *adev = ring->adev; 1602 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me]; 1603 1604 if (amdgpu_sriov_vf(ring->adev)) 1605 return -EOPNOTSUPP; 1606 1607 if (!(adev->vcn.supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE)) 1608 return -EOPNOTSUPP; 1609 1610 vcn_inst = GET_INST(VCN, ring->me); 1611 r = amdgpu_dpm_reset_vcn(adev, 1 << vcn_inst); 1612 1613 if (r) { 1614 DRM_DEV_ERROR(adev->dev, "VCN reset fail : %d\n", r); 1615 return r; 1616 } 1617 1618 /* This flag is not set for VF, assumed to be disabled always */ 1619 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x100) 1620 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 1621 vcn_v4_0_3_hw_init_inst(vinst); 1622 vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[ring->me].indirect_sram); 1623 r = amdgpu_ring_test_helper(ring); 1624 1625 return r; 1626 } 1627 1628 static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { 1629 .type = AMDGPU_RING_TYPE_VCN_ENC, 1630 .align_mask = 0x3f, 1631 .nop = VCN_ENC_CMD_NO_OP, 1632 .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, 1633 .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, 1634 .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, 1635 .emit_frame_size = 1636 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 1637 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + 1638 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ 1639 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ 1640 1, /* vcn_v2_0_enc_ring_insert_end */ 1641 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ 1642 .emit_ib = vcn_v2_0_enc_ring_emit_ib, 1643 .emit_fence = vcn_v2_0_enc_ring_emit_fence, 1644 .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, 1645 .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, 1646 .test_ring = amdgpu_vcn_enc_ring_test_ring, 1647 .test_ib = amdgpu_vcn_unified_ring_test_ib, 1648 .insert_nop = amdgpu_ring_insert_nop, 1649 .insert_end = vcn_v2_0_enc_ring_insert_end, 1650 .pad_ib = amdgpu_ring_generic_pad_ib, 1651 .begin_use = amdgpu_vcn_ring_begin_use, 1652 .end_use = amdgpu_vcn_ring_end_use, 1653 .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, 1654 .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, 1655 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1656 .reset = vcn_v4_0_3_ring_reset, 1657 }; 1658 1659 /** 1660 * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions 1661 * 1662 * @adev: amdgpu_device pointer 1663 * 1664 * Set unified ring functions 1665 */ 1666 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) 1667 { 1668 int i, vcn_inst; 1669 1670 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1671 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; 1672 adev->vcn.inst[i].ring_enc[0].me = i; 1673 vcn_inst = GET_INST(VCN, i); 1674 adev->vcn.inst[i].aid_id = 1675 vcn_inst / adev->vcn.num_inst_per_aid; 1676 } 1677 } 1678 1679 /** 1680 * vcn_v4_0_3_is_idle - check VCN block is idle 1681 * 1682 * @ip_block: Pointer to the amdgpu_ip_block structure 1683 * 1684 * Check whether VCN block is idle 1685 */ 1686 static bool vcn_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block) 1687 { 1688 struct amdgpu_device *adev = ip_block->adev; 1689 int i, ret = 1; 1690 1691 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1692 ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == 1693 UVD_STATUS__IDLE); 1694 } 1695 1696 return ret; 1697 } 1698 1699 /** 1700 * vcn_v4_0_3_wait_for_idle - wait for VCN block idle 1701 * 1702 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 1703 * 1704 * Wait for VCN block idle 1705 */ 1706 static int vcn_v4_0_3_wait_for_idle(struct amdgpu_ip_block *ip_block) 1707 { 1708 struct amdgpu_device *adev = ip_block->adev; 1709 int i, ret = 0; 1710 1711 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1712 ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, 1713 UVD_STATUS__IDLE, UVD_STATUS__IDLE); 1714 if (ret) 1715 return ret; 1716 } 1717 1718 return ret; 1719 } 1720 1721 /* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state 1722 * 1723 * @ip_block: amdgpu_ip_block pointer 1724 * @state: clock gating state 1725 * 1726 * Set VCN block clockgating state 1727 */ 1728 static int vcn_v4_0_3_set_clockgating_state(struct amdgpu_ip_block *ip_block, 1729 enum amd_clockgating_state state) 1730 { 1731 struct amdgpu_device *adev = ip_block->adev; 1732 bool enable = state == AMD_CG_STATE_GATE; 1733 int i; 1734 1735 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1736 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 1737 1738 if (enable) { 1739 if (RREG32_SOC15(VCN, GET_INST(VCN, i), 1740 regUVD_STATUS) != UVD_STATUS__IDLE) 1741 return -EBUSY; 1742 vcn_v4_0_3_enable_clock_gating(vinst); 1743 } else { 1744 vcn_v4_0_3_disable_clock_gating(vinst); 1745 } 1746 } 1747 return 0; 1748 } 1749 1750 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 1751 enum amd_powergating_state state) 1752 { 1753 struct amdgpu_device *adev = vinst->adev; 1754 int ret = 0; 1755 1756 /* for SRIOV, guest should not control VCN Power-gating 1757 * MMSCH FW should control Power-gating and clock-gating 1758 * guest should avoid touching CGC and PG 1759 */ 1760 if (amdgpu_sriov_vf(adev)) { 1761 vinst->cur_state = AMD_PG_STATE_UNGATE; 1762 return 0; 1763 } 1764 1765 if (state == vinst->cur_state) 1766 return 0; 1767 1768 if (state == AMD_PG_STATE_GATE) 1769 ret = vcn_v4_0_3_stop(vinst); 1770 else 1771 ret = vcn_v4_0_3_start(vinst); 1772 1773 if (!ret) 1774 vinst->cur_state = state; 1775 1776 return ret; 1777 } 1778 1779 /** 1780 * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state 1781 * 1782 * @adev: amdgpu_device pointer 1783 * @source: interrupt sources 1784 * @type: interrupt types 1785 * @state: interrupt states 1786 * 1787 * Set VCN block interrupt state 1788 */ 1789 static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, 1790 struct amdgpu_irq_src *source, 1791 unsigned int type, 1792 enum amdgpu_interrupt_state state) 1793 { 1794 return 0; 1795 } 1796 1797 /** 1798 * vcn_v4_0_3_process_interrupt - process VCN block interrupt 1799 * 1800 * @adev: amdgpu_device pointer 1801 * @source: interrupt sources 1802 * @entry: interrupt entry from clients and sources 1803 * 1804 * Process VCN block interrupt 1805 */ 1806 static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, 1807 struct amdgpu_irq_src *source, 1808 struct amdgpu_iv_entry *entry) 1809 { 1810 uint32_t i, inst; 1811 1812 i = node_id_to_phys_map[entry->node_id]; 1813 1814 DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); 1815 1816 for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) 1817 if (adev->vcn.inst[inst].aid_id == i) 1818 break; 1819 1820 if (inst >= adev->vcn.num_vcn_inst) { 1821 dev_WARN_ONCE(adev->dev, 1, 1822 "Interrupt received for unknown VCN instance %d", 1823 entry->node_id); 1824 return 0; 1825 } 1826 1827 switch (entry->src_id) { 1828 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: 1829 amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); 1830 break; 1831 default: 1832 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", 1833 entry->src_id, entry->src_data[0]); 1834 break; 1835 } 1836 1837 return 0; 1838 } 1839 1840 static int vcn_v4_0_3_set_ras_interrupt_state(struct amdgpu_device *adev, 1841 struct amdgpu_irq_src *source, 1842 unsigned int type, 1843 enum amdgpu_interrupt_state state) 1844 { 1845 return 0; 1846 } 1847 1848 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { 1849 .set = vcn_v4_0_3_set_interrupt_state, 1850 .process = vcn_v4_0_3_process_interrupt, 1851 }; 1852 1853 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_ras_irq_funcs = { 1854 .set = vcn_v4_0_3_set_ras_interrupt_state, 1855 .process = amdgpu_vcn_process_poison_irq, 1856 }; 1857 1858 /** 1859 * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions 1860 * 1861 * @adev: amdgpu_device pointer 1862 * 1863 * Set VCN block interrupt irq functions 1864 */ 1865 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) 1866 { 1867 int i; 1868 1869 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1870 adev->vcn.inst->irq.num_types++; 1871 } 1872 adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; 1873 1874 adev->vcn.inst->ras_poison_irq.num_types = 1; 1875 adev->vcn.inst->ras_poison_irq.funcs = &vcn_v4_0_3_ras_irq_funcs; 1876 } 1877 1878 static void vcn_v4_0_3_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 1879 { 1880 struct amdgpu_device *adev = ip_block->adev; 1881 int i, j; 1882 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); 1883 uint32_t inst_off, is_powered; 1884 1885 if (!adev->vcn.ip_dump) 1886 return; 1887 1888 drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst); 1889 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1890 if (adev->vcn.harvest_config & (1 << i)) { 1891 drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i); 1892 continue; 1893 } 1894 1895 inst_off = i * reg_count; 1896 is_powered = (adev->vcn.ip_dump[inst_off] & 1897 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; 1898 1899 if (is_powered) { 1900 drm_printf(p, "\nActive Instance:VCN%d\n", i); 1901 for (j = 0; j < reg_count; j++) 1902 drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0_3[j].reg_name, 1903 adev->vcn.ip_dump[inst_off + j]); 1904 } else { 1905 drm_printf(p, "\nInactive Instance:VCN%d\n", i); 1906 } 1907 } 1908 } 1909 1910 static void vcn_v4_0_3_dump_ip_state(struct amdgpu_ip_block *ip_block) 1911 { 1912 struct amdgpu_device *adev = ip_block->adev; 1913 int i, j; 1914 bool is_powered; 1915 uint32_t inst_off, inst_id; 1916 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); 1917 1918 if (!adev->vcn.ip_dump) 1919 return; 1920 1921 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1922 if (adev->vcn.harvest_config & (1 << i)) 1923 continue; 1924 1925 inst_id = GET_INST(VCN, i); 1926 inst_off = i * reg_count; 1927 /* mmUVD_POWER_STATUS is always readable and is first element of the array */ 1928 adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, inst_id, regUVD_POWER_STATUS); 1929 is_powered = (adev->vcn.ip_dump[inst_off] & 1930 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; 1931 1932 if (is_powered) 1933 for (j = 1; j < reg_count; j++) 1934 adev->vcn.ip_dump[inst_off + j] = 1935 RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0_3[j], 1936 inst_id)); 1937 } 1938 } 1939 1940 static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { 1941 .name = "vcn_v4_0_3", 1942 .early_init = vcn_v4_0_3_early_init, 1943 .sw_init = vcn_v4_0_3_sw_init, 1944 .sw_fini = vcn_v4_0_3_sw_fini, 1945 .hw_init = vcn_v4_0_3_hw_init, 1946 .hw_fini = vcn_v4_0_3_hw_fini, 1947 .suspend = vcn_v4_0_3_suspend, 1948 .resume = vcn_v4_0_3_resume, 1949 .is_idle = vcn_v4_0_3_is_idle, 1950 .wait_for_idle = vcn_v4_0_3_wait_for_idle, 1951 .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, 1952 .set_powergating_state = vcn_set_powergating_state, 1953 .dump_ip_state = vcn_v4_0_3_dump_ip_state, 1954 .print_ip_state = vcn_v4_0_3_print_ip_state, 1955 }; 1956 1957 const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { 1958 .type = AMD_IP_BLOCK_TYPE_VCN, 1959 .major = 4, 1960 .minor = 0, 1961 .rev = 3, 1962 .funcs = &vcn_v4_0_3_ip_funcs, 1963 }; 1964 1965 static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { 1966 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), 1967 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, 1968 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), 1969 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, 1970 }; 1971 1972 static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, 1973 uint32_t vcn_inst, 1974 void *ras_err_status) 1975 { 1976 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; 1977 1978 /* vcn v4_0_3 only support query uncorrectable errors */ 1979 amdgpu_ras_inst_query_ras_error_count(adev, 1980 vcn_v4_0_3_ue_reg_list, 1981 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 1982 NULL, 0, GET_INST(VCN, vcn_inst), 1983 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 1984 &err_data->ue_count); 1985 } 1986 1987 static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, 1988 void *ras_err_status) 1989 { 1990 uint32_t i; 1991 1992 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 1993 dev_warn(adev->dev, "VCN RAS is not supported\n"); 1994 return; 1995 } 1996 1997 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 1998 vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); 1999 } 2000 2001 static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, 2002 uint32_t vcn_inst) 2003 { 2004 amdgpu_ras_inst_reset_ras_error_count(adev, 2005 vcn_v4_0_3_ue_reg_list, 2006 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 2007 GET_INST(VCN, vcn_inst)); 2008 } 2009 2010 static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) 2011 { 2012 uint32_t i; 2013 2014 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 2015 dev_warn(adev->dev, "VCN RAS is not supported\n"); 2016 return; 2017 } 2018 2019 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 2020 vcn_v4_0_3_inst_reset_ras_error_count(adev, i); 2021 } 2022 2023 static uint32_t vcn_v4_0_3_query_poison_by_instance(struct amdgpu_device *adev, 2024 uint32_t instance, uint32_t sub_block) 2025 { 2026 uint32_t poison_stat = 0, reg_value = 0; 2027 2028 switch (sub_block) { 2029 case AMDGPU_VCN_V4_0_3_VCPU_VCODEC: 2030 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS); 2031 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF); 2032 break; 2033 default: 2034 break; 2035 } 2036 2037 if (poison_stat) 2038 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n", 2039 instance, sub_block); 2040 2041 return poison_stat; 2042 } 2043 2044 static bool vcn_v4_0_3_query_poison_status(struct amdgpu_device *adev) 2045 { 2046 uint32_t inst, sub; 2047 uint32_t poison_stat = 0; 2048 2049 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++) 2050 for (sub = 0; sub < AMDGPU_VCN_V4_0_3_MAX_SUB_BLOCK; sub++) 2051 poison_stat += 2052 vcn_v4_0_3_query_poison_by_instance(adev, inst, sub); 2053 2054 return !!poison_stat; 2055 } 2056 2057 static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { 2058 .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, 2059 .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, 2060 .query_poison_status = vcn_v4_0_3_query_poison_status, 2061 }; 2062 2063 static int vcn_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, 2064 enum aca_smu_type type, void *data) 2065 { 2066 struct aca_bank_info info; 2067 u64 misc0; 2068 int ret; 2069 2070 ret = aca_bank_info_decode(bank, &info); 2071 if (ret) 2072 return ret; 2073 2074 misc0 = bank->regs[ACA_REG_IDX_MISC0]; 2075 switch (type) { 2076 case ACA_SMU_TYPE_UE: 2077 bank->aca_err_type = ACA_ERROR_TYPE_UE; 2078 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, 2079 1ULL); 2080 break; 2081 case ACA_SMU_TYPE_CE: 2082 bank->aca_err_type = ACA_ERROR_TYPE_CE; 2083 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 2084 ACA_REG__MISC0__ERRCNT(misc0)); 2085 break; 2086 default: 2087 return -EINVAL; 2088 } 2089 2090 return ret; 2091 } 2092 2093 /* reference to smu driver if header file */ 2094 static int vcn_v4_0_3_err_codes[] = { 2095 14, 15, /* VCN */ 2096 }; 2097 2098 static bool vcn_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, 2099 enum aca_smu_type type, void *data) 2100 { 2101 u32 instlo; 2102 2103 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); 2104 instlo &= GENMASK(31, 1); 2105 2106 if (instlo != mmSMNAID_AID0_MCA_SMU) 2107 return false; 2108 2109 if (aca_bank_check_error_codes(handle->adev, bank, 2110 vcn_v4_0_3_err_codes, 2111 ARRAY_SIZE(vcn_v4_0_3_err_codes))) 2112 return false; 2113 2114 return true; 2115 } 2116 2117 static const struct aca_bank_ops vcn_v4_0_3_aca_bank_ops = { 2118 .aca_bank_parser = vcn_v4_0_3_aca_bank_parser, 2119 .aca_bank_is_valid = vcn_v4_0_3_aca_bank_is_valid, 2120 }; 2121 2122 static const struct aca_info vcn_v4_0_3_aca_info = { 2123 .hwip = ACA_HWIP_TYPE_SMU, 2124 .mask = ACA_ERROR_UE_MASK, 2125 .bank_ops = &vcn_v4_0_3_aca_bank_ops, 2126 }; 2127 2128 static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 2129 { 2130 int r; 2131 2132 r = amdgpu_ras_block_late_init(adev, ras_block); 2133 if (r) 2134 return r; 2135 2136 if (amdgpu_ras_is_supported(adev, ras_block->block) && 2137 adev->vcn.inst->ras_poison_irq.funcs) { 2138 r = amdgpu_irq_get(adev, &adev->vcn.inst->ras_poison_irq, 0); 2139 if (r) 2140 goto late_fini; 2141 } 2142 2143 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN, 2144 &vcn_v4_0_3_aca_info, NULL); 2145 if (r) 2146 goto late_fini; 2147 2148 return 0; 2149 2150 late_fini: 2151 amdgpu_ras_block_late_fini(adev, ras_block); 2152 2153 return r; 2154 } 2155 2156 static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { 2157 .ras_block = { 2158 .hw_ops = &vcn_v4_0_3_ras_hw_ops, 2159 .ras_late_init = vcn_v4_0_3_ras_late_init, 2160 }, 2161 }; 2162 2163 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) 2164 { 2165 adev->vcn.ras = &vcn_v4_0_3_ras; 2166 } 2167 2168 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 2169 int inst_idx, bool indirect) 2170 { 2171 uint32_t tmp; 2172 2173 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) 2174 return; 2175 2176 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | 2177 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | 2178 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | 2179 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; 2180 WREG32_SOC15_DPG_MODE(inst_idx, 2181 SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), 2182 tmp, 0, indirect); 2183 2184 tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; 2185 WREG32_SOC15_DPG_MODE(inst_idx, 2186 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), 2187 tmp, 0, indirect); 2188 2189 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; 2190 WREG32_SOC15_DPG_MODE(inst_idx, 2191 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), 2192 tmp, 0, indirect); 2193 } 2194