1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_drv.h> 26 27 #include "amdgpu.h" 28 #include "amdgpu_vcn.h" 29 #include "amdgpu_pm.h" 30 #include "soc15.h" 31 #include "soc15d.h" 32 #include "soc15_hw_ip.h" 33 #include "vcn_v2_0.h" 34 #include "vcn_v4_0_3.h" 35 #include "mmsch_v4_0_3.h" 36 37 #include "vcn/vcn_4_0_3_offset.h" 38 #include "vcn/vcn_4_0_3_sh_mask.h" 39 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" 40 41 #define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL 42 #define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX 43 #define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA 44 #define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX 45 46 #define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 47 #define VCN1_VID_SOC_ADDRESS_3_0 0x48300 48 #define VCN1_AON_SOC_ADDRESS_3_0 0x48000 49 50 static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { 51 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), 52 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), 53 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), 54 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), 55 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), 56 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), 57 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), 58 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), 59 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), 60 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), 61 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), 62 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), 63 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), 64 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), 65 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), 66 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), 67 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), 68 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), 69 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), 70 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), 71 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), 72 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), 73 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), 74 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), 75 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), 76 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), 77 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), 78 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), 79 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), 80 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), 81 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), 82 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), 83 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) 84 }; 85 86 #define NORMALIZE_VCN_REG_OFFSET(offset) \ 87 (offset & 0x1FFFF) 88 89 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); 90 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); 91 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); 92 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 93 enum amd_powergating_state state); 94 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 95 struct dpg_pause_state *new_state); 96 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); 97 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); 98 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 99 int inst_idx, bool indirect); 100 101 static inline bool vcn_v4_0_3_normalizn_reqd(struct amdgpu_device *adev) 102 { 103 return (adev->vcn.caps & AMDGPU_VCN_CAPS(RRMT_ENABLED)) == 0; 104 } 105 106 /** 107 * vcn_v4_0_3_early_init - set function pointers 108 * 109 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 110 * 111 * Set ring and irq function pointers 112 */ 113 static int vcn_v4_0_3_early_init(struct amdgpu_ip_block *ip_block) 114 { 115 struct amdgpu_device *adev = ip_block->adev; 116 int i, r; 117 118 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) 119 /* re-use enc ring as unified ring */ 120 adev->vcn.inst[i].num_enc_rings = 1; 121 122 vcn_v4_0_3_set_unified_ring_funcs(adev); 123 vcn_v4_0_3_set_irq_funcs(adev); 124 vcn_v4_0_3_set_ras_funcs(adev); 125 126 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 127 adev->vcn.inst[i].set_pg_state = vcn_v4_0_3_set_pg_state; 128 129 r = amdgpu_vcn_early_init(adev, i); 130 if (r) 131 return r; 132 } 133 134 return 0; 135 } 136 137 static int vcn_v4_0_3_fw_shared_init(struct amdgpu_device *adev, int inst_idx) 138 { 139 struct amdgpu_vcn4_fw_shared *fw_shared; 140 141 fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 142 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); 143 fw_shared->sq.is_enabled = 1; 144 145 if (amdgpu_vcnfw_log) 146 amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]); 147 148 return 0; 149 } 150 151 /** 152 * vcn_v4_0_3_sw_init - sw init for VCN block 153 * 154 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 155 * 156 * Load firmware and sw initialization 157 */ 158 static int vcn_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block) 159 { 160 struct amdgpu_device *adev = ip_block->adev; 161 struct amdgpu_ring *ring; 162 int i, r, vcn_inst; 163 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); 164 uint32_t *ptr; 165 166 /* VCN DEC TRAP */ 167 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 168 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); 169 if (r) 170 return r; 171 172 /* VCN POISON TRAP */ 173 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 174 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst->ras_poison_irq); 175 176 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 177 178 r = amdgpu_vcn_sw_init(adev, i); 179 if (r) 180 return r; 181 182 amdgpu_vcn_setup_ucode(adev, i); 183 184 r = amdgpu_vcn_resume(adev, i); 185 if (r) 186 return r; 187 188 vcn_inst = GET_INST(VCN, i); 189 190 ring = &adev->vcn.inst[i].ring_enc[0]; 191 ring->use_doorbell = true; 192 193 if (!amdgpu_sriov_vf(adev)) 194 ring->doorbell_index = 195 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 196 9 * vcn_inst; 197 else 198 ring->doorbell_index = 199 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 200 32 * vcn_inst; 201 202 ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); 203 sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); 204 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst->irq, 0, 205 AMDGPU_RING_PRIO_DEFAULT, 206 &adev->vcn.inst[i].sched_score); 207 if (r) 208 return r; 209 210 vcn_v4_0_3_fw_shared_init(adev, i); 211 212 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 213 adev->vcn.inst[i].pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; 214 } 215 216 /* TODO: Add queue reset mask when FW fully supports it */ 217 adev->vcn.supported_reset = 218 amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]); 219 220 if (amdgpu_sriov_vf(adev)) { 221 r = amdgpu_virt_alloc_mm_table(adev); 222 if (r) 223 return r; 224 } 225 226 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 227 r = amdgpu_vcn_ras_sw_init(adev); 228 if (r) { 229 dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); 230 return r; 231 } 232 } 233 234 /* Allocate memory for VCN IP Dump buffer */ 235 ptr = kcalloc(adev->vcn.num_vcn_inst * reg_count, sizeof(uint32_t), GFP_KERNEL); 236 if (!ptr) { 237 DRM_ERROR("Failed to allocate memory for VCN IP Dump\n"); 238 adev->vcn.ip_dump = NULL; 239 } else { 240 adev->vcn.ip_dump = ptr; 241 } 242 243 r = amdgpu_vcn_sysfs_reset_mask_init(adev); 244 if (r) 245 return r; 246 247 return 0; 248 } 249 250 /** 251 * vcn_v4_0_3_sw_fini - sw fini for VCN block 252 * 253 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 254 * 255 * VCN suspend and free up sw allocation 256 */ 257 static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block) 258 { 259 struct amdgpu_device *adev = ip_block->adev; 260 int i, r, idx; 261 262 if (drm_dev_enter(&adev->ddev, &idx)) { 263 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 264 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 265 266 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 267 fw_shared->present_flag_0 = 0; 268 fw_shared->sq.is_enabled = cpu_to_le32(false); 269 } 270 drm_dev_exit(idx); 271 } 272 273 if (amdgpu_sriov_vf(adev)) 274 amdgpu_virt_free_mm_table(adev); 275 276 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 277 r = amdgpu_vcn_suspend(adev, i); 278 if (r) 279 return r; 280 } 281 282 amdgpu_vcn_sysfs_reset_mask_fini(adev); 283 284 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 285 r = amdgpu_vcn_sw_fini(adev, i); 286 if (r) 287 return r; 288 } 289 290 kfree(adev->vcn.ip_dump); 291 292 return 0; 293 } 294 295 static int vcn_v4_0_3_hw_init_inst(struct amdgpu_vcn_inst *vinst) 296 { 297 int vcn_inst; 298 struct amdgpu_device *adev = vinst->adev; 299 struct amdgpu_ring *ring; 300 int inst_idx = vinst->inst; 301 302 vcn_inst = GET_INST(VCN, inst_idx); 303 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 304 if (ring->use_doorbell) { 305 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, 306 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 9 * vcn_inst, 307 adev->vcn.inst[inst_idx].aid_id); 308 309 WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL, 310 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT | 311 VCN_RB1_DB_CTRL__EN_MASK); 312 313 /* Read DB_CTRL to flush the write DB_CTRL command. */ 314 RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL); 315 } 316 317 return 0; 318 } 319 320 /** 321 * vcn_v4_0_3_hw_init - start and test VCN block 322 * 323 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 324 * 325 * Initialize the hardware, boot up the VCPU and do some testing 326 */ 327 static int vcn_v4_0_3_hw_init(struct amdgpu_ip_block *ip_block) 328 { 329 struct amdgpu_device *adev = ip_block->adev; 330 struct amdgpu_ring *ring; 331 struct amdgpu_vcn_inst *vinst; 332 int i, r; 333 334 if (amdgpu_sriov_vf(adev)) { 335 r = vcn_v4_0_3_start_sriov(adev); 336 if (r) 337 return r; 338 339 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 340 ring = &adev->vcn.inst[i].ring_enc[0]; 341 ring->wptr = 0; 342 ring->wptr_old = 0; 343 vcn_v4_0_3_unified_ring_set_wptr(ring); 344 ring->sched.ready = true; 345 } 346 } else { 347 /* This flag is not set for VF, assumed to be disabled always */ 348 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 349 0x100) 350 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 351 352 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 353 struct amdgpu_vcn4_fw_shared *fw_shared; 354 355 ring = &adev->vcn.inst[i].ring_enc[0]; 356 vinst = &adev->vcn.inst[i]; 357 vcn_v4_0_3_hw_init_inst(vinst); 358 359 /* Re-init fw_shared when RAS fatal error occurred */ 360 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 361 if (!fw_shared->sq.is_enabled) 362 vcn_v4_0_3_fw_shared_init(adev, i); 363 364 r = amdgpu_ring_test_helper(ring); 365 if (r) 366 return r; 367 } 368 } 369 370 return r; 371 } 372 373 /** 374 * vcn_v4_0_3_hw_fini - stop the hardware block 375 * 376 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 377 * 378 * Stop the VCN block, mark ring as not ready any more 379 */ 380 static int vcn_v4_0_3_hw_fini(struct amdgpu_ip_block *ip_block) 381 { 382 struct amdgpu_device *adev = ip_block->adev; 383 int i; 384 385 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 386 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 387 388 cancel_delayed_work_sync(&vinst->idle_work); 389 390 if (vinst->cur_state != AMD_PG_STATE_GATE) 391 vinst->set_pg_state(vinst, AMD_PG_STATE_GATE); 392 } 393 394 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) 395 amdgpu_irq_put(adev, &adev->vcn.inst->ras_poison_irq, 0); 396 397 return 0; 398 } 399 400 /** 401 * vcn_v4_0_3_suspend - suspend VCN block 402 * 403 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 404 * 405 * HW fini and suspend VCN block 406 */ 407 static int vcn_v4_0_3_suspend(struct amdgpu_ip_block *ip_block) 408 { 409 struct amdgpu_device *adev = ip_block->adev; 410 int r, i; 411 412 r = vcn_v4_0_3_hw_fini(ip_block); 413 if (r) 414 return r; 415 416 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 417 r = amdgpu_vcn_suspend(adev, i); 418 if (r) 419 return r; 420 } 421 422 return 0; 423 } 424 425 /** 426 * vcn_v4_0_3_resume - resume VCN block 427 * 428 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 429 * 430 * Resume firmware and hw init VCN block 431 */ 432 static int vcn_v4_0_3_resume(struct amdgpu_ip_block *ip_block) 433 { 434 struct amdgpu_device *adev = ip_block->adev; 435 int r, i; 436 437 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 438 r = amdgpu_vcn_resume(ip_block->adev, i); 439 if (r) 440 return r; 441 } 442 443 r = vcn_v4_0_3_hw_init(ip_block); 444 445 return r; 446 } 447 448 /** 449 * vcn_v4_0_3_mc_resume - memory controller programming 450 * 451 * @vinst: VCN instance 452 * 453 * Let the VCN memory controller know it's offsets 454 */ 455 static void vcn_v4_0_3_mc_resume(struct amdgpu_vcn_inst *vinst) 456 { 457 struct amdgpu_device *adev = vinst->adev; 458 int inst_idx = vinst->inst; 459 uint32_t offset, size, vcn_inst; 460 const struct common_firmware_header *hdr; 461 462 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 463 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 464 465 vcn_inst = GET_INST(VCN, inst_idx); 466 /* cache window 0: fw */ 467 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 468 WREG32_SOC15( 469 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 470 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 471 .tmr_mc_addr_lo)); 472 WREG32_SOC15( 473 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 474 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 475 .tmr_mc_addr_hi)); 476 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); 477 offset = 0; 478 } else { 479 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 480 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 481 WREG32_SOC15(VCN, vcn_inst, 482 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 483 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 484 offset = size; 485 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 486 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 487 } 488 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); 489 490 /* cache window 1: stack */ 491 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, 492 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 493 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, 494 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 495 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); 496 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, 497 AMDGPU_VCN_STACK_SIZE); 498 499 /* cache window 2: context */ 500 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, 501 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 502 AMDGPU_VCN_STACK_SIZE)); 503 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, 504 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 505 AMDGPU_VCN_STACK_SIZE)); 506 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); 507 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, 508 AMDGPU_VCN_CONTEXT_SIZE); 509 510 /* non-cache window */ 511 WREG32_SOC15( 512 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, 513 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 514 WREG32_SOC15( 515 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, 516 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 517 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); 518 WREG32_SOC15( 519 VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, 520 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 521 } 522 523 /** 524 * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode 525 * 526 * @vinst: VCN instance 527 * @indirect: indirectly write sram 528 * 529 * Let the VCN memory controller know it's offsets with dpg mode 530 */ 531 static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst, 532 bool indirect) 533 { 534 struct amdgpu_device *adev = vinst->adev; 535 int inst_idx = vinst->inst; 536 uint32_t offset, size; 537 const struct common_firmware_header *hdr; 538 539 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 540 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 541 542 /* cache window 0: fw */ 543 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 544 if (!indirect) { 545 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 546 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 547 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 548 inst_idx].tmr_mc_addr_lo), 0, indirect); 549 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 550 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 551 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 552 inst_idx].tmr_mc_addr_hi), 0, indirect); 553 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 554 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 555 } else { 556 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 557 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); 558 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 559 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); 560 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 561 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 562 } 563 offset = 0; 564 } else { 565 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 566 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 567 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 568 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 569 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 570 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 571 offset = size; 572 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 573 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 574 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); 575 } 576 577 if (!indirect) 578 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 579 VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); 580 else 581 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 582 VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); 583 584 /* cache window 1: stack */ 585 if (!indirect) { 586 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 587 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 588 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 589 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 590 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 591 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 592 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 593 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 594 } else { 595 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 596 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); 597 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 598 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); 599 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 600 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 601 } 602 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 603 VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); 604 605 /* cache window 2: context */ 606 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 607 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 608 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 609 AMDGPU_VCN_STACK_SIZE), 0, indirect); 610 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 611 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 612 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 613 AMDGPU_VCN_STACK_SIZE), 0, indirect); 614 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 615 VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); 616 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 617 VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); 618 619 /* non-cache window */ 620 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 621 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 622 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 623 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 624 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 625 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 626 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 627 VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); 628 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 629 VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), 630 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); 631 632 /* VCN global tiling registers */ 633 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 634 VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 635 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 636 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 637 } 638 639 /** 640 * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating 641 * 642 * @vinst: VCN instance 643 * 644 * Disable clock gating for VCN block 645 */ 646 static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_vcn_inst *vinst) 647 { 648 struct amdgpu_device *adev = vinst->adev; 649 int inst_idx = vinst->inst; 650 uint32_t data; 651 int vcn_inst; 652 653 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 654 return; 655 656 vcn_inst = GET_INST(VCN, inst_idx); 657 658 /* VCN disable CGC */ 659 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 660 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; 661 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 662 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 663 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 664 665 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); 666 data &= ~(UVD_CGC_GATE__SYS_MASK 667 | UVD_CGC_GATE__MPEG2_MASK 668 | UVD_CGC_GATE__REGS_MASK 669 | UVD_CGC_GATE__RBC_MASK 670 | UVD_CGC_GATE__LMI_MC_MASK 671 | UVD_CGC_GATE__LMI_UMC_MASK 672 | UVD_CGC_GATE__MPC_MASK 673 | UVD_CGC_GATE__LBSI_MASK 674 | UVD_CGC_GATE__LRBBM_MASK 675 | UVD_CGC_GATE__WCB_MASK 676 | UVD_CGC_GATE__VCPU_MASK 677 | UVD_CGC_GATE__MMSCH_MASK); 678 679 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); 680 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); 681 682 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 683 data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK 684 | UVD_CGC_CTRL__MPEG2_MODE_MASK 685 | UVD_CGC_CTRL__REGS_MODE_MASK 686 | UVD_CGC_CTRL__RBC_MODE_MASK 687 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 688 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 689 | UVD_CGC_CTRL__MPC_MODE_MASK 690 | UVD_CGC_CTRL__LBSI_MODE_MASK 691 | UVD_CGC_CTRL__LRBBM_MODE_MASK 692 | UVD_CGC_CTRL__WCB_MODE_MASK 693 | UVD_CGC_CTRL__VCPU_MODE_MASK 694 | UVD_CGC_CTRL__MMSCH_MODE_MASK); 695 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 696 697 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); 698 data |= (UVD_SUVD_CGC_GATE__SRE_MASK 699 | UVD_SUVD_CGC_GATE__SIT_MASK 700 | UVD_SUVD_CGC_GATE__SMP_MASK 701 | UVD_SUVD_CGC_GATE__SCM_MASK 702 | UVD_SUVD_CGC_GATE__SDB_MASK 703 | UVD_SUVD_CGC_GATE__SRE_H264_MASK 704 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK 705 | UVD_SUVD_CGC_GATE__SIT_H264_MASK 706 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK 707 | UVD_SUVD_CGC_GATE__SCM_H264_MASK 708 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK 709 | UVD_SUVD_CGC_GATE__SDB_H264_MASK 710 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK 711 | UVD_SUVD_CGC_GATE__ENT_MASK 712 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK 713 | UVD_SUVD_CGC_GATE__SITE_MASK 714 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK 715 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK 716 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK 717 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK 718 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); 719 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); 720 721 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 722 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 723 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 724 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 725 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 726 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 727 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 728 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 729 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 730 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 731 } 732 733 /** 734 * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode 735 * 736 * @vinst: VCN instance 737 * @sram_sel: sram select 738 * @indirect: indirectly write sram 739 * 740 * Disable clock gating for VCN block with dpg mode 741 */ 742 static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst, 743 uint8_t sram_sel, 744 uint8_t indirect) 745 { 746 struct amdgpu_device *adev = vinst->adev; 747 int inst_idx = vinst->inst; 748 uint32_t reg_data = 0; 749 750 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 751 return; 752 753 /* enable sw clock gating control */ 754 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 755 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 756 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 757 reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | 758 UVD_CGC_CTRL__MPEG2_MODE_MASK | 759 UVD_CGC_CTRL__REGS_MODE_MASK | 760 UVD_CGC_CTRL__RBC_MODE_MASK | 761 UVD_CGC_CTRL__LMI_MC_MODE_MASK | 762 UVD_CGC_CTRL__LMI_UMC_MODE_MASK | 763 UVD_CGC_CTRL__IDCT_MODE_MASK | 764 UVD_CGC_CTRL__MPRD_MODE_MASK | 765 UVD_CGC_CTRL__MPC_MODE_MASK | 766 UVD_CGC_CTRL__LBSI_MODE_MASK | 767 UVD_CGC_CTRL__LRBBM_MODE_MASK | 768 UVD_CGC_CTRL__WCB_MODE_MASK | 769 UVD_CGC_CTRL__VCPU_MODE_MASK); 770 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 771 VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); 772 773 /* turn off clock gating */ 774 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 775 VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); 776 777 /* turn on SUVD clock gating */ 778 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 779 VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); 780 781 /* turn on sw mode in UVD_SUVD_CGC_CTRL */ 782 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 783 VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); 784 } 785 786 /** 787 * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating 788 * 789 * @vinst: VCN instance 790 * 791 * Enable clock gating for VCN block 792 */ 793 static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_vcn_inst *vinst) 794 { 795 struct amdgpu_device *adev = vinst->adev; 796 int inst_idx = vinst->inst; 797 uint32_t data; 798 int vcn_inst; 799 800 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 801 return; 802 803 vcn_inst = GET_INST(VCN, inst_idx); 804 805 /* enable VCN CGC */ 806 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 807 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 808 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 809 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 810 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 811 812 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 813 data |= (UVD_CGC_CTRL__SYS_MODE_MASK 814 | UVD_CGC_CTRL__MPEG2_MODE_MASK 815 | UVD_CGC_CTRL__REGS_MODE_MASK 816 | UVD_CGC_CTRL__RBC_MODE_MASK 817 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 818 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 819 | UVD_CGC_CTRL__MPC_MODE_MASK 820 | UVD_CGC_CTRL__LBSI_MODE_MASK 821 | UVD_CGC_CTRL__LRBBM_MODE_MASK 822 | UVD_CGC_CTRL__WCB_MODE_MASK 823 | UVD_CGC_CTRL__VCPU_MODE_MASK); 824 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 825 826 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 827 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 828 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 829 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 830 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 831 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 832 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 833 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 834 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 835 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 836 } 837 838 /** 839 * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode 840 * 841 * @vinst: VCN instance 842 * @indirect: indirectly write sram 843 * 844 * Start VCN block with dpg mode 845 */ 846 static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_vcn_inst *vinst, 847 bool indirect) 848 { 849 struct amdgpu_device *adev = vinst->adev; 850 int inst_idx = vinst->inst; 851 volatile struct amdgpu_vcn4_fw_shared *fw_shared = 852 adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 853 struct amdgpu_ring *ring; 854 int vcn_inst, ret; 855 uint32_t tmp; 856 857 vcn_inst = GET_INST(VCN, inst_idx); 858 /* disable register anti-hang mechanism */ 859 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, 860 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 861 /* enable dynamic power gating mode */ 862 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); 863 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; 864 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; 865 WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); 866 867 if (indirect) { 868 DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", 869 inst_idx, adev->vcn.inst[inst_idx].aid_id); 870 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = 871 (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; 872 /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ 873 WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, 874 adev->vcn.inst[inst_idx].aid_id, 0, true); 875 } 876 877 /* enable clock gating */ 878 vcn_v4_0_3_disable_clock_gating_dpg_mode(vinst, 0, indirect); 879 880 /* enable VCPU clock */ 881 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 882 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 883 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; 884 885 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 886 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 887 888 /* disable master interrupt */ 889 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 890 VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); 891 892 /* setup regUVD_LMI_CTRL */ 893 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 894 UVD_LMI_CTRL__REQ_MODE_MASK | 895 UVD_LMI_CTRL__CRC_RESET_MASK | 896 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 897 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 898 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | 899 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | 900 0x00100000L); 901 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 902 VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); 903 904 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 905 VCN, 0, regUVD_MPC_CNTL), 906 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); 907 908 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 909 VCN, 0, regUVD_MPC_SET_MUXA0), 910 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 911 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 912 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 913 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); 914 915 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 916 VCN, 0, regUVD_MPC_SET_MUXB0), 917 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 918 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 919 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 920 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); 921 922 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 923 VCN, 0, regUVD_MPC_SET_MUX), 924 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 925 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 926 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); 927 928 vcn_v4_0_3_mc_resume_dpg_mode(vinst, indirect); 929 930 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 931 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 932 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 933 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 934 935 /* enable LMI MC and UMC channels */ 936 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; 937 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 938 VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); 939 940 vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); 941 942 /* enable master interrupt */ 943 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 944 VCN, 0, regUVD_MASTINT_EN), 945 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); 946 947 if (indirect) { 948 ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); 949 if (ret) { 950 dev_err(adev->dev, "vcn sram load failed %d\n", ret); 951 return ret; 952 } 953 } 954 955 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 956 957 /* program the RB_BASE for ring buffer */ 958 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 959 lower_32_bits(ring->gpu_addr)); 960 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 961 upper_32_bits(ring->gpu_addr)); 962 963 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 964 ring->ring_size / sizeof(uint32_t)); 965 966 /* resetting ring, fw should not check RB ring */ 967 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 968 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 969 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 970 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; 971 972 /* Initialize the ring buffer's read and write pointers */ 973 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 974 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 975 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 976 977 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 978 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 979 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 980 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); 981 982 /*resetting done, fw can check RB ring */ 983 fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); 984 985 /* Keeping one read-back to ensure all register writes are done, 986 * otherwise it may introduce race conditions. 987 */ 988 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 989 990 return 0; 991 } 992 993 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) 994 { 995 int i, vcn_inst; 996 struct amdgpu_ring *ring_enc; 997 uint64_t cache_addr; 998 uint64_t rb_enc_addr; 999 uint64_t ctx_addr; 1000 uint32_t param, resp, expected; 1001 uint32_t offset, cache_size; 1002 uint32_t tmp, timeout; 1003 1004 struct amdgpu_mm_table *table = &adev->virt.mm_table; 1005 uint32_t *table_loc; 1006 uint32_t table_size; 1007 uint32_t size, size_dw; 1008 uint32_t init_status; 1009 uint32_t enabled_vcn; 1010 1011 struct mmsch_v4_0_cmd_direct_write 1012 direct_wt = { {0} }; 1013 struct mmsch_v4_0_cmd_direct_read_modify_write 1014 direct_rd_mod_wt = { {0} }; 1015 struct mmsch_v4_0_cmd_end end = { {0} }; 1016 struct mmsch_v4_0_3_init_header header; 1017 1018 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 1019 volatile struct amdgpu_fw_shared_rb_setup *rb_setup; 1020 1021 direct_wt.cmd_header.command_type = 1022 MMSCH_COMMAND__DIRECT_REG_WRITE; 1023 direct_rd_mod_wt.cmd_header.command_type = 1024 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; 1025 end.cmd_header.command_type = MMSCH_COMMAND__END; 1026 1027 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1028 vcn_inst = GET_INST(VCN, i); 1029 1030 vcn_v4_0_3_fw_shared_init(adev, vcn_inst); 1031 1032 memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); 1033 header.version = MMSCH_VERSION; 1034 header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; 1035 1036 table_loc = (uint32_t *)table->cpu_addr; 1037 table_loc += header.total_size; 1038 1039 table_size = 0; 1040 1041 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), 1042 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); 1043 1044 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4); 1045 1046 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1047 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1048 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1049 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); 1050 1051 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1052 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1053 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); 1054 1055 offset = 0; 1056 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1057 regUVD_VCPU_CACHE_OFFSET0), 0); 1058 } else { 1059 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1060 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1061 lower_32_bits(adev->vcn.inst[i].gpu_addr)); 1062 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1063 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1064 upper_32_bits(adev->vcn.inst[i].gpu_addr)); 1065 offset = cache_size; 1066 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1067 regUVD_VCPU_CACHE_OFFSET0), 1068 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 1069 } 1070 1071 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1072 regUVD_VCPU_CACHE_SIZE0), 1073 cache_size); 1074 1075 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; 1076 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1077 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1078 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1079 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1080 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1081 regUVD_VCPU_CACHE_OFFSET1), 0); 1082 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1083 regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); 1084 1085 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + 1086 AMDGPU_VCN_STACK_SIZE; 1087 1088 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1089 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1090 1091 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1092 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1093 1094 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1095 regUVD_VCPU_CACHE_OFFSET2), 0); 1096 1097 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1098 regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); 1099 1100 fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; 1101 rb_setup = &fw_shared->rb_setup; 1102 1103 ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; 1104 ring_enc->wptr = 0; 1105 rb_enc_addr = ring_enc->gpu_addr; 1106 1107 rb_setup->is_rb_enabled_flags |= RB_ENABLED; 1108 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); 1109 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); 1110 rb_setup->rb_size = ring_enc->ring_size / 4; 1111 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); 1112 1113 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1114 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 1115 lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1116 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1117 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 1118 upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1119 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1120 regUVD_VCPU_NONCACHE_SIZE0), 1121 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 1122 MMSCH_V4_0_INSERT_END(); 1123 1124 header.vcn0.init_status = 0; 1125 header.vcn0.table_offset = header.total_size; 1126 header.vcn0.table_size = table_size; 1127 header.total_size += table_size; 1128 1129 /* Send init table to mmsch */ 1130 size = sizeof(struct mmsch_v4_0_3_init_header); 1131 table_loc = (uint32_t *)table->cpu_addr; 1132 memcpy((void *)table_loc, &header, size); 1133 1134 ctx_addr = table->gpu_addr; 1135 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); 1136 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); 1137 1138 tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); 1139 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; 1140 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); 1141 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); 1142 1143 size = header.total_size; 1144 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); 1145 1146 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); 1147 1148 param = 0x00000001; 1149 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); 1150 tmp = 0; 1151 timeout = 1000; 1152 resp = 0; 1153 expected = MMSCH_VF_MAILBOX_RESP__OK; 1154 while (resp != expected) { 1155 resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); 1156 if (resp != 0) 1157 break; 1158 1159 udelay(10); 1160 tmp = tmp + 10; 1161 if (tmp >= timeout) { 1162 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ 1163 " waiting for regMMSCH_VF_MAILBOX_RESP "\ 1164 "(expected=0x%08x, readback=0x%08x)\n", 1165 tmp, expected, resp); 1166 return -EBUSY; 1167 } 1168 } 1169 1170 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; 1171 init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; 1172 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE 1173 && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { 1174 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ 1175 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); 1176 } 1177 } 1178 1179 return 0; 1180 } 1181 1182 /** 1183 * vcn_v4_0_3_start - VCN start 1184 * 1185 * @vinst: VCN instance 1186 * 1187 * Start VCN block 1188 */ 1189 static int vcn_v4_0_3_start(struct amdgpu_vcn_inst *vinst) 1190 { 1191 struct amdgpu_device *adev = vinst->adev; 1192 int i = vinst->inst; 1193 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 1194 struct amdgpu_ring *ring; 1195 int j, k, r, vcn_inst; 1196 uint32_t tmp; 1197 1198 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 1199 return vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram); 1200 1201 vcn_inst = GET_INST(VCN, i); 1202 /* set VCN status busy */ 1203 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | 1204 UVD_STATUS__UVD_BUSY; 1205 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); 1206 1207 /* SW clock gating */ 1208 vcn_v4_0_3_disable_clock_gating(vinst); 1209 1210 /* enable VCPU clock */ 1211 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1212 UVD_VCPU_CNTL__CLK_EN_MASK, 1213 ~UVD_VCPU_CNTL__CLK_EN_MASK); 1214 1215 /* disable master interrupt */ 1216 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, 1217 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1218 1219 /* enable LMI MC and UMC channels */ 1220 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, 1221 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); 1222 1223 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1224 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1225 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1226 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1227 1228 /* setup regUVD_LMI_CTRL */ 1229 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); 1230 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, 1231 tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 1232 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 1233 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 1234 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); 1235 1236 /* setup regUVD_MPC_CNTL */ 1237 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); 1238 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; 1239 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; 1240 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); 1241 1242 /* setup UVD_MPC_SET_MUXA0 */ 1243 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, 1244 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 1245 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 1246 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 1247 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); 1248 1249 /* setup UVD_MPC_SET_MUXB0 */ 1250 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, 1251 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 1252 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 1253 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 1254 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); 1255 1256 /* setup UVD_MPC_SET_MUX */ 1257 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, 1258 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 1259 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 1260 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); 1261 1262 vcn_v4_0_3_mc_resume(vinst); 1263 1264 /* VCN global tiling registers */ 1265 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, 1266 adev->gfx.config.gb_addr_config); 1267 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, 1268 adev->gfx.config.gb_addr_config); 1269 1270 /* unblock VCPU register access */ 1271 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, 1272 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1273 1274 /* release VCPU reset to boot */ 1275 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1276 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1277 1278 for (j = 0; j < 10; ++j) { 1279 uint32_t status; 1280 1281 for (k = 0; k < 100; ++k) { 1282 status = RREG32_SOC15(VCN, vcn_inst, 1283 regUVD_STATUS); 1284 if (status & 2) 1285 break; 1286 mdelay(10); 1287 } 1288 r = 0; 1289 if (status & 2) 1290 break; 1291 1292 DRM_DEV_ERROR(adev->dev, 1293 "VCN decode not responding, trying to reset the VCPU!!!\n"); 1294 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1295 regUVD_VCPU_CNTL), 1296 UVD_VCPU_CNTL__BLK_RST_MASK, 1297 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1298 mdelay(10); 1299 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1300 regUVD_VCPU_CNTL), 1301 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); 1302 1303 mdelay(10); 1304 r = -1; 1305 } 1306 1307 if (r) { 1308 DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); 1309 return r; 1310 } 1311 1312 /* enable master interrupt */ 1313 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 1314 UVD_MASTINT_EN__VCPU_EN_MASK, 1315 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1316 1317 /* clear the busy bit of VCN_STATUS */ 1318 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, 1319 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); 1320 1321 ring = &adev->vcn.inst[i].ring_enc[0]; 1322 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1323 1324 /* program the RB_BASE for ring buffer */ 1325 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 1326 lower_32_bits(ring->gpu_addr)); 1327 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 1328 upper_32_bits(ring->gpu_addr)); 1329 1330 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 1331 ring->ring_size / sizeof(uint32_t)); 1332 1333 /* resetting ring, fw should not check RB ring */ 1334 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1335 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 1336 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1337 1338 /* Initialize the ring buffer's read and write pointers */ 1339 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 1340 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 1341 1342 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1343 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 1344 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1345 1346 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1347 fw_shared->sq.queue_mode &= 1348 cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); 1349 1350 return 0; 1351 } 1352 1353 /** 1354 * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode 1355 * 1356 * @vinst: VCN instance 1357 * 1358 * Stop VCN block with dpg mode 1359 */ 1360 static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_vcn_inst *vinst) 1361 { 1362 struct amdgpu_device *adev = vinst->adev; 1363 int inst_idx = vinst->inst; 1364 uint32_t tmp; 1365 int vcn_inst; 1366 1367 vcn_inst = GET_INST(VCN, inst_idx); 1368 1369 /* Wait for power status to be 1 */ 1370 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1371 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1372 1373 /* wait for read ptr to be equal to write ptr */ 1374 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1375 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); 1376 1377 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1378 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1379 1380 /* disable dynamic power gating mode */ 1381 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, 1382 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); 1383 1384 /* Keeping one read-back to ensure all register writes are done, 1385 * otherwise it may introduce race conditions. 1386 */ 1387 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1388 1389 return 0; 1390 } 1391 1392 /** 1393 * vcn_v4_0_3_stop - VCN stop 1394 * 1395 * @vinst: VCN instance 1396 * 1397 * Stop VCN block 1398 */ 1399 static int vcn_v4_0_3_stop(struct amdgpu_vcn_inst *vinst) 1400 { 1401 struct amdgpu_device *adev = vinst->adev; 1402 int i = vinst->inst; 1403 volatile struct amdgpu_vcn4_fw_shared *fw_shared; 1404 int r = 0, vcn_inst; 1405 uint32_t tmp; 1406 1407 vcn_inst = GET_INST(VCN, i); 1408 1409 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1410 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; 1411 1412 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { 1413 vcn_v4_0_3_stop_dpg_mode(vinst); 1414 goto Done; 1415 } 1416 1417 /* wait for vcn idle */ 1418 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, 1419 UVD_STATUS__IDLE, 0x7); 1420 if (r) 1421 goto Done; 1422 1423 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | 1424 UVD_LMI_STATUS__READ_CLEAN_MASK | 1425 UVD_LMI_STATUS__WRITE_CLEAN_MASK | 1426 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; 1427 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1428 tmp); 1429 if (r) 1430 goto Done; 1431 1432 /* stall UMC channel */ 1433 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); 1434 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; 1435 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); 1436 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | 1437 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; 1438 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1439 tmp); 1440 if (r) 1441 goto Done; 1442 1443 /* Unblock VCPU Register access */ 1444 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 1445 UVD_RB_ARB_CTRL__VCPU_DIS_MASK, 1446 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1447 1448 /* release VCPU reset to boot */ 1449 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1450 UVD_VCPU_CNTL__BLK_RST_MASK, 1451 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1452 1453 /* disable VCPU clock */ 1454 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1455 ~(UVD_VCPU_CNTL__CLK_EN_MASK)); 1456 1457 /* reset LMI UMC/LMI/VCPU */ 1458 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1459 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1460 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1461 1462 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1463 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1464 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1465 1466 /* clear VCN status */ 1467 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); 1468 1469 /* apply HW clock gating */ 1470 vcn_v4_0_3_enable_clock_gating(vinst); 1471 1472 /* Keeping one read-back to ensure all register writes are done, 1473 * otherwise it may introduce race conditions. 1474 */ 1475 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1476 1477 Done: 1478 return 0; 1479 } 1480 1481 /** 1482 * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode 1483 * 1484 * @vinst: VCN instance 1485 * @new_state: pause state 1486 * 1487 * Pause dpg mode for VCN block 1488 */ 1489 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 1490 struct dpg_pause_state *new_state) 1491 { 1492 1493 return 0; 1494 } 1495 1496 /** 1497 * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer 1498 * 1499 * @ring: amdgpu_ring pointer 1500 * 1501 * Returns the current hardware unified read pointer 1502 */ 1503 static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) 1504 { 1505 struct amdgpu_device *adev = ring->adev; 1506 1507 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1508 DRM_ERROR("wrong ring id is identified in %s", __func__); 1509 1510 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); 1511 } 1512 1513 /** 1514 * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer 1515 * 1516 * @ring: amdgpu_ring pointer 1517 * 1518 * Returns the current hardware unified write pointer 1519 */ 1520 static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) 1521 { 1522 struct amdgpu_device *adev = ring->adev; 1523 1524 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1525 DRM_ERROR("wrong ring id is identified in %s", __func__); 1526 1527 if (ring->use_doorbell) 1528 return *ring->wptr_cpu_addr; 1529 else 1530 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), 1531 regUVD_RB_WPTR); 1532 } 1533 1534 void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 1535 uint32_t val, uint32_t mask) 1536 { 1537 /* Use normalized offsets when required */ 1538 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1539 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1540 1541 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); 1542 amdgpu_ring_write(ring, reg << 2); 1543 amdgpu_ring_write(ring, mask); 1544 amdgpu_ring_write(ring, val); 1545 } 1546 1547 void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 1548 uint32_t val) 1549 { 1550 /* Use normalized offsets when required */ 1551 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1552 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1553 1554 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); 1555 amdgpu_ring_write(ring, reg << 2); 1556 amdgpu_ring_write(ring, val); 1557 } 1558 1559 void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, 1560 unsigned int vmid, uint64_t pd_addr) 1561 { 1562 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; 1563 1564 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 1565 1566 /* wait for reg writes */ 1567 vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + 1568 vmid * hub->ctx_addr_distance, 1569 lower_32_bits(pd_addr), 0xffffffff); 1570 } 1571 1572 void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) 1573 { 1574 /* VCN engine access for HDP flush doesn't work when RRMT is enabled. 1575 * This is a workaround to avoid any HDP flush through VCN ring. 1576 */ 1577 } 1578 1579 /** 1580 * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer 1581 * 1582 * @ring: amdgpu_ring pointer 1583 * 1584 * Commits the enc write pointer to the hardware 1585 */ 1586 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) 1587 { 1588 struct amdgpu_device *adev = ring->adev; 1589 1590 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1591 DRM_ERROR("wrong ring id is identified in %s", __func__); 1592 1593 if (ring->use_doorbell) { 1594 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 1595 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1596 } else { 1597 WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, 1598 lower_32_bits(ring->wptr)); 1599 } 1600 } 1601 1602 static int vcn_v4_0_3_ring_reset(struct amdgpu_ring *ring, 1603 unsigned int vmid, 1604 struct amdgpu_fence *timedout_fence) 1605 { 1606 int r = 0; 1607 int vcn_inst; 1608 struct amdgpu_device *adev = ring->adev; 1609 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me]; 1610 1611 amdgpu_ring_reset_helper_begin(ring, timedout_fence); 1612 1613 vcn_inst = GET_INST(VCN, ring->me); 1614 r = amdgpu_dpm_reset_vcn(adev, 1 << vcn_inst); 1615 1616 if (r) { 1617 DRM_DEV_ERROR(adev->dev, "VCN reset fail : %d\n", r); 1618 return r; 1619 } 1620 1621 /* This flag is not set for VF, assumed to be disabled always */ 1622 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x100) 1623 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 1624 vcn_v4_0_3_hw_init_inst(vinst); 1625 vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[ring->me].indirect_sram); 1626 1627 return amdgpu_ring_reset_helper_end(ring, timedout_fence); 1628 } 1629 1630 static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { 1631 .type = AMDGPU_RING_TYPE_VCN_ENC, 1632 .align_mask = 0x3f, 1633 .nop = VCN_ENC_CMD_NO_OP, 1634 .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, 1635 .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, 1636 .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, 1637 .emit_frame_size = 1638 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 1639 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + 1640 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ 1641 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ 1642 1, /* vcn_v2_0_enc_ring_insert_end */ 1643 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ 1644 .emit_ib = vcn_v2_0_enc_ring_emit_ib, 1645 .emit_fence = vcn_v2_0_enc_ring_emit_fence, 1646 .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, 1647 .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, 1648 .test_ring = amdgpu_vcn_enc_ring_test_ring, 1649 .test_ib = amdgpu_vcn_unified_ring_test_ib, 1650 .insert_nop = amdgpu_ring_insert_nop, 1651 .insert_end = vcn_v2_0_enc_ring_insert_end, 1652 .pad_ib = amdgpu_ring_generic_pad_ib, 1653 .begin_use = amdgpu_vcn_ring_begin_use, 1654 .end_use = amdgpu_vcn_ring_end_use, 1655 .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, 1656 .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, 1657 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1658 .reset = vcn_v4_0_3_ring_reset, 1659 }; 1660 1661 /** 1662 * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions 1663 * 1664 * @adev: amdgpu_device pointer 1665 * 1666 * Set unified ring functions 1667 */ 1668 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) 1669 { 1670 int i, vcn_inst; 1671 1672 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1673 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; 1674 adev->vcn.inst[i].ring_enc[0].me = i; 1675 vcn_inst = GET_INST(VCN, i); 1676 adev->vcn.inst[i].aid_id = 1677 vcn_inst / adev->vcn.num_inst_per_aid; 1678 } 1679 } 1680 1681 /** 1682 * vcn_v4_0_3_is_idle - check VCN block is idle 1683 * 1684 * @ip_block: Pointer to the amdgpu_ip_block structure 1685 * 1686 * Check whether VCN block is idle 1687 */ 1688 static bool vcn_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block) 1689 { 1690 struct amdgpu_device *adev = ip_block->adev; 1691 int i, ret = 1; 1692 1693 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1694 ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == 1695 UVD_STATUS__IDLE); 1696 } 1697 1698 return ret; 1699 } 1700 1701 /** 1702 * vcn_v4_0_3_wait_for_idle - wait for VCN block idle 1703 * 1704 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 1705 * 1706 * Wait for VCN block idle 1707 */ 1708 static int vcn_v4_0_3_wait_for_idle(struct amdgpu_ip_block *ip_block) 1709 { 1710 struct amdgpu_device *adev = ip_block->adev; 1711 int i, ret = 0; 1712 1713 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1714 ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, 1715 UVD_STATUS__IDLE, UVD_STATUS__IDLE); 1716 if (ret) 1717 return ret; 1718 } 1719 1720 return ret; 1721 } 1722 1723 /* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state 1724 * 1725 * @ip_block: amdgpu_ip_block pointer 1726 * @state: clock gating state 1727 * 1728 * Set VCN block clockgating state 1729 */ 1730 static int vcn_v4_0_3_set_clockgating_state(struct amdgpu_ip_block *ip_block, 1731 enum amd_clockgating_state state) 1732 { 1733 struct amdgpu_device *adev = ip_block->adev; 1734 bool enable = state == AMD_CG_STATE_GATE; 1735 int i; 1736 1737 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1738 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 1739 1740 if (enable) { 1741 if (RREG32_SOC15(VCN, GET_INST(VCN, i), 1742 regUVD_STATUS) != UVD_STATUS__IDLE) 1743 return -EBUSY; 1744 vcn_v4_0_3_enable_clock_gating(vinst); 1745 } else { 1746 vcn_v4_0_3_disable_clock_gating(vinst); 1747 } 1748 } 1749 return 0; 1750 } 1751 1752 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 1753 enum amd_powergating_state state) 1754 { 1755 struct amdgpu_device *adev = vinst->adev; 1756 int ret = 0; 1757 1758 /* for SRIOV, guest should not control VCN Power-gating 1759 * MMSCH FW should control Power-gating and clock-gating 1760 * guest should avoid touching CGC and PG 1761 */ 1762 if (amdgpu_sriov_vf(adev)) { 1763 vinst->cur_state = AMD_PG_STATE_UNGATE; 1764 return 0; 1765 } 1766 1767 if (state == vinst->cur_state) 1768 return 0; 1769 1770 if (state == AMD_PG_STATE_GATE) 1771 ret = vcn_v4_0_3_stop(vinst); 1772 else 1773 ret = vcn_v4_0_3_start(vinst); 1774 1775 if (!ret) 1776 vinst->cur_state = state; 1777 1778 return ret; 1779 } 1780 1781 /** 1782 * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state 1783 * 1784 * @adev: amdgpu_device pointer 1785 * @source: interrupt sources 1786 * @type: interrupt types 1787 * @state: interrupt states 1788 * 1789 * Set VCN block interrupt state 1790 */ 1791 static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, 1792 struct amdgpu_irq_src *source, 1793 unsigned int type, 1794 enum amdgpu_interrupt_state state) 1795 { 1796 return 0; 1797 } 1798 1799 /** 1800 * vcn_v4_0_3_process_interrupt - process VCN block interrupt 1801 * 1802 * @adev: amdgpu_device pointer 1803 * @source: interrupt sources 1804 * @entry: interrupt entry from clients and sources 1805 * 1806 * Process VCN block interrupt 1807 */ 1808 static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, 1809 struct amdgpu_irq_src *source, 1810 struct amdgpu_iv_entry *entry) 1811 { 1812 uint32_t i, inst; 1813 1814 i = node_id_to_phys_map[entry->node_id]; 1815 1816 DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); 1817 1818 for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) 1819 if (adev->vcn.inst[inst].aid_id == i) 1820 break; 1821 1822 if (inst >= adev->vcn.num_vcn_inst) { 1823 dev_WARN_ONCE(adev->dev, 1, 1824 "Interrupt received for unknown VCN instance %d", 1825 entry->node_id); 1826 return 0; 1827 } 1828 1829 switch (entry->src_id) { 1830 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: 1831 amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); 1832 break; 1833 default: 1834 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", 1835 entry->src_id, entry->src_data[0]); 1836 break; 1837 } 1838 1839 return 0; 1840 } 1841 1842 static int vcn_v4_0_3_set_ras_interrupt_state(struct amdgpu_device *adev, 1843 struct amdgpu_irq_src *source, 1844 unsigned int type, 1845 enum amdgpu_interrupt_state state) 1846 { 1847 return 0; 1848 } 1849 1850 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { 1851 .set = vcn_v4_0_3_set_interrupt_state, 1852 .process = vcn_v4_0_3_process_interrupt, 1853 }; 1854 1855 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_ras_irq_funcs = { 1856 .set = vcn_v4_0_3_set_ras_interrupt_state, 1857 .process = amdgpu_vcn_process_poison_irq, 1858 }; 1859 1860 /** 1861 * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions 1862 * 1863 * @adev: amdgpu_device pointer 1864 * 1865 * Set VCN block interrupt irq functions 1866 */ 1867 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) 1868 { 1869 int i; 1870 1871 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1872 adev->vcn.inst->irq.num_types++; 1873 } 1874 adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; 1875 1876 adev->vcn.inst->ras_poison_irq.num_types = 1; 1877 adev->vcn.inst->ras_poison_irq.funcs = &vcn_v4_0_3_ras_irq_funcs; 1878 } 1879 1880 static void vcn_v4_0_3_print_ip_state(struct amdgpu_ip_block *ip_block, struct drm_printer *p) 1881 { 1882 struct amdgpu_device *adev = ip_block->adev; 1883 int i, j; 1884 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); 1885 uint32_t inst_off, is_powered; 1886 1887 if (!adev->vcn.ip_dump) 1888 return; 1889 1890 drm_printf(p, "num_instances:%d\n", adev->vcn.num_vcn_inst); 1891 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1892 if (adev->vcn.harvest_config & (1 << i)) { 1893 drm_printf(p, "\nHarvested Instance:VCN%d Skipping dump\n", i); 1894 continue; 1895 } 1896 1897 inst_off = i * reg_count; 1898 is_powered = (adev->vcn.ip_dump[inst_off] & 1899 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; 1900 1901 if (is_powered) { 1902 drm_printf(p, "\nActive Instance:VCN%d\n", i); 1903 for (j = 0; j < reg_count; j++) 1904 drm_printf(p, "%-50s \t 0x%08x\n", vcn_reg_list_4_0_3[j].reg_name, 1905 adev->vcn.ip_dump[inst_off + j]); 1906 } else { 1907 drm_printf(p, "\nInactive Instance:VCN%d\n", i); 1908 } 1909 } 1910 } 1911 1912 static void vcn_v4_0_3_dump_ip_state(struct amdgpu_ip_block *ip_block) 1913 { 1914 struct amdgpu_device *adev = ip_block->adev; 1915 int i, j; 1916 bool is_powered; 1917 uint32_t inst_off, inst_id; 1918 uint32_t reg_count = ARRAY_SIZE(vcn_reg_list_4_0_3); 1919 1920 if (!adev->vcn.ip_dump) 1921 return; 1922 1923 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1924 if (adev->vcn.harvest_config & (1 << i)) 1925 continue; 1926 1927 inst_id = GET_INST(VCN, i); 1928 inst_off = i * reg_count; 1929 /* mmUVD_POWER_STATUS is always readable and is first element of the array */ 1930 adev->vcn.ip_dump[inst_off] = RREG32_SOC15(VCN, inst_id, regUVD_POWER_STATUS); 1931 is_powered = (adev->vcn.ip_dump[inst_off] & 1932 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK) != 1; 1933 1934 if (is_powered) 1935 for (j = 1; j < reg_count; j++) 1936 adev->vcn.ip_dump[inst_off + j] = 1937 RREG32(SOC15_REG_ENTRY_OFFSET_INST(vcn_reg_list_4_0_3[j], 1938 inst_id)); 1939 } 1940 } 1941 1942 static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { 1943 .name = "vcn_v4_0_3", 1944 .early_init = vcn_v4_0_3_early_init, 1945 .sw_init = vcn_v4_0_3_sw_init, 1946 .sw_fini = vcn_v4_0_3_sw_fini, 1947 .hw_init = vcn_v4_0_3_hw_init, 1948 .hw_fini = vcn_v4_0_3_hw_fini, 1949 .suspend = vcn_v4_0_3_suspend, 1950 .resume = vcn_v4_0_3_resume, 1951 .is_idle = vcn_v4_0_3_is_idle, 1952 .wait_for_idle = vcn_v4_0_3_wait_for_idle, 1953 .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, 1954 .set_powergating_state = vcn_set_powergating_state, 1955 .dump_ip_state = vcn_v4_0_3_dump_ip_state, 1956 .print_ip_state = vcn_v4_0_3_print_ip_state, 1957 }; 1958 1959 const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { 1960 .type = AMD_IP_BLOCK_TYPE_VCN, 1961 .major = 4, 1962 .minor = 0, 1963 .rev = 3, 1964 .funcs = &vcn_v4_0_3_ip_funcs, 1965 }; 1966 1967 static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { 1968 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), 1969 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, 1970 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), 1971 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, 1972 }; 1973 1974 static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, 1975 uint32_t vcn_inst, 1976 void *ras_err_status) 1977 { 1978 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; 1979 1980 /* vcn v4_0_3 only support query uncorrectable errors */ 1981 amdgpu_ras_inst_query_ras_error_count(adev, 1982 vcn_v4_0_3_ue_reg_list, 1983 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 1984 NULL, 0, GET_INST(VCN, vcn_inst), 1985 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 1986 &err_data->ue_count); 1987 } 1988 1989 static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, 1990 void *ras_err_status) 1991 { 1992 uint32_t i; 1993 1994 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 1995 dev_warn(adev->dev, "VCN RAS is not supported\n"); 1996 return; 1997 } 1998 1999 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 2000 vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); 2001 } 2002 2003 static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, 2004 uint32_t vcn_inst) 2005 { 2006 amdgpu_ras_inst_reset_ras_error_count(adev, 2007 vcn_v4_0_3_ue_reg_list, 2008 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 2009 GET_INST(VCN, vcn_inst)); 2010 } 2011 2012 static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) 2013 { 2014 uint32_t i; 2015 2016 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 2017 dev_warn(adev->dev, "VCN RAS is not supported\n"); 2018 return; 2019 } 2020 2021 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 2022 vcn_v4_0_3_inst_reset_ras_error_count(adev, i); 2023 } 2024 2025 static uint32_t vcn_v4_0_3_query_poison_by_instance(struct amdgpu_device *adev, 2026 uint32_t instance, uint32_t sub_block) 2027 { 2028 uint32_t poison_stat = 0, reg_value = 0; 2029 2030 switch (sub_block) { 2031 case AMDGPU_VCN_V4_0_3_VCPU_VCODEC: 2032 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS); 2033 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF); 2034 break; 2035 default: 2036 break; 2037 } 2038 2039 if (poison_stat) 2040 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n", 2041 instance, sub_block); 2042 2043 return poison_stat; 2044 } 2045 2046 static bool vcn_v4_0_3_query_poison_status(struct amdgpu_device *adev) 2047 { 2048 uint32_t inst, sub; 2049 uint32_t poison_stat = 0; 2050 2051 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++) 2052 for (sub = 0; sub < AMDGPU_VCN_V4_0_3_MAX_SUB_BLOCK; sub++) 2053 poison_stat += 2054 vcn_v4_0_3_query_poison_by_instance(adev, inst, sub); 2055 2056 return !!poison_stat; 2057 } 2058 2059 static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { 2060 .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, 2061 .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, 2062 .query_poison_status = vcn_v4_0_3_query_poison_status, 2063 }; 2064 2065 static int vcn_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, 2066 enum aca_smu_type type, void *data) 2067 { 2068 struct aca_bank_info info; 2069 u64 misc0; 2070 int ret; 2071 2072 ret = aca_bank_info_decode(bank, &info); 2073 if (ret) 2074 return ret; 2075 2076 misc0 = bank->regs[ACA_REG_IDX_MISC0]; 2077 switch (type) { 2078 case ACA_SMU_TYPE_UE: 2079 bank->aca_err_type = ACA_ERROR_TYPE_UE; 2080 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, 2081 1ULL); 2082 break; 2083 case ACA_SMU_TYPE_CE: 2084 bank->aca_err_type = ACA_ERROR_TYPE_CE; 2085 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 2086 ACA_REG__MISC0__ERRCNT(misc0)); 2087 break; 2088 default: 2089 return -EINVAL; 2090 } 2091 2092 return ret; 2093 } 2094 2095 /* reference to smu driver if header file */ 2096 static int vcn_v4_0_3_err_codes[] = { 2097 14, 15, /* VCN */ 2098 }; 2099 2100 static bool vcn_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, 2101 enum aca_smu_type type, void *data) 2102 { 2103 u32 instlo; 2104 2105 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); 2106 instlo &= GENMASK(31, 1); 2107 2108 if (instlo != mmSMNAID_AID0_MCA_SMU) 2109 return false; 2110 2111 if (aca_bank_check_error_codes(handle->adev, bank, 2112 vcn_v4_0_3_err_codes, 2113 ARRAY_SIZE(vcn_v4_0_3_err_codes))) 2114 return false; 2115 2116 return true; 2117 } 2118 2119 static const struct aca_bank_ops vcn_v4_0_3_aca_bank_ops = { 2120 .aca_bank_parser = vcn_v4_0_3_aca_bank_parser, 2121 .aca_bank_is_valid = vcn_v4_0_3_aca_bank_is_valid, 2122 }; 2123 2124 static const struct aca_info vcn_v4_0_3_aca_info = { 2125 .hwip = ACA_HWIP_TYPE_SMU, 2126 .mask = ACA_ERROR_UE_MASK, 2127 .bank_ops = &vcn_v4_0_3_aca_bank_ops, 2128 }; 2129 2130 static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 2131 { 2132 int r; 2133 2134 r = amdgpu_ras_block_late_init(adev, ras_block); 2135 if (r) 2136 return r; 2137 2138 if (amdgpu_ras_is_supported(adev, ras_block->block) && 2139 adev->vcn.inst->ras_poison_irq.funcs) { 2140 r = amdgpu_irq_get(adev, &adev->vcn.inst->ras_poison_irq, 0); 2141 if (r) 2142 goto late_fini; 2143 } 2144 2145 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN, 2146 &vcn_v4_0_3_aca_info, NULL); 2147 if (r) 2148 goto late_fini; 2149 2150 return 0; 2151 2152 late_fini: 2153 amdgpu_ras_block_late_fini(adev, ras_block); 2154 2155 return r; 2156 } 2157 2158 static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { 2159 .ras_block = { 2160 .hw_ops = &vcn_v4_0_3_ras_hw_ops, 2161 .ras_late_init = vcn_v4_0_3_ras_late_init, 2162 }, 2163 }; 2164 2165 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) 2166 { 2167 adev->vcn.ras = &vcn_v4_0_3_ras; 2168 } 2169 2170 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 2171 int inst_idx, bool indirect) 2172 { 2173 uint32_t tmp; 2174 2175 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) 2176 return; 2177 2178 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | 2179 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | 2180 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | 2181 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; 2182 WREG32_SOC15_DPG_MODE(inst_idx, 2183 SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), 2184 tmp, 0, indirect); 2185 2186 tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; 2187 WREG32_SOC15_DPG_MODE(inst_idx, 2188 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), 2189 tmp, 0, indirect); 2190 2191 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; 2192 WREG32_SOC15_DPG_MODE(inst_idx, 2193 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), 2194 tmp, 0, indirect); 2195 } 2196