1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_drv.h> 26 27 #include "amdgpu.h" 28 #include "amdgpu_vcn.h" 29 #include "amdgpu_pm.h" 30 #include "soc15.h" 31 #include "soc15d.h" 32 #include "soc15_hw_ip.h" 33 #include "vcn_v2_0.h" 34 #include "vcn_v4_0_3.h" 35 #include "mmsch_v4_0_3.h" 36 37 #include "vcn/vcn_4_0_3_offset.h" 38 #include "vcn/vcn_4_0_3_sh_mask.h" 39 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" 40 41 #define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL 42 #define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX 43 #define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA 44 #define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX 45 46 #define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 47 #define VCN1_VID_SOC_ADDRESS_3_0 0x48300 48 #define VCN1_AON_SOC_ADDRESS_3_0 0x48000 49 50 static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { 51 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), 52 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), 53 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), 54 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), 55 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), 56 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), 57 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), 58 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), 59 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), 60 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), 61 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), 62 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), 63 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), 64 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), 65 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), 66 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), 67 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), 68 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), 69 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), 70 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), 71 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), 72 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), 73 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), 74 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), 75 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), 76 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), 77 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), 78 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), 79 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), 80 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), 81 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), 82 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), 83 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) 84 }; 85 86 #define NORMALIZE_VCN_REG_OFFSET(offset) \ 87 (offset & 0x1FFFF) 88 89 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); 90 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); 91 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); 92 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 93 enum amd_powergating_state state); 94 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 95 struct dpg_pause_state *new_state); 96 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); 97 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); 98 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 99 int inst_idx, bool indirect); 100 101 static inline bool vcn_v4_0_3_normalizn_reqd(struct amdgpu_device *adev) 102 { 103 return (adev->vcn.caps & AMDGPU_VCN_CAPS(RRMT_ENABLED)) == 0; 104 } 105 106 /** 107 * vcn_v4_0_3_early_init - set function pointers 108 * 109 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 110 * 111 * Set ring and irq function pointers 112 */ 113 static int vcn_v4_0_3_early_init(struct amdgpu_ip_block *ip_block) 114 { 115 struct amdgpu_device *adev = ip_block->adev; 116 int i, r; 117 118 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) 119 /* re-use enc ring as unified ring */ 120 adev->vcn.inst[i].num_enc_rings = 1; 121 122 vcn_v4_0_3_set_unified_ring_funcs(adev); 123 vcn_v4_0_3_set_irq_funcs(adev); 124 vcn_v4_0_3_set_ras_funcs(adev); 125 126 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 127 adev->vcn.inst[i].set_pg_state = vcn_v4_0_3_set_pg_state; 128 129 r = amdgpu_vcn_early_init(adev, i); 130 if (r) 131 return r; 132 } 133 134 return 0; 135 } 136 137 static int vcn_v4_0_3_late_init(struct amdgpu_ip_block *ip_block) 138 { 139 struct amdgpu_device *adev = ip_block->adev; 140 141 adev->vcn.supported_reset = 142 amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]); 143 144 if (amdgpu_dpm_reset_vcn_is_supported(adev)) 145 adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 146 147 return 0; 148 } 149 150 static int vcn_v4_0_3_fw_shared_init(struct amdgpu_device *adev, int inst_idx) 151 { 152 struct amdgpu_vcn4_fw_shared *fw_shared; 153 154 fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 155 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); 156 fw_shared->sq.is_enabled = 1; 157 158 if (amdgpu_vcnfw_log) 159 amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]); 160 161 return 0; 162 } 163 164 /** 165 * vcn_v4_0_3_sw_init - sw init for VCN block 166 * 167 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 168 * 169 * Load firmware and sw initialization 170 */ 171 static int vcn_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block) 172 { 173 struct amdgpu_device *adev = ip_block->adev; 174 struct amdgpu_ring *ring; 175 int i, r, vcn_inst; 176 177 /* VCN DEC TRAP */ 178 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 179 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); 180 if (r) 181 return r; 182 183 /* VCN POISON TRAP */ 184 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 185 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst->ras_poison_irq); 186 187 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 188 189 r = amdgpu_vcn_sw_init(adev, i); 190 if (r) 191 return r; 192 193 amdgpu_vcn_setup_ucode(adev, i); 194 195 r = amdgpu_vcn_resume(adev, i); 196 if (r) 197 return r; 198 199 vcn_inst = GET_INST(VCN, i); 200 201 ring = &adev->vcn.inst[i].ring_enc[0]; 202 ring->use_doorbell = true; 203 204 if (!amdgpu_sriov_vf(adev)) 205 ring->doorbell_index = 206 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 207 9 * vcn_inst; 208 else 209 ring->doorbell_index = 210 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 211 32 * vcn_inst; 212 213 ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); 214 sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); 215 216 /* There are no per-instance irq source IDs on 4.0.3, the IH 217 * packets use a separate field to differentiate instances. 218 */ 219 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[0].irq, 0, 220 AMDGPU_RING_PRIO_DEFAULT, 221 &adev->vcn.inst[i].sched_score); 222 if (r) 223 return r; 224 225 vcn_v4_0_3_fw_shared_init(adev, i); 226 227 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 228 adev->vcn.inst[i].pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; 229 } 230 231 if (amdgpu_sriov_vf(adev)) { 232 r = amdgpu_virt_alloc_mm_table(adev); 233 if (r) 234 return r; 235 } 236 237 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 238 r = amdgpu_vcn_ras_sw_init(adev); 239 if (r) { 240 dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); 241 return r; 242 } 243 } 244 245 r = amdgpu_vcn_reg_dump_init(adev, vcn_reg_list_4_0_3, ARRAY_SIZE(vcn_reg_list_4_0_3)); 246 if (r) 247 return r; 248 249 return amdgpu_vcn_sysfs_reset_mask_init(adev); 250 } 251 252 /** 253 * vcn_v4_0_3_sw_fini - sw fini for VCN block 254 * 255 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 256 * 257 * VCN suspend and free up sw allocation 258 */ 259 static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block) 260 { 261 struct amdgpu_device *adev = ip_block->adev; 262 int i, r, idx; 263 264 if (drm_dev_enter(&adev->ddev, &idx)) { 265 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 266 struct amdgpu_vcn4_fw_shared *fw_shared; 267 268 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 269 fw_shared->present_flag_0 = 0; 270 fw_shared->sq.is_enabled = cpu_to_le32(false); 271 } 272 drm_dev_exit(idx); 273 } 274 275 if (amdgpu_sriov_vf(adev)) 276 amdgpu_virt_free_mm_table(adev); 277 278 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 279 r = amdgpu_vcn_suspend(adev, i); 280 if (r) 281 return r; 282 } 283 284 amdgpu_vcn_sysfs_reset_mask_fini(adev); 285 286 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 287 amdgpu_vcn_sw_fini(adev, i); 288 289 return 0; 290 } 291 292 static int vcn_v4_0_3_hw_init_inst(struct amdgpu_vcn_inst *vinst) 293 { 294 int vcn_inst; 295 struct amdgpu_device *adev = vinst->adev; 296 struct amdgpu_ring *ring; 297 int inst_idx = vinst->inst; 298 299 vcn_inst = GET_INST(VCN, inst_idx); 300 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 301 if (ring->use_doorbell) { 302 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, 303 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 9 * vcn_inst, 304 adev->vcn.inst[inst_idx].aid_id); 305 306 WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL, 307 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT | 308 VCN_RB1_DB_CTRL__EN_MASK); 309 310 /* Read DB_CTRL to flush the write DB_CTRL command. */ 311 RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL); 312 } 313 314 return 0; 315 } 316 317 /** 318 * vcn_v4_0_3_hw_init - start and test VCN block 319 * 320 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 321 * 322 * Initialize the hardware, boot up the VCPU and do some testing 323 */ 324 static int vcn_v4_0_3_hw_init(struct amdgpu_ip_block *ip_block) 325 { 326 struct amdgpu_device *adev = ip_block->adev; 327 struct amdgpu_ring *ring; 328 struct amdgpu_vcn_inst *vinst; 329 int i, r; 330 331 if (amdgpu_sriov_vf(adev)) { 332 r = vcn_v4_0_3_start_sriov(adev); 333 if (r) 334 return r; 335 336 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 337 ring = &adev->vcn.inst[i].ring_enc[0]; 338 ring->wptr = 0; 339 ring->wptr_old = 0; 340 vcn_v4_0_3_unified_ring_set_wptr(ring); 341 ring->sched.ready = true; 342 } 343 } else { 344 /* This flag is not set for VF, assumed to be disabled always */ 345 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 346 0x100) 347 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 348 349 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 350 struct amdgpu_vcn4_fw_shared *fw_shared; 351 352 ring = &adev->vcn.inst[i].ring_enc[0]; 353 vinst = &adev->vcn.inst[i]; 354 vcn_v4_0_3_hw_init_inst(vinst); 355 356 /* Re-init fw_shared when RAS fatal error occurred */ 357 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 358 if (!fw_shared->sq.is_enabled) 359 vcn_v4_0_3_fw_shared_init(adev, i); 360 361 r = amdgpu_ring_test_helper(ring); 362 if (r) 363 return r; 364 } 365 } 366 367 return r; 368 } 369 370 /** 371 * vcn_v4_0_3_hw_fini - stop the hardware block 372 * 373 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 374 * 375 * Stop the VCN block, mark ring as not ready any more 376 */ 377 static int vcn_v4_0_3_hw_fini(struct amdgpu_ip_block *ip_block) 378 { 379 struct amdgpu_device *adev = ip_block->adev; 380 int i; 381 382 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 383 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 384 385 cancel_delayed_work_sync(&vinst->idle_work); 386 387 if (vinst->cur_state != AMD_PG_STATE_GATE) 388 vinst->set_pg_state(vinst, AMD_PG_STATE_GATE); 389 } 390 391 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN) && !amdgpu_sriov_vf(adev)) 392 amdgpu_irq_put(adev, &adev->vcn.inst->ras_poison_irq, 0); 393 394 return 0; 395 } 396 397 /** 398 * vcn_v4_0_3_suspend - suspend VCN block 399 * 400 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 401 * 402 * HW fini and suspend VCN block 403 */ 404 static int vcn_v4_0_3_suspend(struct amdgpu_ip_block *ip_block) 405 { 406 struct amdgpu_device *adev = ip_block->adev; 407 int r, i; 408 409 r = vcn_v4_0_3_hw_fini(ip_block); 410 if (r) 411 return r; 412 413 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 414 r = amdgpu_vcn_suspend(adev, i); 415 if (r) 416 return r; 417 } 418 419 return 0; 420 } 421 422 /** 423 * vcn_v4_0_3_resume - resume VCN block 424 * 425 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 426 * 427 * Resume firmware and hw init VCN block 428 */ 429 static int vcn_v4_0_3_resume(struct amdgpu_ip_block *ip_block) 430 { 431 struct amdgpu_device *adev = ip_block->adev; 432 int r, i; 433 434 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 435 r = amdgpu_vcn_resume(ip_block->adev, i); 436 if (r) 437 return r; 438 } 439 440 r = vcn_v4_0_3_hw_init(ip_block); 441 442 return r; 443 } 444 445 /** 446 * vcn_v4_0_3_mc_resume - memory controller programming 447 * 448 * @vinst: VCN instance 449 * 450 * Let the VCN memory controller know it's offsets 451 */ 452 static void vcn_v4_0_3_mc_resume(struct amdgpu_vcn_inst *vinst) 453 { 454 struct amdgpu_device *adev = vinst->adev; 455 int inst_idx = vinst->inst; 456 uint32_t offset, size, vcn_inst; 457 const struct common_firmware_header *hdr; 458 459 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 460 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 461 462 vcn_inst = GET_INST(VCN, inst_idx); 463 /* cache window 0: fw */ 464 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 465 WREG32_SOC15( 466 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 467 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 468 .tmr_mc_addr_lo)); 469 WREG32_SOC15( 470 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 471 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 472 .tmr_mc_addr_hi)); 473 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); 474 offset = 0; 475 } else { 476 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 477 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 478 WREG32_SOC15(VCN, vcn_inst, 479 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 480 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 481 offset = size; 482 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 483 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 484 } 485 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); 486 487 /* cache window 1: stack */ 488 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, 489 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 490 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, 491 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 492 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); 493 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, 494 AMDGPU_VCN_STACK_SIZE); 495 496 /* cache window 2: context */ 497 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, 498 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 499 AMDGPU_VCN_STACK_SIZE)); 500 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, 501 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 502 AMDGPU_VCN_STACK_SIZE)); 503 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); 504 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, 505 AMDGPU_VCN_CONTEXT_SIZE); 506 507 /* non-cache window */ 508 WREG32_SOC15( 509 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, 510 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 511 WREG32_SOC15( 512 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, 513 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 514 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); 515 WREG32_SOC15( 516 VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, 517 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 518 } 519 520 /** 521 * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode 522 * 523 * @vinst: VCN instance 524 * @indirect: indirectly write sram 525 * 526 * Let the VCN memory controller know it's offsets with dpg mode 527 */ 528 static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst, 529 bool indirect) 530 { 531 struct amdgpu_device *adev = vinst->adev; 532 int inst_idx = vinst->inst; 533 uint32_t offset, size; 534 const struct common_firmware_header *hdr; 535 536 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 537 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 538 539 /* cache window 0: fw */ 540 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 541 if (!indirect) { 542 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 543 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 544 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 545 inst_idx].tmr_mc_addr_lo), 0, indirect); 546 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 547 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 548 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 549 inst_idx].tmr_mc_addr_hi), 0, indirect); 550 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 551 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 552 } else { 553 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 554 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); 555 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 556 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); 557 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 558 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 559 } 560 offset = 0; 561 } else { 562 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 563 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 564 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 565 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 566 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 567 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 568 offset = size; 569 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 570 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 571 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); 572 } 573 574 if (!indirect) 575 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 576 VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); 577 else 578 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 579 VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); 580 581 /* cache window 1: stack */ 582 if (!indirect) { 583 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 584 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 585 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 586 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 587 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 588 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 589 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 590 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 591 } else { 592 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 593 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); 594 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 595 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); 596 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 597 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 598 } 599 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 600 VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); 601 602 /* cache window 2: context */ 603 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 604 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 605 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 606 AMDGPU_VCN_STACK_SIZE), 0, indirect); 607 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 608 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 609 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 610 AMDGPU_VCN_STACK_SIZE), 0, indirect); 611 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 612 VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); 613 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 614 VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); 615 616 /* non-cache window */ 617 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 618 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 619 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 620 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 621 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 622 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 623 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 624 VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); 625 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 626 VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), 627 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); 628 629 /* VCN global tiling registers */ 630 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 631 VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 632 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 633 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 634 } 635 636 /** 637 * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating 638 * 639 * @vinst: VCN instance 640 * 641 * Disable clock gating for VCN block 642 */ 643 static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_vcn_inst *vinst) 644 { 645 struct amdgpu_device *adev = vinst->adev; 646 int inst_idx = vinst->inst; 647 uint32_t data; 648 int vcn_inst; 649 650 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 651 return; 652 653 vcn_inst = GET_INST(VCN, inst_idx); 654 655 /* VCN disable CGC */ 656 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 657 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; 658 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 659 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 660 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 661 662 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); 663 data &= ~(UVD_CGC_GATE__SYS_MASK 664 | UVD_CGC_GATE__MPEG2_MASK 665 | UVD_CGC_GATE__REGS_MASK 666 | UVD_CGC_GATE__RBC_MASK 667 | UVD_CGC_GATE__LMI_MC_MASK 668 | UVD_CGC_GATE__LMI_UMC_MASK 669 | UVD_CGC_GATE__MPC_MASK 670 | UVD_CGC_GATE__LBSI_MASK 671 | UVD_CGC_GATE__LRBBM_MASK 672 | UVD_CGC_GATE__WCB_MASK 673 | UVD_CGC_GATE__VCPU_MASK 674 | UVD_CGC_GATE__MMSCH_MASK); 675 676 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); 677 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); 678 679 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 680 data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK 681 | UVD_CGC_CTRL__MPEG2_MODE_MASK 682 | UVD_CGC_CTRL__REGS_MODE_MASK 683 | UVD_CGC_CTRL__RBC_MODE_MASK 684 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 685 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 686 | UVD_CGC_CTRL__MPC_MODE_MASK 687 | UVD_CGC_CTRL__LBSI_MODE_MASK 688 | UVD_CGC_CTRL__LRBBM_MODE_MASK 689 | UVD_CGC_CTRL__WCB_MODE_MASK 690 | UVD_CGC_CTRL__VCPU_MODE_MASK 691 | UVD_CGC_CTRL__MMSCH_MODE_MASK); 692 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 693 694 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); 695 data |= (UVD_SUVD_CGC_GATE__SRE_MASK 696 | UVD_SUVD_CGC_GATE__SIT_MASK 697 | UVD_SUVD_CGC_GATE__SMP_MASK 698 | UVD_SUVD_CGC_GATE__SCM_MASK 699 | UVD_SUVD_CGC_GATE__SDB_MASK 700 | UVD_SUVD_CGC_GATE__SRE_H264_MASK 701 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK 702 | UVD_SUVD_CGC_GATE__SIT_H264_MASK 703 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK 704 | UVD_SUVD_CGC_GATE__SCM_H264_MASK 705 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK 706 | UVD_SUVD_CGC_GATE__SDB_H264_MASK 707 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK 708 | UVD_SUVD_CGC_GATE__ENT_MASK 709 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK 710 | UVD_SUVD_CGC_GATE__SITE_MASK 711 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK 712 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK 713 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK 714 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK 715 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); 716 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); 717 718 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 719 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 720 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 721 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 722 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 723 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 724 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 725 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 726 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 727 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 728 } 729 730 /** 731 * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode 732 * 733 * @vinst: VCN instance 734 * @sram_sel: sram select 735 * @indirect: indirectly write sram 736 * 737 * Disable clock gating for VCN block with dpg mode 738 */ 739 static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst, 740 uint8_t sram_sel, 741 uint8_t indirect) 742 { 743 struct amdgpu_device *adev = vinst->adev; 744 int inst_idx = vinst->inst; 745 uint32_t reg_data = 0; 746 747 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 748 return; 749 750 /* enable sw clock gating control */ 751 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 752 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 753 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 754 reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | 755 UVD_CGC_CTRL__MPEG2_MODE_MASK | 756 UVD_CGC_CTRL__REGS_MODE_MASK | 757 UVD_CGC_CTRL__RBC_MODE_MASK | 758 UVD_CGC_CTRL__LMI_MC_MODE_MASK | 759 UVD_CGC_CTRL__LMI_UMC_MODE_MASK | 760 UVD_CGC_CTRL__IDCT_MODE_MASK | 761 UVD_CGC_CTRL__MPRD_MODE_MASK | 762 UVD_CGC_CTRL__MPC_MODE_MASK | 763 UVD_CGC_CTRL__LBSI_MODE_MASK | 764 UVD_CGC_CTRL__LRBBM_MODE_MASK | 765 UVD_CGC_CTRL__WCB_MODE_MASK | 766 UVD_CGC_CTRL__VCPU_MODE_MASK); 767 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 768 VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); 769 770 /* turn off clock gating */ 771 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 772 VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); 773 774 /* turn on SUVD clock gating */ 775 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 776 VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); 777 778 /* turn on sw mode in UVD_SUVD_CGC_CTRL */ 779 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 780 VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); 781 } 782 783 /** 784 * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating 785 * 786 * @vinst: VCN instance 787 * 788 * Enable clock gating for VCN block 789 */ 790 static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_vcn_inst *vinst) 791 { 792 struct amdgpu_device *adev = vinst->adev; 793 int inst_idx = vinst->inst; 794 uint32_t data; 795 int vcn_inst; 796 797 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 798 return; 799 800 vcn_inst = GET_INST(VCN, inst_idx); 801 802 /* enable VCN CGC */ 803 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 804 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 805 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 806 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 807 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 808 809 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 810 data |= (UVD_CGC_CTRL__SYS_MODE_MASK 811 | UVD_CGC_CTRL__MPEG2_MODE_MASK 812 | UVD_CGC_CTRL__REGS_MODE_MASK 813 | UVD_CGC_CTRL__RBC_MODE_MASK 814 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 815 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 816 | UVD_CGC_CTRL__MPC_MODE_MASK 817 | UVD_CGC_CTRL__LBSI_MODE_MASK 818 | UVD_CGC_CTRL__LRBBM_MODE_MASK 819 | UVD_CGC_CTRL__WCB_MODE_MASK 820 | UVD_CGC_CTRL__VCPU_MODE_MASK); 821 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 822 823 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 824 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 825 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 826 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 827 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 828 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 829 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 830 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 831 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 832 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 833 } 834 835 /** 836 * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode 837 * 838 * @vinst: VCN instance 839 * @indirect: indirectly write sram 840 * 841 * Start VCN block with dpg mode 842 */ 843 static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_vcn_inst *vinst, 844 bool indirect) 845 { 846 struct amdgpu_device *adev = vinst->adev; 847 int inst_idx = vinst->inst; 848 struct amdgpu_vcn4_fw_shared *fw_shared = 849 adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 850 struct amdgpu_ring *ring; 851 int vcn_inst, ret; 852 uint32_t tmp; 853 854 vcn_inst = GET_INST(VCN, inst_idx); 855 /* disable register anti-hang mechanism */ 856 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, 857 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 858 /* enable dynamic power gating mode */ 859 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); 860 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; 861 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; 862 WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); 863 864 if (indirect) { 865 DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", 866 inst_idx, adev->vcn.inst[inst_idx].aid_id); 867 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = 868 (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; 869 /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ 870 WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, 871 adev->vcn.inst[inst_idx].aid_id, 0, true); 872 } 873 874 /* enable clock gating */ 875 vcn_v4_0_3_disable_clock_gating_dpg_mode(vinst, 0, indirect); 876 877 /* enable VCPU clock */ 878 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 879 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 880 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; 881 882 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 883 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 884 885 /* disable master interrupt */ 886 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 887 VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); 888 889 /* setup regUVD_LMI_CTRL */ 890 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 891 UVD_LMI_CTRL__REQ_MODE_MASK | 892 UVD_LMI_CTRL__CRC_RESET_MASK | 893 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 894 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 895 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | 896 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | 897 0x00100000L); 898 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 899 VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); 900 901 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 902 VCN, 0, regUVD_MPC_CNTL), 903 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); 904 905 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 906 VCN, 0, regUVD_MPC_SET_MUXA0), 907 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 908 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 909 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 910 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); 911 912 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 913 VCN, 0, regUVD_MPC_SET_MUXB0), 914 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 915 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 916 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 917 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); 918 919 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 920 VCN, 0, regUVD_MPC_SET_MUX), 921 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 922 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 923 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); 924 925 vcn_v4_0_3_mc_resume_dpg_mode(vinst, indirect); 926 927 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 928 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 929 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 930 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 931 932 /* enable LMI MC and UMC channels */ 933 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; 934 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 935 VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); 936 937 vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); 938 939 /* enable master interrupt */ 940 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 941 VCN, 0, regUVD_MASTINT_EN), 942 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); 943 944 if (indirect) { 945 ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); 946 if (ret) { 947 dev_err(adev->dev, "vcn sram load failed %d\n", ret); 948 return ret; 949 } 950 } 951 952 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 953 954 /* program the RB_BASE for ring buffer */ 955 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 956 lower_32_bits(ring->gpu_addr)); 957 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 958 upper_32_bits(ring->gpu_addr)); 959 960 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 961 ring->ring_size / sizeof(uint32_t)); 962 963 /* resetting ring, fw should not check RB ring */ 964 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 965 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 966 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 967 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; 968 969 /* Initialize the ring buffer's read and write pointers */ 970 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 971 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 972 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 973 974 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 975 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 976 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 977 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); 978 979 /*resetting done, fw can check RB ring */ 980 fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); 981 982 /* Keeping one read-back to ensure all register writes are done, 983 * otherwise it may introduce race conditions. 984 */ 985 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 986 987 return 0; 988 } 989 990 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) 991 { 992 int i, vcn_inst; 993 struct amdgpu_ring *ring_enc; 994 uint64_t cache_addr; 995 uint64_t rb_enc_addr; 996 uint64_t ctx_addr; 997 uint32_t param, resp, expected; 998 uint32_t offset, cache_size; 999 uint32_t tmp, timeout; 1000 1001 struct amdgpu_mm_table *table = &adev->virt.mm_table; 1002 uint32_t *table_loc; 1003 uint32_t table_size; 1004 uint32_t size, size_dw; 1005 uint32_t init_status; 1006 uint32_t enabled_vcn; 1007 1008 struct mmsch_v4_0_cmd_direct_write 1009 direct_wt = { {0} }; 1010 struct mmsch_v4_0_cmd_direct_read_modify_write 1011 direct_rd_mod_wt = { {0} }; 1012 struct mmsch_v4_0_cmd_end end = { {0} }; 1013 struct mmsch_v4_0_3_init_header header; 1014 1015 struct amdgpu_vcn4_fw_shared *fw_shared; 1016 struct amdgpu_fw_shared_rb_setup *rb_setup; 1017 1018 direct_wt.cmd_header.command_type = 1019 MMSCH_COMMAND__DIRECT_REG_WRITE; 1020 direct_rd_mod_wt.cmd_header.command_type = 1021 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; 1022 end.cmd_header.command_type = MMSCH_COMMAND__END; 1023 1024 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1025 vcn_inst = GET_INST(VCN, i); 1026 1027 vcn_v4_0_3_fw_shared_init(adev, vcn_inst); 1028 1029 memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); 1030 header.version = MMSCH_VERSION; 1031 header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; 1032 1033 table_loc = (uint32_t *)table->cpu_addr; 1034 table_loc += header.total_size; 1035 1036 table_size = 0; 1037 1038 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), 1039 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); 1040 1041 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4); 1042 1043 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1044 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1045 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1046 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); 1047 1048 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1049 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1050 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); 1051 1052 offset = 0; 1053 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1054 regUVD_VCPU_CACHE_OFFSET0), 0); 1055 } else { 1056 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1057 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1058 lower_32_bits(adev->vcn.inst[i].gpu_addr)); 1059 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1060 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1061 upper_32_bits(adev->vcn.inst[i].gpu_addr)); 1062 offset = cache_size; 1063 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1064 regUVD_VCPU_CACHE_OFFSET0), 1065 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 1066 } 1067 1068 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1069 regUVD_VCPU_CACHE_SIZE0), 1070 cache_size); 1071 1072 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; 1073 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1074 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1075 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1076 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1077 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1078 regUVD_VCPU_CACHE_OFFSET1), 0); 1079 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1080 regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); 1081 1082 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + 1083 AMDGPU_VCN_STACK_SIZE; 1084 1085 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1086 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1087 1088 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1089 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1090 1091 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1092 regUVD_VCPU_CACHE_OFFSET2), 0); 1093 1094 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1095 regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); 1096 1097 fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; 1098 rb_setup = &fw_shared->rb_setup; 1099 1100 ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; 1101 ring_enc->wptr = 0; 1102 rb_enc_addr = ring_enc->gpu_addr; 1103 1104 rb_setup->is_rb_enabled_flags |= RB_ENABLED; 1105 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); 1106 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); 1107 rb_setup->rb_size = ring_enc->ring_size / 4; 1108 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); 1109 1110 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1111 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 1112 lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1113 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1114 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 1115 upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1116 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1117 regUVD_VCPU_NONCACHE_SIZE0), 1118 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 1119 MMSCH_V4_0_INSERT_END(); 1120 1121 header.vcn0.init_status = 0; 1122 header.vcn0.table_offset = header.total_size; 1123 header.vcn0.table_size = table_size; 1124 header.total_size += table_size; 1125 1126 /* Send init table to mmsch */ 1127 size = sizeof(struct mmsch_v4_0_3_init_header); 1128 table_loc = (uint32_t *)table->cpu_addr; 1129 memcpy((void *)table_loc, &header, size); 1130 1131 ctx_addr = table->gpu_addr; 1132 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); 1133 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); 1134 1135 tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); 1136 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; 1137 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); 1138 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); 1139 1140 size = header.total_size; 1141 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); 1142 1143 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); 1144 1145 param = 0x00000001; 1146 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); 1147 tmp = 0; 1148 timeout = 1000; 1149 resp = 0; 1150 expected = MMSCH_VF_MAILBOX_RESP__OK; 1151 while (resp != expected) { 1152 resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); 1153 if (resp != 0) 1154 break; 1155 1156 udelay(10); 1157 tmp = tmp + 10; 1158 if (tmp >= timeout) { 1159 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ 1160 " waiting for regMMSCH_VF_MAILBOX_RESP "\ 1161 "(expected=0x%08x, readback=0x%08x)\n", 1162 tmp, expected, resp); 1163 return -EBUSY; 1164 } 1165 } 1166 1167 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; 1168 init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; 1169 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE 1170 && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { 1171 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ 1172 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); 1173 } 1174 } 1175 1176 return 0; 1177 } 1178 1179 /** 1180 * vcn_v4_0_3_start - VCN start 1181 * 1182 * @vinst: VCN instance 1183 * 1184 * Start VCN block 1185 */ 1186 static int vcn_v4_0_3_start(struct amdgpu_vcn_inst *vinst) 1187 { 1188 struct amdgpu_device *adev = vinst->adev; 1189 int i = vinst->inst; 1190 struct amdgpu_vcn4_fw_shared *fw_shared; 1191 struct amdgpu_ring *ring; 1192 int j, k, r, vcn_inst; 1193 uint32_t tmp; 1194 1195 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 1196 return vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram); 1197 1198 vcn_inst = GET_INST(VCN, i); 1199 /* set VCN status busy */ 1200 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | 1201 UVD_STATUS__UVD_BUSY; 1202 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); 1203 1204 /* SW clock gating */ 1205 vcn_v4_0_3_disable_clock_gating(vinst); 1206 1207 /* enable VCPU clock */ 1208 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1209 UVD_VCPU_CNTL__CLK_EN_MASK, 1210 ~UVD_VCPU_CNTL__CLK_EN_MASK); 1211 1212 /* disable master interrupt */ 1213 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, 1214 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1215 1216 /* enable LMI MC and UMC channels */ 1217 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, 1218 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); 1219 1220 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1221 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1222 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1223 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1224 1225 /* setup regUVD_LMI_CTRL */ 1226 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); 1227 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, 1228 tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 1229 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 1230 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 1231 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); 1232 1233 /* setup regUVD_MPC_CNTL */ 1234 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); 1235 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; 1236 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; 1237 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); 1238 1239 /* setup UVD_MPC_SET_MUXA0 */ 1240 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, 1241 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 1242 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 1243 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 1244 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); 1245 1246 /* setup UVD_MPC_SET_MUXB0 */ 1247 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, 1248 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 1249 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 1250 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 1251 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); 1252 1253 /* setup UVD_MPC_SET_MUX */ 1254 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, 1255 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 1256 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 1257 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); 1258 1259 vcn_v4_0_3_mc_resume(vinst); 1260 1261 /* VCN global tiling registers */ 1262 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, 1263 adev->gfx.config.gb_addr_config); 1264 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, 1265 adev->gfx.config.gb_addr_config); 1266 1267 /* unblock VCPU register access */ 1268 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, 1269 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1270 1271 /* release VCPU reset to boot */ 1272 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1273 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1274 1275 for (j = 0; j < 10; ++j) { 1276 uint32_t status; 1277 1278 for (k = 0; k < 100; ++k) { 1279 status = RREG32_SOC15(VCN, vcn_inst, 1280 regUVD_STATUS); 1281 if (status & 2) 1282 break; 1283 mdelay(10); 1284 } 1285 r = 0; 1286 if (status & 2) 1287 break; 1288 1289 DRM_DEV_ERROR(adev->dev, 1290 "VCN decode not responding, trying to reset the VCPU!!!\n"); 1291 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1292 regUVD_VCPU_CNTL), 1293 UVD_VCPU_CNTL__BLK_RST_MASK, 1294 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1295 mdelay(10); 1296 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1297 regUVD_VCPU_CNTL), 1298 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); 1299 1300 mdelay(10); 1301 r = -1; 1302 } 1303 1304 if (r) { 1305 DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); 1306 return r; 1307 } 1308 1309 /* enable master interrupt */ 1310 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 1311 UVD_MASTINT_EN__VCPU_EN_MASK, 1312 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1313 1314 /* clear the busy bit of VCN_STATUS */ 1315 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, 1316 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); 1317 1318 ring = &adev->vcn.inst[i].ring_enc[0]; 1319 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1320 1321 /* program the RB_BASE for ring buffer */ 1322 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 1323 lower_32_bits(ring->gpu_addr)); 1324 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 1325 upper_32_bits(ring->gpu_addr)); 1326 1327 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 1328 ring->ring_size / sizeof(uint32_t)); 1329 1330 /* resetting ring, fw should not check RB ring */ 1331 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1332 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 1333 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1334 1335 /* Initialize the ring buffer's read and write pointers */ 1336 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 1337 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 1338 1339 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1340 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 1341 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1342 1343 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1344 fw_shared->sq.queue_mode &= 1345 cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); 1346 1347 return 0; 1348 } 1349 1350 /** 1351 * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode 1352 * 1353 * @vinst: VCN instance 1354 * 1355 * Stop VCN block with dpg mode 1356 */ 1357 static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_vcn_inst *vinst) 1358 { 1359 struct amdgpu_device *adev = vinst->adev; 1360 int inst_idx = vinst->inst; 1361 uint32_t tmp; 1362 int vcn_inst; 1363 1364 vcn_inst = GET_INST(VCN, inst_idx); 1365 1366 /* Wait for power status to be 1 */ 1367 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1368 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1369 1370 /* wait for read ptr to be equal to write ptr */ 1371 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1372 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); 1373 1374 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1375 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1376 1377 /* disable dynamic power gating mode */ 1378 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, 1379 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); 1380 1381 /* Keeping one read-back to ensure all register writes are done, 1382 * otherwise it may introduce race conditions. 1383 */ 1384 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1385 1386 return 0; 1387 } 1388 1389 /** 1390 * vcn_v4_0_3_stop - VCN stop 1391 * 1392 * @vinst: VCN instance 1393 * 1394 * Stop VCN block 1395 */ 1396 static int vcn_v4_0_3_stop(struct amdgpu_vcn_inst *vinst) 1397 { 1398 struct amdgpu_device *adev = vinst->adev; 1399 int i = vinst->inst; 1400 struct amdgpu_vcn4_fw_shared *fw_shared; 1401 int r = 0, vcn_inst; 1402 uint32_t tmp; 1403 1404 vcn_inst = GET_INST(VCN, i); 1405 1406 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1407 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; 1408 1409 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { 1410 vcn_v4_0_3_stop_dpg_mode(vinst); 1411 goto Done; 1412 } 1413 1414 /* wait for vcn idle */ 1415 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, 1416 UVD_STATUS__IDLE, 0x7); 1417 if (r) 1418 goto Done; 1419 1420 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | 1421 UVD_LMI_STATUS__READ_CLEAN_MASK | 1422 UVD_LMI_STATUS__WRITE_CLEAN_MASK | 1423 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; 1424 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1425 tmp); 1426 if (r) 1427 goto Done; 1428 1429 /* stall UMC channel */ 1430 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); 1431 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; 1432 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); 1433 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | 1434 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; 1435 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1436 tmp); 1437 if (r) 1438 goto Done; 1439 1440 /* Unblock VCPU Register access */ 1441 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 1442 UVD_RB_ARB_CTRL__VCPU_DIS_MASK, 1443 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1444 1445 /* release VCPU reset to boot */ 1446 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1447 UVD_VCPU_CNTL__BLK_RST_MASK, 1448 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1449 1450 /* disable VCPU clock */ 1451 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1452 ~(UVD_VCPU_CNTL__CLK_EN_MASK)); 1453 1454 /* reset LMI UMC/LMI/VCPU */ 1455 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1456 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1457 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1458 1459 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1460 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1461 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1462 1463 /* clear VCN status */ 1464 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); 1465 1466 /* apply HW clock gating */ 1467 vcn_v4_0_3_enable_clock_gating(vinst); 1468 1469 /* Keeping one read-back to ensure all register writes are done, 1470 * otherwise it may introduce race conditions. 1471 */ 1472 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1473 1474 Done: 1475 return 0; 1476 } 1477 1478 /** 1479 * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode 1480 * 1481 * @vinst: VCN instance 1482 * @new_state: pause state 1483 * 1484 * Pause dpg mode for VCN block 1485 */ 1486 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 1487 struct dpg_pause_state *new_state) 1488 { 1489 1490 return 0; 1491 } 1492 1493 /** 1494 * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer 1495 * 1496 * @ring: amdgpu_ring pointer 1497 * 1498 * Returns the current hardware unified read pointer 1499 */ 1500 static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) 1501 { 1502 struct amdgpu_device *adev = ring->adev; 1503 1504 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1505 DRM_ERROR("wrong ring id is identified in %s", __func__); 1506 1507 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); 1508 } 1509 1510 /** 1511 * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer 1512 * 1513 * @ring: amdgpu_ring pointer 1514 * 1515 * Returns the current hardware unified write pointer 1516 */ 1517 static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) 1518 { 1519 struct amdgpu_device *adev = ring->adev; 1520 1521 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1522 DRM_ERROR("wrong ring id is identified in %s", __func__); 1523 1524 if (ring->use_doorbell) 1525 return *ring->wptr_cpu_addr; 1526 else 1527 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), 1528 regUVD_RB_WPTR); 1529 } 1530 1531 void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 1532 uint32_t val, uint32_t mask) 1533 { 1534 /* Use normalized offsets when required */ 1535 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1536 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1537 1538 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); 1539 amdgpu_ring_write(ring, reg << 2); 1540 amdgpu_ring_write(ring, mask); 1541 amdgpu_ring_write(ring, val); 1542 } 1543 1544 void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 1545 uint32_t val) 1546 { 1547 /* Use normalized offsets when required */ 1548 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1549 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1550 1551 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); 1552 amdgpu_ring_write(ring, reg << 2); 1553 amdgpu_ring_write(ring, val); 1554 } 1555 1556 void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, 1557 unsigned int vmid, uint64_t pd_addr) 1558 { 1559 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; 1560 1561 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 1562 1563 /* wait for reg writes */ 1564 vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + 1565 vmid * hub->ctx_addr_distance, 1566 lower_32_bits(pd_addr), 0xffffffff); 1567 } 1568 1569 void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) 1570 { 1571 /* VCN engine access for HDP flush doesn't work when RRMT is enabled. 1572 * This is a workaround to avoid any HDP flush through VCN ring. 1573 */ 1574 } 1575 1576 /** 1577 * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer 1578 * 1579 * @ring: amdgpu_ring pointer 1580 * 1581 * Commits the enc write pointer to the hardware 1582 */ 1583 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) 1584 { 1585 struct amdgpu_device *adev = ring->adev; 1586 1587 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1588 DRM_ERROR("wrong ring id is identified in %s", __func__); 1589 1590 if (ring->use_doorbell) { 1591 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 1592 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1593 } else { 1594 WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, 1595 lower_32_bits(ring->wptr)); 1596 } 1597 } 1598 1599 static int vcn_v4_0_3_ring_reset(struct amdgpu_ring *ring, 1600 unsigned int vmid, 1601 struct amdgpu_fence *timedout_fence) 1602 { 1603 int r = 0; 1604 int vcn_inst; 1605 struct amdgpu_device *adev = ring->adev; 1606 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me]; 1607 1608 amdgpu_ring_reset_helper_begin(ring, timedout_fence); 1609 1610 vcn_inst = GET_INST(VCN, ring->me); 1611 r = amdgpu_dpm_reset_vcn(adev, 1 << vcn_inst); 1612 1613 if (r) { 1614 DRM_DEV_ERROR(adev->dev, "VCN reset fail : %d\n", r); 1615 return r; 1616 } 1617 1618 /* This flag is not set for VF, assumed to be disabled always */ 1619 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x100) 1620 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 1621 vcn_v4_0_3_hw_init_inst(vinst); 1622 vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[ring->me].indirect_sram); 1623 1624 return amdgpu_ring_reset_helper_end(ring, timedout_fence); 1625 } 1626 1627 static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { 1628 .type = AMDGPU_RING_TYPE_VCN_ENC, 1629 .align_mask = 0x3f, 1630 .nop = VCN_ENC_CMD_NO_OP, 1631 .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, 1632 .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, 1633 .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, 1634 .emit_frame_size = 1635 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 1636 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + 1637 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ 1638 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ 1639 1, /* vcn_v2_0_enc_ring_insert_end */ 1640 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ 1641 .emit_ib = vcn_v2_0_enc_ring_emit_ib, 1642 .emit_fence = vcn_v2_0_enc_ring_emit_fence, 1643 .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, 1644 .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, 1645 .test_ring = amdgpu_vcn_enc_ring_test_ring, 1646 .test_ib = amdgpu_vcn_unified_ring_test_ib, 1647 .insert_nop = amdgpu_ring_insert_nop, 1648 .insert_end = vcn_v2_0_enc_ring_insert_end, 1649 .pad_ib = amdgpu_ring_generic_pad_ib, 1650 .begin_use = amdgpu_vcn_ring_begin_use, 1651 .end_use = amdgpu_vcn_ring_end_use, 1652 .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, 1653 .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, 1654 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1655 .reset = vcn_v4_0_3_ring_reset, 1656 }; 1657 1658 /** 1659 * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions 1660 * 1661 * @adev: amdgpu_device pointer 1662 * 1663 * Set unified ring functions 1664 */ 1665 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) 1666 { 1667 int i, vcn_inst; 1668 1669 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1670 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; 1671 adev->vcn.inst[i].ring_enc[0].me = i; 1672 vcn_inst = GET_INST(VCN, i); 1673 adev->vcn.inst[i].aid_id = 1674 vcn_inst / adev->vcn.num_inst_per_aid; 1675 } 1676 } 1677 1678 /** 1679 * vcn_v4_0_3_is_idle - check VCN block is idle 1680 * 1681 * @ip_block: Pointer to the amdgpu_ip_block structure 1682 * 1683 * Check whether VCN block is idle 1684 */ 1685 static bool vcn_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block) 1686 { 1687 struct amdgpu_device *adev = ip_block->adev; 1688 int i, ret = 1; 1689 1690 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1691 ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == 1692 UVD_STATUS__IDLE); 1693 } 1694 1695 return ret; 1696 } 1697 1698 /** 1699 * vcn_v4_0_3_wait_for_idle - wait for VCN block idle 1700 * 1701 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 1702 * 1703 * Wait for VCN block idle 1704 */ 1705 static int vcn_v4_0_3_wait_for_idle(struct amdgpu_ip_block *ip_block) 1706 { 1707 struct amdgpu_device *adev = ip_block->adev; 1708 int i, ret = 0; 1709 1710 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1711 ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, 1712 UVD_STATUS__IDLE, UVD_STATUS__IDLE); 1713 if (ret) 1714 return ret; 1715 } 1716 1717 return ret; 1718 } 1719 1720 /* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state 1721 * 1722 * @ip_block: amdgpu_ip_block pointer 1723 * @state: clock gating state 1724 * 1725 * Set VCN block clockgating state 1726 */ 1727 static int vcn_v4_0_3_set_clockgating_state(struct amdgpu_ip_block *ip_block, 1728 enum amd_clockgating_state state) 1729 { 1730 struct amdgpu_device *adev = ip_block->adev; 1731 bool enable = state == AMD_CG_STATE_GATE; 1732 int i; 1733 1734 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1735 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 1736 1737 if (enable) { 1738 if (RREG32_SOC15(VCN, GET_INST(VCN, i), 1739 regUVD_STATUS) != UVD_STATUS__IDLE) 1740 return -EBUSY; 1741 vcn_v4_0_3_enable_clock_gating(vinst); 1742 } else { 1743 vcn_v4_0_3_disable_clock_gating(vinst); 1744 } 1745 } 1746 return 0; 1747 } 1748 1749 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 1750 enum amd_powergating_state state) 1751 { 1752 struct amdgpu_device *adev = vinst->adev; 1753 int ret = 0; 1754 1755 /* for SRIOV, guest should not control VCN Power-gating 1756 * MMSCH FW should control Power-gating and clock-gating 1757 * guest should avoid touching CGC and PG 1758 */ 1759 if (amdgpu_sriov_vf(adev)) { 1760 vinst->cur_state = AMD_PG_STATE_UNGATE; 1761 return 0; 1762 } 1763 1764 if (state == vinst->cur_state) 1765 return 0; 1766 1767 if (state == AMD_PG_STATE_GATE) 1768 ret = vcn_v4_0_3_stop(vinst); 1769 else 1770 ret = vcn_v4_0_3_start(vinst); 1771 1772 if (!ret) 1773 vinst->cur_state = state; 1774 1775 return ret; 1776 } 1777 1778 /** 1779 * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state 1780 * 1781 * @adev: amdgpu_device pointer 1782 * @source: interrupt sources 1783 * @type: interrupt types 1784 * @state: interrupt states 1785 * 1786 * Set VCN block interrupt state 1787 */ 1788 static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, 1789 struct amdgpu_irq_src *source, 1790 unsigned int type, 1791 enum amdgpu_interrupt_state state) 1792 { 1793 return 0; 1794 } 1795 1796 /** 1797 * vcn_v4_0_3_process_interrupt - process VCN block interrupt 1798 * 1799 * @adev: amdgpu_device pointer 1800 * @source: interrupt sources 1801 * @entry: interrupt entry from clients and sources 1802 * 1803 * Process VCN block interrupt 1804 */ 1805 static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, 1806 struct amdgpu_irq_src *source, 1807 struct amdgpu_iv_entry *entry) 1808 { 1809 uint32_t i, inst; 1810 1811 i = node_id_to_phys_map[entry->node_id]; 1812 1813 DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); 1814 1815 for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) 1816 if (adev->vcn.inst[inst].aid_id == i) 1817 break; 1818 1819 if (inst >= adev->vcn.num_vcn_inst) { 1820 dev_WARN_ONCE(adev->dev, 1, 1821 "Interrupt received for unknown VCN instance %d", 1822 entry->node_id); 1823 return 0; 1824 } 1825 1826 switch (entry->src_id) { 1827 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: 1828 amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); 1829 break; 1830 default: 1831 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", 1832 entry->src_id, entry->src_data[0]); 1833 break; 1834 } 1835 1836 return 0; 1837 } 1838 1839 static int vcn_v4_0_3_set_ras_interrupt_state(struct amdgpu_device *adev, 1840 struct amdgpu_irq_src *source, 1841 unsigned int type, 1842 enum amdgpu_interrupt_state state) 1843 { 1844 return 0; 1845 } 1846 1847 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { 1848 .set = vcn_v4_0_3_set_interrupt_state, 1849 .process = vcn_v4_0_3_process_interrupt, 1850 }; 1851 1852 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_ras_irq_funcs = { 1853 .set = vcn_v4_0_3_set_ras_interrupt_state, 1854 .process = amdgpu_vcn_process_poison_irq, 1855 }; 1856 1857 /** 1858 * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions 1859 * 1860 * @adev: amdgpu_device pointer 1861 * 1862 * Set VCN block interrupt irq functions 1863 */ 1864 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) 1865 { 1866 int i; 1867 1868 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1869 adev->vcn.inst->irq.num_types++; 1870 } 1871 adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; 1872 1873 adev->vcn.inst->ras_poison_irq.num_types = 1; 1874 adev->vcn.inst->ras_poison_irq.funcs = &vcn_v4_0_3_ras_irq_funcs; 1875 } 1876 1877 static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { 1878 .name = "vcn_v4_0_3", 1879 .early_init = vcn_v4_0_3_early_init, 1880 .late_init = vcn_v4_0_3_late_init, 1881 .sw_init = vcn_v4_0_3_sw_init, 1882 .sw_fini = vcn_v4_0_3_sw_fini, 1883 .hw_init = vcn_v4_0_3_hw_init, 1884 .hw_fini = vcn_v4_0_3_hw_fini, 1885 .suspend = vcn_v4_0_3_suspend, 1886 .resume = vcn_v4_0_3_resume, 1887 .is_idle = vcn_v4_0_3_is_idle, 1888 .wait_for_idle = vcn_v4_0_3_wait_for_idle, 1889 .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, 1890 .set_powergating_state = vcn_set_powergating_state, 1891 .dump_ip_state = amdgpu_vcn_dump_ip_state, 1892 .print_ip_state = amdgpu_vcn_print_ip_state, 1893 }; 1894 1895 const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { 1896 .type = AMD_IP_BLOCK_TYPE_VCN, 1897 .major = 4, 1898 .minor = 0, 1899 .rev = 3, 1900 .funcs = &vcn_v4_0_3_ip_funcs, 1901 }; 1902 1903 static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { 1904 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), 1905 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, 1906 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), 1907 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, 1908 }; 1909 1910 static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, 1911 uint32_t vcn_inst, 1912 void *ras_err_status) 1913 { 1914 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; 1915 1916 /* vcn v4_0_3 only support query uncorrectable errors */ 1917 amdgpu_ras_inst_query_ras_error_count(adev, 1918 vcn_v4_0_3_ue_reg_list, 1919 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 1920 NULL, 0, GET_INST(VCN, vcn_inst), 1921 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 1922 &err_data->ue_count); 1923 } 1924 1925 static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, 1926 void *ras_err_status) 1927 { 1928 uint32_t i; 1929 1930 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 1931 dev_warn(adev->dev, "VCN RAS is not supported\n"); 1932 return; 1933 } 1934 1935 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 1936 vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); 1937 } 1938 1939 static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, 1940 uint32_t vcn_inst) 1941 { 1942 amdgpu_ras_inst_reset_ras_error_count(adev, 1943 vcn_v4_0_3_ue_reg_list, 1944 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 1945 GET_INST(VCN, vcn_inst)); 1946 } 1947 1948 static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) 1949 { 1950 uint32_t i; 1951 1952 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 1953 dev_warn(adev->dev, "VCN RAS is not supported\n"); 1954 return; 1955 } 1956 1957 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 1958 vcn_v4_0_3_inst_reset_ras_error_count(adev, i); 1959 } 1960 1961 static uint32_t vcn_v4_0_3_query_poison_by_instance(struct amdgpu_device *adev, 1962 uint32_t instance, uint32_t sub_block) 1963 { 1964 uint32_t poison_stat = 0, reg_value = 0; 1965 1966 switch (sub_block) { 1967 case AMDGPU_VCN_V4_0_3_VCPU_VCODEC: 1968 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS); 1969 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF); 1970 break; 1971 default: 1972 break; 1973 } 1974 1975 if (poison_stat) 1976 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n", 1977 instance, sub_block); 1978 1979 return poison_stat; 1980 } 1981 1982 static bool vcn_v4_0_3_query_poison_status(struct amdgpu_device *adev) 1983 { 1984 uint32_t inst, sub; 1985 uint32_t poison_stat = 0; 1986 1987 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++) 1988 for (sub = 0; sub < AMDGPU_VCN_V4_0_3_MAX_SUB_BLOCK; sub++) 1989 poison_stat += 1990 vcn_v4_0_3_query_poison_by_instance(adev, inst, sub); 1991 1992 return !!poison_stat; 1993 } 1994 1995 static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { 1996 .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, 1997 .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, 1998 .query_poison_status = vcn_v4_0_3_query_poison_status, 1999 }; 2000 2001 static int vcn_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, 2002 enum aca_smu_type type, void *data) 2003 { 2004 struct aca_bank_info info; 2005 u64 misc0; 2006 int ret; 2007 2008 ret = aca_bank_info_decode(bank, &info); 2009 if (ret) 2010 return ret; 2011 2012 misc0 = bank->regs[ACA_REG_IDX_MISC0]; 2013 switch (type) { 2014 case ACA_SMU_TYPE_UE: 2015 bank->aca_err_type = ACA_ERROR_TYPE_UE; 2016 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, 2017 1ULL); 2018 break; 2019 case ACA_SMU_TYPE_CE: 2020 bank->aca_err_type = ACA_ERROR_TYPE_CE; 2021 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 2022 ACA_REG__MISC0__ERRCNT(misc0)); 2023 break; 2024 default: 2025 return -EINVAL; 2026 } 2027 2028 return ret; 2029 } 2030 2031 /* reference to smu driver if header file */ 2032 static int vcn_v4_0_3_err_codes[] = { 2033 14, 15, /* VCN */ 2034 }; 2035 2036 static bool vcn_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, 2037 enum aca_smu_type type, void *data) 2038 { 2039 u32 instlo; 2040 2041 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); 2042 instlo &= GENMASK(31, 1); 2043 2044 if (instlo != mmSMNAID_AID0_MCA_SMU) 2045 return false; 2046 2047 if (aca_bank_check_error_codes(handle->adev, bank, 2048 vcn_v4_0_3_err_codes, 2049 ARRAY_SIZE(vcn_v4_0_3_err_codes))) 2050 return false; 2051 2052 return true; 2053 } 2054 2055 static const struct aca_bank_ops vcn_v4_0_3_aca_bank_ops = { 2056 .aca_bank_parser = vcn_v4_0_3_aca_bank_parser, 2057 .aca_bank_is_valid = vcn_v4_0_3_aca_bank_is_valid, 2058 }; 2059 2060 static const struct aca_info vcn_v4_0_3_aca_info = { 2061 .hwip = ACA_HWIP_TYPE_SMU, 2062 .mask = ACA_ERROR_UE_MASK, 2063 .bank_ops = &vcn_v4_0_3_aca_bank_ops, 2064 }; 2065 2066 static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 2067 { 2068 int r; 2069 2070 r = amdgpu_ras_block_late_init(adev, ras_block); 2071 if (r) 2072 return r; 2073 2074 if (amdgpu_ras_is_supported(adev, ras_block->block) && 2075 adev->vcn.inst->ras_poison_irq.funcs) { 2076 r = amdgpu_irq_get(adev, &adev->vcn.inst->ras_poison_irq, 0); 2077 if (r) 2078 goto late_fini; 2079 } 2080 2081 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN, 2082 &vcn_v4_0_3_aca_info, NULL); 2083 if (r) 2084 goto late_fini; 2085 2086 return 0; 2087 2088 late_fini: 2089 amdgpu_ras_block_late_fini(adev, ras_block); 2090 2091 return r; 2092 } 2093 2094 static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { 2095 .ras_block = { 2096 .hw_ops = &vcn_v4_0_3_ras_hw_ops, 2097 .ras_late_init = vcn_v4_0_3_ras_late_init, 2098 }, 2099 }; 2100 2101 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) 2102 { 2103 adev->vcn.ras = &vcn_v4_0_3_ras; 2104 } 2105 2106 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 2107 int inst_idx, bool indirect) 2108 { 2109 uint32_t tmp; 2110 2111 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) 2112 return; 2113 2114 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | 2115 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | 2116 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | 2117 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; 2118 WREG32_SOC15_DPG_MODE(inst_idx, 2119 SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), 2120 tmp, 0, indirect); 2121 2122 tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; 2123 WREG32_SOC15_DPG_MODE(inst_idx, 2124 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), 2125 tmp, 0, indirect); 2126 2127 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; 2128 WREG32_SOC15_DPG_MODE(inst_idx, 2129 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), 2130 tmp, 0, indirect); 2131 } 2132