1 /* 2 * Copyright 2022 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <drm/drm_drv.h> 26 27 #include "amdgpu.h" 28 #include "amdgpu_vcn.h" 29 #include "amdgpu_pm.h" 30 #include "soc15.h" 31 #include "soc15d.h" 32 #include "soc15_hw_ip.h" 33 #include "vcn_v2_0.h" 34 #include "vcn_v4_0_3.h" 35 #include "mmsch_v4_0_3.h" 36 37 #include "vcn/vcn_4_0_3_offset.h" 38 #include "vcn/vcn_4_0_3_sh_mask.h" 39 #include "ivsrcid/vcn/irqsrcs_vcn_4_0.h" 40 41 #define mmUVD_DPG_LMA_CTL regUVD_DPG_LMA_CTL 42 #define mmUVD_DPG_LMA_CTL_BASE_IDX regUVD_DPG_LMA_CTL_BASE_IDX 43 #define mmUVD_DPG_LMA_DATA regUVD_DPG_LMA_DATA 44 #define mmUVD_DPG_LMA_DATA_BASE_IDX regUVD_DPG_LMA_DATA_BASE_IDX 45 46 #define VCN_VID_SOC_ADDRESS_2_0 0x1fb00 47 #define VCN1_VID_SOC_ADDRESS_3_0 0x48300 48 #define VCN1_AON_SOC_ADDRESS_3_0 0x48000 49 50 static const struct amdgpu_hwip_reg_entry vcn_reg_list_4_0_3[] = { 51 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_POWER_STATUS), 52 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_STATUS), 53 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID), 54 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_CONTEXT_ID2), 55 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA0), 56 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_DATA1), 57 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_GPCOM_VCPU_CMD), 58 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI), 59 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO), 60 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI2), 61 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO2), 62 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI3), 63 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO3), 64 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_HI4), 65 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_BASE_LO4), 66 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR), 67 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR), 68 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR2), 69 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR2), 70 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR3), 71 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR3), 72 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_RPTR4), 73 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_WPTR4), 74 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE), 75 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE2), 76 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE3), 77 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_RB_SIZE4), 78 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_CONFIG), 79 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_PGFSM_STATUS), 80 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_CTL), 81 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_DATA), 82 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_LMA_MASK), 83 SOC15_REG_ENTRY_STR(VCN, 0, regUVD_DPG_PAUSE) 84 }; 85 86 #define NORMALIZE_VCN_REG_OFFSET(offset) \ 87 (offset & 0x1FFFF) 88 89 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev); 90 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev); 91 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev); 92 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 93 enum amd_powergating_state state); 94 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 95 struct dpg_pause_state *new_state); 96 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring); 97 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev); 98 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 99 int inst_idx, bool indirect); 100 101 static inline bool vcn_v4_0_3_normalizn_reqd(struct amdgpu_device *adev) 102 { 103 return (adev->vcn.caps & AMDGPU_VCN_CAPS(RRMT_ENABLED)) == 0; 104 } 105 106 /** 107 * vcn_v4_0_3_early_init - set function pointers 108 * 109 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 110 * 111 * Set ring and irq function pointers 112 */ 113 static int vcn_v4_0_3_early_init(struct amdgpu_ip_block *ip_block) 114 { 115 struct amdgpu_device *adev = ip_block->adev; 116 int i, r; 117 118 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) 119 /* re-use enc ring as unified ring */ 120 adev->vcn.inst[i].num_enc_rings = 1; 121 122 vcn_v4_0_3_set_unified_ring_funcs(adev); 123 vcn_v4_0_3_set_irq_funcs(adev); 124 vcn_v4_0_3_set_ras_funcs(adev); 125 126 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 127 adev->vcn.inst[i].set_pg_state = vcn_v4_0_3_set_pg_state; 128 129 r = amdgpu_vcn_early_init(adev, i); 130 if (r) 131 return r; 132 } 133 134 return 0; 135 } 136 137 static int vcn_v4_0_3_late_init(struct amdgpu_ip_block *ip_block) 138 { 139 struct amdgpu_device *adev = ip_block->adev; 140 141 adev->vcn.supported_reset = 142 amdgpu_get_soft_full_reset_mask(&adev->vcn.inst[0].ring_enc[0]); 143 144 if (amdgpu_dpm_reset_vcn_is_supported(adev) && !amdgpu_sriov_vf(adev)) 145 adev->vcn.supported_reset |= AMDGPU_RESET_TYPE_PER_QUEUE; 146 147 return 0; 148 } 149 150 static int vcn_v4_0_3_fw_shared_init(struct amdgpu_device *adev, int inst_idx) 151 { 152 struct amdgpu_vcn4_fw_shared *fw_shared; 153 154 fw_shared = adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 155 fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE); 156 fw_shared->sq.is_enabled = 1; 157 158 if (amdgpu_vcnfw_log) 159 amdgpu_vcn_fwlog_init(&adev->vcn.inst[inst_idx]); 160 161 return 0; 162 } 163 164 /** 165 * vcn_v4_0_3_sw_init - sw init for VCN block 166 * 167 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 168 * 169 * Load firmware and sw initialization 170 */ 171 static int vcn_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block) 172 { 173 struct amdgpu_device *adev = ip_block->adev; 174 struct amdgpu_ring *ring; 175 int i, r, vcn_inst; 176 177 /* VCN DEC TRAP */ 178 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 179 VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE, &adev->vcn.inst->irq); 180 if (r) 181 return r; 182 183 /* VCN POISON TRAP */ 184 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN, 185 VCN_4_0__SRCID_UVD_POISON, &adev->vcn.inst->ras_poison_irq); 186 187 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 188 189 r = amdgpu_vcn_sw_init(adev, i); 190 if (r) 191 return r; 192 193 amdgpu_vcn_setup_ucode(adev, i); 194 195 r = amdgpu_vcn_resume(adev, i); 196 if (r) 197 return r; 198 199 vcn_inst = GET_INST(VCN, i); 200 201 ring = &adev->vcn.inst[i].ring_enc[0]; 202 ring->use_doorbell = true; 203 204 if (!amdgpu_sriov_vf(adev)) 205 ring->doorbell_index = 206 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 207 9 * vcn_inst; 208 else 209 ring->doorbell_index = 210 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 211 32 * vcn_inst; 212 213 ring->vm_hub = AMDGPU_MMHUB0(adev->vcn.inst[i].aid_id); 214 sprintf(ring->name, "vcn_unified_%d", adev->vcn.inst[i].aid_id); 215 216 /* There are no per-instance irq source IDs on 4.0.3, the IH 217 * packets use a separate field to differentiate instances. 218 */ 219 r = amdgpu_ring_init(adev, ring, 512, &adev->vcn.inst[0].irq, 0, 220 AMDGPU_RING_PRIO_DEFAULT, 221 &adev->vcn.inst[i].sched_score); 222 if (r) 223 return r; 224 225 vcn_v4_0_3_fw_shared_init(adev, i); 226 227 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 228 adev->vcn.inst[i].pause_dpg_mode = vcn_v4_0_3_pause_dpg_mode; 229 } 230 231 if (amdgpu_sriov_vf(adev)) { 232 r = amdgpu_virt_alloc_mm_table(adev); 233 if (r) 234 return r; 235 } 236 237 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 238 r = amdgpu_vcn_ras_sw_init(adev); 239 if (r) { 240 dev_err(adev->dev, "Failed to initialize vcn ras block!\n"); 241 return r; 242 } 243 } 244 245 r = amdgpu_vcn_reg_dump_init(adev, vcn_reg_list_4_0_3, ARRAY_SIZE(vcn_reg_list_4_0_3)); 246 if (r) 247 return r; 248 249 return amdgpu_vcn_sysfs_reset_mask_init(adev); 250 } 251 252 /** 253 * vcn_v4_0_3_sw_fini - sw fini for VCN block 254 * 255 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 256 * 257 * VCN suspend and free up sw allocation 258 */ 259 static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block) 260 { 261 struct amdgpu_device *adev = ip_block->adev; 262 int i, r, idx; 263 264 if (drm_dev_enter(&adev->ddev, &idx)) { 265 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 266 struct amdgpu_vcn4_fw_shared *fw_shared; 267 268 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 269 fw_shared->present_flag_0 = 0; 270 fw_shared->sq.is_enabled = cpu_to_le32(false); 271 } 272 drm_dev_exit(idx); 273 } 274 275 if (amdgpu_sriov_vf(adev)) 276 amdgpu_virt_free_mm_table(adev); 277 278 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 279 r = amdgpu_vcn_suspend(adev, i); 280 if (r) 281 return r; 282 } 283 284 amdgpu_vcn_sysfs_reset_mask_fini(adev); 285 286 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 287 amdgpu_vcn_sw_fini(adev, i); 288 289 return 0; 290 } 291 292 static int vcn_v4_0_3_hw_init_inst(struct amdgpu_vcn_inst *vinst) 293 { 294 int vcn_inst; 295 struct amdgpu_device *adev = vinst->adev; 296 struct amdgpu_ring *ring; 297 int inst_idx = vinst->inst; 298 299 vcn_inst = GET_INST(VCN, inst_idx); 300 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 301 if (ring->use_doorbell) { 302 adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell, 303 (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 9 * vcn_inst, 304 adev->vcn.inst[inst_idx].aid_id); 305 306 WREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL, 307 ring->doorbell_index << VCN_RB1_DB_CTRL__OFFSET__SHIFT | 308 VCN_RB1_DB_CTRL__EN_MASK); 309 310 /* Read DB_CTRL to flush the write DB_CTRL command. */ 311 RREG32_SOC15(VCN, vcn_inst, regVCN_RB1_DB_CTRL); 312 } 313 314 return 0; 315 } 316 317 /** 318 * vcn_v4_0_3_hw_init - start and test VCN block 319 * 320 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 321 * 322 * Initialize the hardware, boot up the VCPU and do some testing 323 */ 324 static int vcn_v4_0_3_hw_init(struct amdgpu_ip_block *ip_block) 325 { 326 struct amdgpu_device *adev = ip_block->adev; 327 struct amdgpu_ring *ring; 328 struct amdgpu_vcn_inst *vinst; 329 int i, r; 330 331 if (amdgpu_sriov_vf(adev)) { 332 r = vcn_v4_0_3_start_sriov(adev); 333 if (r) 334 return r; 335 336 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 337 ring = &adev->vcn.inst[i].ring_enc[0]; 338 ring->wptr = 0; 339 ring->wptr_old = 0; 340 vcn_v4_0_3_unified_ring_set_wptr(ring); 341 ring->sched.ready = true; 342 } 343 } else { 344 /* This flag is not set for VF, assumed to be disabled always */ 345 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 346 0x100) 347 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 348 349 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 350 struct amdgpu_vcn4_fw_shared *fw_shared; 351 352 ring = &adev->vcn.inst[i].ring_enc[0]; 353 vinst = &adev->vcn.inst[i]; 354 vcn_v4_0_3_hw_init_inst(vinst); 355 356 /* Re-init fw_shared when RAS fatal error occurred */ 357 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 358 if (!fw_shared->sq.is_enabled) 359 vcn_v4_0_3_fw_shared_init(adev, i); 360 361 r = amdgpu_ring_test_helper(ring); 362 if (r) 363 return r; 364 } 365 } 366 367 return r; 368 } 369 370 /** 371 * vcn_v4_0_3_hw_fini - stop the hardware block 372 * 373 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 374 * 375 * Stop the VCN block, mark ring as not ready any more 376 */ 377 static int vcn_v4_0_3_hw_fini(struct amdgpu_ip_block *ip_block) 378 { 379 struct amdgpu_device *adev = ip_block->adev; 380 int i; 381 382 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 383 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 384 385 cancel_delayed_work_sync(&vinst->idle_work); 386 387 if (vinst->cur_state != AMD_PG_STATE_GATE) 388 vinst->set_pg_state(vinst, AMD_PG_STATE_GATE); 389 } 390 391 if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN) && !amdgpu_sriov_vf(adev)) 392 amdgpu_irq_put(adev, &adev->vcn.inst->ras_poison_irq, 0); 393 394 return 0; 395 } 396 397 /** 398 * vcn_v4_0_3_suspend - suspend VCN block 399 * 400 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 401 * 402 * HW fini and suspend VCN block 403 */ 404 static int vcn_v4_0_3_suspend(struct amdgpu_ip_block *ip_block) 405 { 406 struct amdgpu_device *adev = ip_block->adev; 407 int r, i; 408 409 r = vcn_v4_0_3_hw_fini(ip_block); 410 if (r) 411 return r; 412 413 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 414 r = amdgpu_vcn_suspend(adev, i); 415 if (r) 416 return r; 417 } 418 419 return 0; 420 } 421 422 /** 423 * vcn_v4_0_3_resume - resume VCN block 424 * 425 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 426 * 427 * Resume firmware and hw init VCN block 428 */ 429 static int vcn_v4_0_3_resume(struct amdgpu_ip_block *ip_block) 430 { 431 struct amdgpu_device *adev = ip_block->adev; 432 int r, i; 433 434 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 435 r = amdgpu_vcn_resume(ip_block->adev, i); 436 if (r) 437 return r; 438 } 439 440 r = vcn_v4_0_3_hw_init(ip_block); 441 442 return r; 443 } 444 445 /** 446 * vcn_v4_0_3_mc_resume - memory controller programming 447 * 448 * @vinst: VCN instance 449 * 450 * Let the VCN memory controller know it's offsets 451 */ 452 static void vcn_v4_0_3_mc_resume(struct amdgpu_vcn_inst *vinst) 453 { 454 struct amdgpu_device *adev = vinst->adev; 455 int inst_idx = vinst->inst; 456 uint32_t offset, size, vcn_inst; 457 const struct common_firmware_header *hdr; 458 459 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 460 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 461 462 vcn_inst = GET_INST(VCN, inst_idx); 463 /* cache window 0: fw */ 464 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 465 WREG32_SOC15( 466 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 467 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 468 .tmr_mc_addr_lo)); 469 WREG32_SOC15( 470 VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 471 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + inst_idx] 472 .tmr_mc_addr_hi)); 473 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 0); 474 offset = 0; 475 } else { 476 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, 477 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 478 WREG32_SOC15(VCN, vcn_inst, 479 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, 480 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr)); 481 offset = size; 482 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET0, 483 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 484 } 485 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE0, size); 486 487 /* cache window 1: stack */ 488 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, 489 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 490 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, 491 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset)); 492 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET1, 0); 493 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE1, 494 AMDGPU_VCN_STACK_SIZE); 495 496 /* cache window 2: context */ 497 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, 498 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 499 AMDGPU_VCN_STACK_SIZE)); 500 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, 501 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 502 AMDGPU_VCN_STACK_SIZE)); 503 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_OFFSET2, 0); 504 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_CACHE_SIZE2, 505 AMDGPU_VCN_CONTEXT_SIZE); 506 507 /* non-cache window */ 508 WREG32_SOC15( 509 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW, 510 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 511 WREG32_SOC15( 512 VCN, vcn_inst, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH, 513 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr)); 514 WREG32_SOC15(VCN, vcn_inst, regUVD_VCPU_NONCACHE_OFFSET0, 0); 515 WREG32_SOC15( 516 VCN, vcn_inst, regUVD_VCPU_NONCACHE_SIZE0, 517 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 518 } 519 520 /** 521 * vcn_v4_0_3_mc_resume_dpg_mode - memory controller programming for dpg mode 522 * 523 * @vinst: VCN instance 524 * @indirect: indirectly write sram 525 * 526 * Let the VCN memory controller know it's offsets with dpg mode 527 */ 528 static void vcn_v4_0_3_mc_resume_dpg_mode(struct amdgpu_vcn_inst *vinst, 529 bool indirect) 530 { 531 struct amdgpu_device *adev = vinst->adev; 532 int inst_idx = vinst->inst; 533 uint32_t offset, size; 534 const struct common_firmware_header *hdr; 535 536 hdr = (const struct common_firmware_header *)adev->vcn.inst[inst_idx].fw->data; 537 size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); 538 539 /* cache window 0: fw */ 540 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 541 if (!indirect) { 542 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 543 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 544 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 545 inst_idx].tmr_mc_addr_lo), 0, indirect); 546 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 547 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 548 (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + 549 inst_idx].tmr_mc_addr_hi), 0, indirect); 550 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 551 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 552 } else { 553 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 554 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 0, 0, indirect); 555 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 556 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 0, 0, indirect); 557 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 558 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 0, 0, indirect); 559 } 560 offset = 0; 561 } else { 562 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 563 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 564 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 565 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 566 VCN, 0, regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 567 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr), 0, indirect); 568 offset = size; 569 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 570 VCN, 0, regUVD_VCPU_CACHE_OFFSET0), 571 AMDGPU_UVD_FIRMWARE_OFFSET >> 3, 0, indirect); 572 } 573 574 if (!indirect) 575 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 576 VCN, 0, regUVD_VCPU_CACHE_SIZE0), size, 0, indirect); 577 else 578 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 579 VCN, 0, regUVD_VCPU_CACHE_SIZE0), 0, 0, indirect); 580 581 /* cache window 1: stack */ 582 if (!indirect) { 583 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 584 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 585 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 586 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 587 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 588 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset), 0, indirect); 589 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 590 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 591 } else { 592 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 593 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), 0, 0, indirect); 594 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 595 VCN, 0, regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), 0, 0, indirect); 596 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 597 VCN, 0, regUVD_VCPU_CACHE_OFFSET1), 0, 0, indirect); 598 } 599 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 600 VCN, 0, regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE, 0, indirect); 601 602 /* cache window 2: context */ 603 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 604 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), 605 lower_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 606 AMDGPU_VCN_STACK_SIZE), 0, indirect); 607 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 608 VCN, 0, regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), 609 upper_32_bits(adev->vcn.inst[inst_idx].gpu_addr + offset + 610 AMDGPU_VCN_STACK_SIZE), 0, indirect); 611 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 612 VCN, 0, regUVD_VCPU_CACHE_OFFSET2), 0, 0, indirect); 613 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 614 VCN, 0, regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE, 0, indirect); 615 616 /* non-cache window */ 617 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 618 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 619 lower_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 620 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 621 VCN, 0, regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 622 upper_32_bits(adev->vcn.inst[inst_idx].fw_shared.gpu_addr), 0, indirect); 623 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 624 VCN, 0, regUVD_VCPU_NONCACHE_OFFSET0), 0, 0, indirect); 625 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 626 VCN, 0, regUVD_VCPU_NONCACHE_SIZE0), 627 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)), 0, indirect); 628 629 /* VCN global tiling registers */ 630 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 631 VCN, 0, regUVD_GFX8_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 632 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 633 VCN, 0, regUVD_GFX10_ADDR_CONFIG), adev->gfx.config.gb_addr_config, 0, indirect); 634 } 635 636 /** 637 * vcn_v4_0_3_disable_clock_gating - disable VCN clock gating 638 * 639 * @vinst: VCN instance 640 * 641 * Disable clock gating for VCN block 642 */ 643 static void vcn_v4_0_3_disable_clock_gating(struct amdgpu_vcn_inst *vinst) 644 { 645 struct amdgpu_device *adev = vinst->adev; 646 int inst_idx = vinst->inst; 647 uint32_t data; 648 int vcn_inst; 649 650 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 651 return; 652 653 vcn_inst = GET_INST(VCN, inst_idx); 654 655 /* VCN disable CGC */ 656 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 657 data &= ~UVD_CGC_CTRL__DYN_CLOCK_MODE_MASK; 658 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 659 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 660 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 661 662 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE); 663 data &= ~(UVD_CGC_GATE__SYS_MASK 664 | UVD_CGC_GATE__MPEG2_MASK 665 | UVD_CGC_GATE__REGS_MASK 666 | UVD_CGC_GATE__RBC_MASK 667 | UVD_CGC_GATE__LMI_MC_MASK 668 | UVD_CGC_GATE__LMI_UMC_MASK 669 | UVD_CGC_GATE__MPC_MASK 670 | UVD_CGC_GATE__LBSI_MASK 671 | UVD_CGC_GATE__LRBBM_MASK 672 | UVD_CGC_GATE__WCB_MASK 673 | UVD_CGC_GATE__VCPU_MASK 674 | UVD_CGC_GATE__MMSCH_MASK); 675 676 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_GATE, data); 677 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_CGC_GATE, 0, 0xFFFFFFFF); 678 679 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 680 data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK 681 | UVD_CGC_CTRL__MPEG2_MODE_MASK 682 | UVD_CGC_CTRL__REGS_MODE_MASK 683 | UVD_CGC_CTRL__RBC_MODE_MASK 684 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 685 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 686 | UVD_CGC_CTRL__MPC_MODE_MASK 687 | UVD_CGC_CTRL__LBSI_MODE_MASK 688 | UVD_CGC_CTRL__LRBBM_MODE_MASK 689 | UVD_CGC_CTRL__WCB_MODE_MASK 690 | UVD_CGC_CTRL__VCPU_MODE_MASK 691 | UVD_CGC_CTRL__MMSCH_MODE_MASK); 692 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 693 694 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE); 695 data |= (UVD_SUVD_CGC_GATE__SRE_MASK 696 | UVD_SUVD_CGC_GATE__SIT_MASK 697 | UVD_SUVD_CGC_GATE__SMP_MASK 698 | UVD_SUVD_CGC_GATE__SCM_MASK 699 | UVD_SUVD_CGC_GATE__SDB_MASK 700 | UVD_SUVD_CGC_GATE__SRE_H264_MASK 701 | UVD_SUVD_CGC_GATE__SRE_HEVC_MASK 702 | UVD_SUVD_CGC_GATE__SIT_H264_MASK 703 | UVD_SUVD_CGC_GATE__SIT_HEVC_MASK 704 | UVD_SUVD_CGC_GATE__SCM_H264_MASK 705 | UVD_SUVD_CGC_GATE__SCM_HEVC_MASK 706 | UVD_SUVD_CGC_GATE__SDB_H264_MASK 707 | UVD_SUVD_CGC_GATE__SDB_HEVC_MASK 708 | UVD_SUVD_CGC_GATE__ENT_MASK 709 | UVD_SUVD_CGC_GATE__SIT_HEVC_DEC_MASK 710 | UVD_SUVD_CGC_GATE__SITE_MASK 711 | UVD_SUVD_CGC_GATE__SRE_VP9_MASK 712 | UVD_SUVD_CGC_GATE__SCM_VP9_MASK 713 | UVD_SUVD_CGC_GATE__SIT_VP9_DEC_MASK 714 | UVD_SUVD_CGC_GATE__SDB_VP9_MASK 715 | UVD_SUVD_CGC_GATE__IME_HEVC_MASK); 716 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_GATE, data); 717 718 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 719 data &= ~(UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 720 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 721 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 722 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 723 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 724 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 725 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 726 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 727 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 728 } 729 730 /** 731 * vcn_v4_0_3_disable_clock_gating_dpg_mode - disable VCN clock gating dpg mode 732 * 733 * @vinst: VCN instance 734 * @sram_sel: sram select 735 * @indirect: indirectly write sram 736 * 737 * Disable clock gating for VCN block with dpg mode 738 */ 739 static void vcn_v4_0_3_disable_clock_gating_dpg_mode(struct amdgpu_vcn_inst *vinst, 740 uint8_t sram_sel, 741 uint8_t indirect) 742 { 743 struct amdgpu_device *adev = vinst->adev; 744 int inst_idx = vinst->inst; 745 uint32_t reg_data = 0; 746 747 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 748 return; 749 750 /* enable sw clock gating control */ 751 reg_data = 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 752 reg_data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 753 reg_data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 754 reg_data &= ~(UVD_CGC_CTRL__SYS_MODE_MASK | 755 UVD_CGC_CTRL__MPEG2_MODE_MASK | 756 UVD_CGC_CTRL__REGS_MODE_MASK | 757 UVD_CGC_CTRL__RBC_MODE_MASK | 758 UVD_CGC_CTRL__LMI_MC_MODE_MASK | 759 UVD_CGC_CTRL__LMI_UMC_MODE_MASK | 760 UVD_CGC_CTRL__IDCT_MODE_MASK | 761 UVD_CGC_CTRL__MPRD_MODE_MASK | 762 UVD_CGC_CTRL__MPC_MODE_MASK | 763 UVD_CGC_CTRL__LBSI_MODE_MASK | 764 UVD_CGC_CTRL__LRBBM_MODE_MASK | 765 UVD_CGC_CTRL__WCB_MODE_MASK | 766 UVD_CGC_CTRL__VCPU_MODE_MASK); 767 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 768 VCN, 0, regUVD_CGC_CTRL), reg_data, sram_sel, indirect); 769 770 /* turn off clock gating */ 771 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 772 VCN, 0, regUVD_CGC_GATE), 0, sram_sel, indirect); 773 774 /* turn on SUVD clock gating */ 775 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 776 VCN, 0, regUVD_SUVD_CGC_GATE), 1, sram_sel, indirect); 777 778 /* turn on sw mode in UVD_SUVD_CGC_CTRL */ 779 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 780 VCN, 0, regUVD_SUVD_CGC_CTRL), 0, sram_sel, indirect); 781 } 782 783 /** 784 * vcn_v4_0_3_enable_clock_gating - enable VCN clock gating 785 * 786 * @vinst: VCN instance 787 * 788 * Enable clock gating for VCN block 789 */ 790 static void vcn_v4_0_3_enable_clock_gating(struct amdgpu_vcn_inst *vinst) 791 { 792 struct amdgpu_device *adev = vinst->adev; 793 int inst_idx = vinst->inst; 794 uint32_t data; 795 int vcn_inst; 796 797 if (adev->cg_flags & AMD_CG_SUPPORT_VCN_MGCG) 798 return; 799 800 vcn_inst = GET_INST(VCN, inst_idx); 801 802 /* enable VCN CGC */ 803 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 804 data |= 0 << UVD_CGC_CTRL__DYN_CLOCK_MODE__SHIFT; 805 data |= 1 << UVD_CGC_CTRL__CLK_GATE_DLY_TIMER__SHIFT; 806 data |= 4 << UVD_CGC_CTRL__CLK_OFF_DELAY__SHIFT; 807 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 808 809 data = RREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL); 810 data |= (UVD_CGC_CTRL__SYS_MODE_MASK 811 | UVD_CGC_CTRL__MPEG2_MODE_MASK 812 | UVD_CGC_CTRL__REGS_MODE_MASK 813 | UVD_CGC_CTRL__RBC_MODE_MASK 814 | UVD_CGC_CTRL__LMI_MC_MODE_MASK 815 | UVD_CGC_CTRL__LMI_UMC_MODE_MASK 816 | UVD_CGC_CTRL__MPC_MODE_MASK 817 | UVD_CGC_CTRL__LBSI_MODE_MASK 818 | UVD_CGC_CTRL__LRBBM_MODE_MASK 819 | UVD_CGC_CTRL__WCB_MODE_MASK 820 | UVD_CGC_CTRL__VCPU_MODE_MASK); 821 WREG32_SOC15(VCN, vcn_inst, regUVD_CGC_CTRL, data); 822 823 data = RREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL); 824 data |= (UVD_SUVD_CGC_CTRL__SRE_MODE_MASK 825 | UVD_SUVD_CGC_CTRL__SIT_MODE_MASK 826 | UVD_SUVD_CGC_CTRL__SMP_MODE_MASK 827 | UVD_SUVD_CGC_CTRL__SCM_MODE_MASK 828 | UVD_SUVD_CGC_CTRL__SDB_MODE_MASK 829 | UVD_SUVD_CGC_CTRL__ENT_MODE_MASK 830 | UVD_SUVD_CGC_CTRL__IME_MODE_MASK 831 | UVD_SUVD_CGC_CTRL__SITE_MODE_MASK); 832 WREG32_SOC15(VCN, vcn_inst, regUVD_SUVD_CGC_CTRL, data); 833 } 834 835 /** 836 * vcn_v4_0_3_start_dpg_mode - VCN start with dpg mode 837 * 838 * @vinst: VCN instance 839 * @indirect: indirectly write sram 840 * 841 * Start VCN block with dpg mode 842 */ 843 static int vcn_v4_0_3_start_dpg_mode(struct amdgpu_vcn_inst *vinst, 844 bool indirect) 845 { 846 struct amdgpu_device *adev = vinst->adev; 847 int inst_idx = vinst->inst; 848 struct amdgpu_vcn4_fw_shared *fw_shared = 849 adev->vcn.inst[inst_idx].fw_shared.cpu_addr; 850 struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__PAUSE}; 851 struct amdgpu_ring *ring; 852 int vcn_inst, ret; 853 uint32_t tmp; 854 855 vcn_inst = GET_INST(VCN, inst_idx); 856 /* disable register anti-hang mechanism */ 857 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 1, 858 ~UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 859 /* enable dynamic power gating mode */ 860 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS); 861 tmp |= UVD_POWER_STATUS__UVD_PG_MODE_MASK; 862 tmp |= UVD_POWER_STATUS__UVD_PG_EN_MASK; 863 WREG32_SOC15(VCN, vcn_inst, regUVD_POWER_STATUS, tmp); 864 865 if (indirect) { 866 DRM_DEV_DEBUG(adev->dev, "VCN %d start: on AID %d", 867 inst_idx, adev->vcn.inst[inst_idx].aid_id); 868 adev->vcn.inst[inst_idx].dpg_sram_curr_addr = 869 (uint32_t *)adev->vcn.inst[inst_idx].dpg_sram_cpu_addr; 870 /* Use dummy register 0xDEADBEEF passing AID selection to PSP FW */ 871 WREG32_SOC15_DPG_MODE(inst_idx, 0xDEADBEEF, 872 adev->vcn.inst[inst_idx].aid_id, 0, true); 873 } 874 875 /* enable clock gating */ 876 vcn_v4_0_3_disable_clock_gating_dpg_mode(vinst, 0, indirect); 877 878 /* enable VCPU clock */ 879 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 880 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 881 tmp |= UVD_VCPU_CNTL__BLK_RST_MASK; 882 883 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 884 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 885 886 /* disable master interrupt */ 887 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 888 VCN, 0, regUVD_MASTINT_EN), 0, 0, indirect); 889 890 /* setup regUVD_LMI_CTRL */ 891 tmp = (UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 892 UVD_LMI_CTRL__REQ_MODE_MASK | 893 UVD_LMI_CTRL__CRC_RESET_MASK | 894 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 895 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 896 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK | 897 (8 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) | 898 0x00100000L); 899 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 900 VCN, 0, regUVD_LMI_CTRL), tmp, 0, indirect); 901 902 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 903 VCN, 0, regUVD_MPC_CNTL), 904 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT, 0, indirect); 905 906 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 907 VCN, 0, regUVD_MPC_SET_MUXA0), 908 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 909 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 910 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 911 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT)), 0, indirect); 912 913 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 914 VCN, 0, regUVD_MPC_SET_MUXB0), 915 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 916 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 917 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 918 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT)), 0, indirect); 919 920 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 921 VCN, 0, regUVD_MPC_SET_MUX), 922 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 923 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 924 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT)), 0, indirect); 925 926 vcn_v4_0_3_mc_resume_dpg_mode(vinst, indirect); 927 928 tmp = (0xFF << UVD_VCPU_CNTL__PRB_TIMEOUT_VAL__SHIFT); 929 tmp |= UVD_VCPU_CNTL__CLK_EN_MASK; 930 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 931 VCN, 0, regUVD_VCPU_CNTL), tmp, 0, indirect); 932 933 /* enable LMI MC and UMC channels */ 934 tmp = 0x1f << UVD_LMI_CTRL2__RE_OFLD_MIF_WR_REQ_NUM__SHIFT; 935 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 936 VCN, 0, regUVD_LMI_CTRL2), tmp, 0, indirect); 937 938 vcn_v4_0_3_enable_ras(adev, inst_idx, indirect); 939 940 /* enable master interrupt */ 941 WREG32_SOC15_DPG_MODE(inst_idx, SOC15_DPG_MODE_OFFSET( 942 VCN, 0, regUVD_MASTINT_EN), 943 UVD_MASTINT_EN__VCPU_EN_MASK, 0, indirect); 944 945 if (indirect) { 946 ret = amdgpu_vcn_psp_update_sram(adev, inst_idx, AMDGPU_UCODE_ID_VCN0_RAM); 947 if (ret) { 948 dev_err(adev->dev, "vcn sram load failed %d\n", ret); 949 return ret; 950 } 951 } 952 953 ring = &adev->vcn.inst[inst_idx].ring_enc[0]; 954 955 /* Pause dpg */ 956 vcn_v4_0_3_pause_dpg_mode(vinst, &state); 957 958 /* program the RB_BASE for ring buffer */ 959 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 960 lower_32_bits(ring->gpu_addr)); 961 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 962 upper_32_bits(ring->gpu_addr)); 963 964 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 965 ring->ring_size / sizeof(uint32_t)); 966 967 /* resetting ring, fw should not check RB ring */ 968 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 969 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 970 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 971 fw_shared->sq.queue_mode |= FW_QUEUE_RING_RESET; 972 973 /* Initialize the ring buffer's read and write pointers */ 974 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 975 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 976 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 977 978 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 979 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 980 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 981 fw_shared->sq.queue_mode &= ~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF); 982 983 /*resetting done, fw can check RB ring */ 984 fw_shared->sq.queue_mode &= cpu_to_le32(~FW_QUEUE_RING_RESET); 985 986 /* Keeping one read-back to ensure all register writes are done, 987 * otherwise it may introduce race conditions. 988 */ 989 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 990 991 return 0; 992 } 993 994 static int vcn_v4_0_3_start_sriov(struct amdgpu_device *adev) 995 { 996 int i, vcn_inst; 997 struct amdgpu_ring *ring_enc; 998 uint64_t cache_addr; 999 uint64_t rb_enc_addr; 1000 uint64_t ctx_addr; 1001 uint32_t param, resp, expected; 1002 uint32_t offset, cache_size; 1003 uint32_t tmp, timeout; 1004 1005 struct amdgpu_mm_table *table = &adev->virt.mm_table; 1006 uint32_t *table_loc; 1007 uint32_t table_size; 1008 uint32_t size, size_dw; 1009 uint32_t init_status; 1010 uint32_t enabled_vcn; 1011 1012 struct mmsch_v4_0_cmd_direct_write 1013 direct_wt = { {0} }; 1014 struct mmsch_v4_0_cmd_direct_read_modify_write 1015 direct_rd_mod_wt = { {0} }; 1016 struct mmsch_v4_0_cmd_end end = { {0} }; 1017 struct mmsch_v4_0_3_init_header header; 1018 1019 struct amdgpu_vcn4_fw_shared *fw_shared; 1020 struct amdgpu_fw_shared_rb_setup *rb_setup; 1021 1022 direct_wt.cmd_header.command_type = 1023 MMSCH_COMMAND__DIRECT_REG_WRITE; 1024 direct_rd_mod_wt.cmd_header.command_type = 1025 MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE; 1026 end.cmd_header.command_type = MMSCH_COMMAND__END; 1027 1028 for (i = 0; i < adev->vcn.num_vcn_inst; i++) { 1029 vcn_inst = GET_INST(VCN, i); 1030 1031 vcn_v4_0_3_fw_shared_init(adev, vcn_inst); 1032 1033 memset(&header, 0, sizeof(struct mmsch_v4_0_3_init_header)); 1034 header.version = MMSCH_VERSION; 1035 header.total_size = sizeof(struct mmsch_v4_0_3_init_header) >> 2; 1036 1037 table_loc = (uint32_t *)table->cpu_addr; 1038 table_loc += header.total_size; 1039 1040 table_size = 0; 1041 1042 MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, 0, regUVD_STATUS), 1043 ~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY); 1044 1045 cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.inst[i].fw->size + 4); 1046 1047 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { 1048 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1049 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1050 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo); 1051 1052 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1053 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1054 adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi); 1055 1056 offset = 0; 1057 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1058 regUVD_VCPU_CACHE_OFFSET0), 0); 1059 } else { 1060 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1061 regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW), 1062 lower_32_bits(adev->vcn.inst[i].gpu_addr)); 1063 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1064 regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH), 1065 upper_32_bits(adev->vcn.inst[i].gpu_addr)); 1066 offset = cache_size; 1067 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1068 regUVD_VCPU_CACHE_OFFSET0), 1069 AMDGPU_UVD_FIRMWARE_OFFSET >> 3); 1070 } 1071 1072 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1073 regUVD_VCPU_CACHE_SIZE0), 1074 cache_size); 1075 1076 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset; 1077 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1078 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1079 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1080 regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1081 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1082 regUVD_VCPU_CACHE_OFFSET1), 0); 1083 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1084 regUVD_VCPU_CACHE_SIZE1), AMDGPU_VCN_STACK_SIZE); 1085 1086 cache_addr = adev->vcn.inst[vcn_inst].gpu_addr + offset + 1087 AMDGPU_VCN_STACK_SIZE; 1088 1089 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1090 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW), lower_32_bits(cache_addr)); 1091 1092 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1093 regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH), upper_32_bits(cache_addr)); 1094 1095 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1096 regUVD_VCPU_CACHE_OFFSET2), 0); 1097 1098 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1099 regUVD_VCPU_CACHE_SIZE2), AMDGPU_VCN_CONTEXT_SIZE); 1100 1101 fw_shared = adev->vcn.inst[vcn_inst].fw_shared.cpu_addr; 1102 rb_setup = &fw_shared->rb_setup; 1103 1104 ring_enc = &adev->vcn.inst[vcn_inst].ring_enc[0]; 1105 ring_enc->wptr = 0; 1106 rb_enc_addr = ring_enc->gpu_addr; 1107 1108 rb_setup->is_rb_enabled_flags |= RB_ENABLED; 1109 rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr); 1110 rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr); 1111 rb_setup->rb_size = ring_enc->ring_size / 4; 1112 fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG); 1113 1114 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1115 regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW), 1116 lower_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1117 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1118 regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH), 1119 upper_32_bits(adev->vcn.inst[vcn_inst].fw_shared.gpu_addr)); 1120 MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, 0, 1121 regUVD_VCPU_NONCACHE_SIZE0), 1122 AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared))); 1123 MMSCH_V4_0_INSERT_END(); 1124 1125 header.vcn0.init_status = 0; 1126 header.vcn0.table_offset = header.total_size; 1127 header.vcn0.table_size = table_size; 1128 header.total_size += table_size; 1129 1130 /* Send init table to mmsch */ 1131 size = sizeof(struct mmsch_v4_0_3_init_header); 1132 table_loc = (uint32_t *)table->cpu_addr; 1133 memcpy((void *)table_loc, &header, size); 1134 1135 ctx_addr = table->gpu_addr; 1136 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr)); 1137 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr)); 1138 1139 tmp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID); 1140 tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK; 1141 tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT); 1142 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_VMID, tmp); 1143 1144 size = header.total_size; 1145 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_CTX_SIZE, size); 1146 1147 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP, 0); 1148 1149 param = 0x00000001; 1150 WREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_HOST, param); 1151 tmp = 0; 1152 timeout = 1000; 1153 resp = 0; 1154 expected = MMSCH_VF_MAILBOX_RESP__OK; 1155 while (resp != expected) { 1156 resp = RREG32_SOC15(VCN, vcn_inst, regMMSCH_VF_MAILBOX_RESP); 1157 if (resp != 0) 1158 break; 1159 1160 udelay(10); 1161 tmp = tmp + 10; 1162 if (tmp >= timeout) { 1163 DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\ 1164 " waiting for regMMSCH_VF_MAILBOX_RESP "\ 1165 "(expected=0x%08x, readback=0x%08x)\n", 1166 tmp, expected, resp); 1167 return -EBUSY; 1168 } 1169 } 1170 1171 enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0; 1172 init_status = ((struct mmsch_v4_0_3_init_header *)(table_loc))->vcn0.init_status; 1173 if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE 1174 && init_status != MMSCH_VF_ENGINE_STATUS__PASS) { 1175 DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\ 1176 "status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status); 1177 } 1178 } 1179 1180 return 0; 1181 } 1182 1183 /** 1184 * vcn_v4_0_3_start - VCN start 1185 * 1186 * @vinst: VCN instance 1187 * 1188 * Start VCN block 1189 */ 1190 static int vcn_v4_0_3_start(struct amdgpu_vcn_inst *vinst) 1191 { 1192 struct amdgpu_device *adev = vinst->adev; 1193 int i = vinst->inst; 1194 struct amdgpu_vcn4_fw_shared *fw_shared; 1195 struct amdgpu_ring *ring; 1196 int j, k, r, vcn_inst; 1197 uint32_t tmp; 1198 1199 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) 1200 return vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[i].indirect_sram); 1201 1202 vcn_inst = GET_INST(VCN, i); 1203 /* set VCN status busy */ 1204 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS) | 1205 UVD_STATUS__UVD_BUSY; 1206 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, tmp); 1207 1208 /* SW clock gating */ 1209 vcn_v4_0_3_disable_clock_gating(vinst); 1210 1211 /* enable VCPU clock */ 1212 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1213 UVD_VCPU_CNTL__CLK_EN_MASK, 1214 ~UVD_VCPU_CNTL__CLK_EN_MASK); 1215 1216 /* disable master interrupt */ 1217 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 0, 1218 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1219 1220 /* enable LMI MC and UMC channels */ 1221 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_LMI_CTRL2), 0, 1222 ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK); 1223 1224 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1225 tmp &= ~UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1226 tmp &= ~UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1227 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1228 1229 /* setup regUVD_LMI_CTRL */ 1230 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL); 1231 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL, 1232 tmp | UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK | 1233 UVD_LMI_CTRL__MASK_MC_URGENT_MASK | 1234 UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK | 1235 UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK); 1236 1237 /* setup regUVD_MPC_CNTL */ 1238 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL); 1239 tmp &= ~UVD_MPC_CNTL__REPLACEMENT_MODE_MASK; 1240 tmp |= 0x2 << UVD_MPC_CNTL__REPLACEMENT_MODE__SHIFT; 1241 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_CNTL, tmp); 1242 1243 /* setup UVD_MPC_SET_MUXA0 */ 1244 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXA0, 1245 ((0x1 << UVD_MPC_SET_MUXA0__VARA_1__SHIFT) | 1246 (0x2 << UVD_MPC_SET_MUXA0__VARA_2__SHIFT) | 1247 (0x3 << UVD_MPC_SET_MUXA0__VARA_3__SHIFT) | 1248 (0x4 << UVD_MPC_SET_MUXA0__VARA_4__SHIFT))); 1249 1250 /* setup UVD_MPC_SET_MUXB0 */ 1251 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUXB0, 1252 ((0x1 << UVD_MPC_SET_MUXB0__VARB_1__SHIFT) | 1253 (0x2 << UVD_MPC_SET_MUXB0__VARB_2__SHIFT) | 1254 (0x3 << UVD_MPC_SET_MUXB0__VARB_3__SHIFT) | 1255 (0x4 << UVD_MPC_SET_MUXB0__VARB_4__SHIFT))); 1256 1257 /* setup UVD_MPC_SET_MUX */ 1258 WREG32_SOC15(VCN, vcn_inst, regUVD_MPC_SET_MUX, 1259 ((0x0 << UVD_MPC_SET_MUX__SET_0__SHIFT) | 1260 (0x1 << UVD_MPC_SET_MUX__SET_1__SHIFT) | 1261 (0x2 << UVD_MPC_SET_MUX__SET_2__SHIFT))); 1262 1263 vcn_v4_0_3_mc_resume(vinst); 1264 1265 /* VCN global tiling registers */ 1266 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX8_ADDR_CONFIG, 1267 adev->gfx.config.gb_addr_config); 1268 WREG32_SOC15(VCN, vcn_inst, regUVD_GFX10_ADDR_CONFIG, 1269 adev->gfx.config.gb_addr_config); 1270 1271 /* unblock VCPU register access */ 1272 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 0, 1273 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1274 1275 /* release VCPU reset to boot */ 1276 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1277 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1278 1279 for (j = 0; j < 10; ++j) { 1280 uint32_t status; 1281 1282 for (k = 0; k < 100; ++k) { 1283 status = RREG32_SOC15(VCN, vcn_inst, 1284 regUVD_STATUS); 1285 if (status & 2) 1286 break; 1287 mdelay(10); 1288 } 1289 r = 0; 1290 if (status & 2) 1291 break; 1292 1293 DRM_DEV_ERROR(adev->dev, 1294 "VCN decode not responding, trying to reset the VCPU!!!\n"); 1295 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1296 regUVD_VCPU_CNTL), 1297 UVD_VCPU_CNTL__BLK_RST_MASK, 1298 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1299 mdelay(10); 1300 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, 1301 regUVD_VCPU_CNTL), 1302 0, ~UVD_VCPU_CNTL__BLK_RST_MASK); 1303 1304 mdelay(10); 1305 r = -1; 1306 } 1307 1308 if (r) { 1309 DRM_DEV_ERROR(adev->dev, "VCN decode not responding, giving up!!!\n"); 1310 return r; 1311 } 1312 1313 /* enable master interrupt */ 1314 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_MASTINT_EN), 1315 UVD_MASTINT_EN__VCPU_EN_MASK, 1316 ~UVD_MASTINT_EN__VCPU_EN_MASK); 1317 1318 /* clear the busy bit of VCN_STATUS */ 1319 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_STATUS), 0, 1320 ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT)); 1321 1322 ring = &adev->vcn.inst[i].ring_enc[0]; 1323 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1324 1325 /* program the RB_BASE for ring buffer */ 1326 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_LO, 1327 lower_32_bits(ring->gpu_addr)); 1328 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_BASE_HI, 1329 upper_32_bits(ring->gpu_addr)); 1330 1331 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_SIZE, 1332 ring->ring_size / sizeof(uint32_t)); 1333 1334 /* resetting ring, fw should not check RB ring */ 1335 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1336 tmp &= ~(VCN_RB_ENABLE__RB_EN_MASK); 1337 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1338 1339 /* Initialize the ring buffer's read and write pointers */ 1340 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_RPTR, 0); 1341 WREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR, 0); 1342 1343 tmp = RREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE); 1344 tmp |= VCN_RB_ENABLE__RB_EN_MASK; 1345 WREG32_SOC15(VCN, vcn_inst, regVCN_RB_ENABLE, tmp); 1346 1347 ring->wptr = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1348 fw_shared->sq.queue_mode &= 1349 cpu_to_le32(~(FW_QUEUE_RING_RESET | FW_QUEUE_DPG_HOLD_OFF)); 1350 1351 return 0; 1352 } 1353 1354 /** 1355 * vcn_v4_0_3_stop_dpg_mode - VCN stop with dpg mode 1356 * 1357 * @vinst: VCN instance 1358 * 1359 * Stop VCN block with dpg mode 1360 */ 1361 static int vcn_v4_0_3_stop_dpg_mode(struct amdgpu_vcn_inst *vinst) 1362 { 1363 struct amdgpu_device *adev = vinst->adev; 1364 int inst_idx = vinst->inst; 1365 uint32_t tmp; 1366 int vcn_inst; 1367 struct dpg_pause_state state = {.fw_based = VCN_DPG_STATE__UNPAUSE}; 1368 1369 vcn_inst = GET_INST(VCN, inst_idx); 1370 1371 /* Unpause dpg */ 1372 vcn_v4_0_3_pause_dpg_mode(vinst, &state); 1373 1374 /* Wait for power status to be 1 */ 1375 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1376 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1377 1378 /* wait for read ptr to be equal to write ptr */ 1379 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_RB_WPTR); 1380 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_RB_RPTR, tmp, 0xFFFFFFFF); 1381 1382 SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_POWER_STATUS, 1, 1383 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1384 1385 /* disable dynamic power gating mode */ 1386 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_POWER_STATUS), 0, 1387 ~UVD_POWER_STATUS__UVD_PG_MODE_MASK); 1388 1389 /* Keeping one read-back to ensure all register writes are done, 1390 * otherwise it may introduce race conditions. 1391 */ 1392 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1393 1394 return 0; 1395 } 1396 1397 /** 1398 * vcn_v4_0_3_stop - VCN stop 1399 * 1400 * @vinst: VCN instance 1401 * 1402 * Stop VCN block 1403 */ 1404 static int vcn_v4_0_3_stop(struct amdgpu_vcn_inst *vinst) 1405 { 1406 struct amdgpu_device *adev = vinst->adev; 1407 int i = vinst->inst; 1408 struct amdgpu_vcn4_fw_shared *fw_shared; 1409 int r = 0, vcn_inst; 1410 uint32_t tmp; 1411 1412 vcn_inst = GET_INST(VCN, i); 1413 1414 fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr; 1415 fw_shared->sq.queue_mode |= FW_QUEUE_DPG_HOLD_OFF; 1416 1417 if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) { 1418 vcn_v4_0_3_stop_dpg_mode(vinst); 1419 goto Done; 1420 } 1421 1422 /* wait for vcn idle */ 1423 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_STATUS, 1424 UVD_STATUS__IDLE, 0x7); 1425 if (r) 1426 goto Done; 1427 1428 tmp = UVD_LMI_STATUS__VCPU_LMI_WRITE_CLEAN_MASK | 1429 UVD_LMI_STATUS__READ_CLEAN_MASK | 1430 UVD_LMI_STATUS__WRITE_CLEAN_MASK | 1431 UVD_LMI_STATUS__WRITE_CLEAN_RAW_MASK; 1432 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1433 tmp); 1434 if (r) 1435 goto Done; 1436 1437 /* stall UMC channel */ 1438 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2); 1439 tmp |= UVD_LMI_CTRL2__STALL_ARB_UMC_MASK; 1440 WREG32_SOC15(VCN, vcn_inst, regUVD_LMI_CTRL2, tmp); 1441 tmp = UVD_LMI_STATUS__UMC_READ_CLEAN_RAW_MASK | 1442 UVD_LMI_STATUS__UMC_WRITE_CLEAN_RAW_MASK; 1443 r = SOC15_WAIT_ON_RREG(VCN, vcn_inst, regUVD_LMI_STATUS, tmp, 1444 tmp); 1445 if (r) 1446 goto Done; 1447 1448 /* Unblock VCPU Register access */ 1449 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_RB_ARB_CTRL), 1450 UVD_RB_ARB_CTRL__VCPU_DIS_MASK, 1451 ~UVD_RB_ARB_CTRL__VCPU_DIS_MASK); 1452 1453 /* release VCPU reset to boot */ 1454 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 1455 UVD_VCPU_CNTL__BLK_RST_MASK, 1456 ~UVD_VCPU_CNTL__BLK_RST_MASK); 1457 1458 /* disable VCPU clock */ 1459 WREG32_P(SOC15_REG_OFFSET(VCN, vcn_inst, regUVD_VCPU_CNTL), 0, 1460 ~(UVD_VCPU_CNTL__CLK_EN_MASK)); 1461 1462 /* reset LMI UMC/LMI/VCPU */ 1463 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1464 tmp |= UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK; 1465 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1466 1467 tmp = RREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET); 1468 tmp |= UVD_SOFT_RESET__LMI_SOFT_RESET_MASK; 1469 WREG32_SOC15(VCN, vcn_inst, regUVD_SOFT_RESET, tmp); 1470 1471 /* clear VCN status */ 1472 WREG32_SOC15(VCN, vcn_inst, regUVD_STATUS, 0); 1473 1474 /* apply HW clock gating */ 1475 vcn_v4_0_3_enable_clock_gating(vinst); 1476 1477 /* Keeping one read-back to ensure all register writes are done, 1478 * otherwise it may introduce race conditions. 1479 */ 1480 RREG32_SOC15(VCN, vcn_inst, regUVD_STATUS); 1481 1482 Done: 1483 return 0; 1484 } 1485 1486 /** 1487 * vcn_v4_0_3_pause_dpg_mode - VCN pause with dpg mode 1488 * 1489 * @vinst: VCN instance 1490 * @new_state: pause state 1491 * 1492 * Pause dpg mode for VCN block 1493 */ 1494 static int vcn_v4_0_3_pause_dpg_mode(struct amdgpu_vcn_inst *vinst, 1495 struct dpg_pause_state *new_state) 1496 { 1497 struct amdgpu_device *adev = vinst->adev; 1498 int inst_idx = vinst->inst; 1499 uint32_t reg_data = 0; 1500 int ret_code; 1501 1502 /* pause/unpause if state is changed */ 1503 if (adev->vcn.inst[inst_idx].pause_state.fw_based != new_state->fw_based) { 1504 DRM_DEV_DEBUG(adev->dev, "dpg pause state changed %d -> %d", 1505 adev->vcn.inst[inst_idx].pause_state.fw_based, new_state->fw_based); 1506 reg_data = RREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE) & 1507 (~UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK); 1508 1509 if (new_state->fw_based == VCN_DPG_STATE__PAUSE) { 1510 ret_code = SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_POWER_STATUS, 0x1, 1511 UVD_POWER_STATUS__UVD_POWER_STATUS_MASK); 1512 1513 if (!ret_code) { 1514 /* pause DPG */ 1515 reg_data |= UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK; 1516 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data); 1517 1518 /* wait for ACK */ 1519 SOC15_WAIT_ON_RREG(VCN, inst_idx, regUVD_DPG_PAUSE, 1520 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK, 1521 UVD_DPG_PAUSE__NJ_PAUSE_DPG_ACK_MASK); 1522 } 1523 } else { 1524 /* unpause dpg, no need to wait */ 1525 reg_data &= ~UVD_DPG_PAUSE__NJ_PAUSE_DPG_REQ_MASK; 1526 WREG32_SOC15(VCN, inst_idx, regUVD_DPG_PAUSE, reg_data); 1527 } 1528 adev->vcn.inst[inst_idx].pause_state.fw_based = new_state->fw_based; 1529 } 1530 1531 return 0; 1532 } 1533 1534 /** 1535 * vcn_v4_0_3_unified_ring_get_rptr - get unified read pointer 1536 * 1537 * @ring: amdgpu_ring pointer 1538 * 1539 * Returns the current hardware unified read pointer 1540 */ 1541 static uint64_t vcn_v4_0_3_unified_ring_get_rptr(struct amdgpu_ring *ring) 1542 { 1543 struct amdgpu_device *adev = ring->adev; 1544 1545 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1546 DRM_ERROR("wrong ring id is identified in %s", __func__); 1547 1548 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_RPTR); 1549 } 1550 1551 /** 1552 * vcn_v4_0_3_unified_ring_get_wptr - get unified write pointer 1553 * 1554 * @ring: amdgpu_ring pointer 1555 * 1556 * Returns the current hardware unified write pointer 1557 */ 1558 static uint64_t vcn_v4_0_3_unified_ring_get_wptr(struct amdgpu_ring *ring) 1559 { 1560 struct amdgpu_device *adev = ring->adev; 1561 1562 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1563 DRM_ERROR("wrong ring id is identified in %s", __func__); 1564 1565 if (ring->use_doorbell) 1566 return *ring->wptr_cpu_addr; 1567 else 1568 return RREG32_SOC15(VCN, GET_INST(VCN, ring->me), 1569 regUVD_RB_WPTR); 1570 } 1571 1572 void vcn_v4_0_3_enc_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg, 1573 uint32_t val, uint32_t mask) 1574 { 1575 /* Use normalized offsets when required */ 1576 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1577 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1578 1579 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WAIT); 1580 amdgpu_ring_write(ring, reg << 2); 1581 amdgpu_ring_write(ring, mask); 1582 amdgpu_ring_write(ring, val); 1583 } 1584 1585 void vcn_v4_0_3_enc_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg, 1586 uint32_t val) 1587 { 1588 /* Use normalized offsets when required */ 1589 if (vcn_v4_0_3_normalizn_reqd(ring->adev)) 1590 reg = NORMALIZE_VCN_REG_OFFSET(reg); 1591 1592 amdgpu_ring_write(ring, VCN_ENC_CMD_REG_WRITE); 1593 amdgpu_ring_write(ring, reg << 2); 1594 amdgpu_ring_write(ring, val); 1595 } 1596 1597 void vcn_v4_0_3_enc_ring_emit_vm_flush(struct amdgpu_ring *ring, 1598 unsigned int vmid, uint64_t pd_addr) 1599 { 1600 struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub]; 1601 1602 pd_addr = amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr); 1603 1604 /* wait for reg writes */ 1605 vcn_v4_0_3_enc_ring_emit_reg_wait(ring, hub->ctx0_ptb_addr_lo32 + 1606 vmid * hub->ctx_addr_distance, 1607 lower_32_bits(pd_addr), 0xffffffff); 1608 } 1609 1610 void vcn_v4_0_3_ring_emit_hdp_flush(struct amdgpu_ring *ring) 1611 { 1612 /* VCN engine access for HDP flush doesn't work when RRMT is enabled. 1613 * This is a workaround to avoid any HDP flush through VCN ring. 1614 */ 1615 } 1616 1617 /** 1618 * vcn_v4_0_3_unified_ring_set_wptr - set enc write pointer 1619 * 1620 * @ring: amdgpu_ring pointer 1621 * 1622 * Commits the enc write pointer to the hardware 1623 */ 1624 static void vcn_v4_0_3_unified_ring_set_wptr(struct amdgpu_ring *ring) 1625 { 1626 struct amdgpu_device *adev = ring->adev; 1627 1628 if (ring != &adev->vcn.inst[ring->me].ring_enc[0]) 1629 DRM_ERROR("wrong ring id is identified in %s", __func__); 1630 1631 if (ring->use_doorbell) { 1632 *ring->wptr_cpu_addr = lower_32_bits(ring->wptr); 1633 WDOORBELL32(ring->doorbell_index, lower_32_bits(ring->wptr)); 1634 } else { 1635 WREG32_SOC15(VCN, GET_INST(VCN, ring->me), regUVD_RB_WPTR, 1636 lower_32_bits(ring->wptr)); 1637 } 1638 } 1639 1640 static int vcn_v4_0_3_reset_jpeg_pre_helper(struct amdgpu_device *adev, int inst) 1641 { 1642 struct amdgpu_ring *ring; 1643 uint32_t wait_seq = 0; 1644 int i; 1645 1646 for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) { 1647 ring = &adev->jpeg.inst[inst].ring_dec[i]; 1648 1649 drm_sched_wqueue_stop(&ring->sched); 1650 /* Get the last emitted fence sequence */ 1651 wait_seq = atomic_read(&ring->fence_drv.last_seq); 1652 if (wait_seq) 1653 continue; 1654 1655 /* if Jobs are still pending after timeout, 1656 * We'll handle them in the bottom helper 1657 */ 1658 amdgpu_fence_wait_polling(ring, wait_seq, adev->video_timeout); 1659 } 1660 1661 return 0; 1662 } 1663 1664 static int vcn_v4_0_3_reset_jpeg_post_helper(struct amdgpu_device *adev, int inst) 1665 { 1666 struct amdgpu_ring *ring; 1667 int i, r = 0; 1668 1669 for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) { 1670 ring = &adev->jpeg.inst[inst].ring_dec[i]; 1671 /* Force completion of any remaining jobs */ 1672 amdgpu_fence_driver_force_completion(ring); 1673 1674 if (ring->use_doorbell) 1675 WREG32_SOC15_OFFSET( 1676 VCN, GET_INST(VCN, inst), 1677 regVCN_JPEG_DB_CTRL, 1678 (ring->pipe ? (ring->pipe - 0x15) : 0), 1679 ring->doorbell_index << VCN_JPEG_DB_CTRL__OFFSET__SHIFT | 1680 VCN_JPEG_DB_CTRL__EN_MASK); 1681 1682 r = amdgpu_ring_test_helper(ring); 1683 if (r) 1684 return r; 1685 1686 drm_sched_wqueue_start(&ring->sched); 1687 1688 DRM_DEV_DEBUG(adev->dev, "JPEG ring %d (inst %d) restored and sched restarted\n", 1689 i, inst); 1690 } 1691 return 0; 1692 } 1693 1694 static int vcn_v4_0_3_ring_reset(struct amdgpu_ring *ring, 1695 unsigned int vmid, 1696 struct amdgpu_fence *timedout_fence) 1697 { 1698 int r = 0; 1699 int vcn_inst; 1700 struct amdgpu_device *adev = ring->adev; 1701 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me]; 1702 bool pg_state = false; 1703 1704 /* take the vcn reset mutex here because resetting VCN will reset jpeg as well */ 1705 mutex_lock(&vinst->engine_reset_mutex); 1706 mutex_lock(&adev->jpeg.jpeg_pg_lock); 1707 /* Ensure JPEG is powered on during reset if currently gated */ 1708 if (adev->jpeg.cur_state == AMD_PG_STATE_GATE) { 1709 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG, 1710 AMD_PG_STATE_UNGATE); 1711 pg_state = true; 1712 } 1713 1714 vcn_v4_0_3_reset_jpeg_pre_helper(adev, ring->me); 1715 amdgpu_ring_reset_helper_begin(ring, timedout_fence); 1716 1717 vcn_inst = GET_INST(VCN, ring->me); 1718 r = amdgpu_dpm_reset_vcn(adev, 1 << vcn_inst); 1719 1720 if (r) { 1721 DRM_DEV_ERROR(adev->dev, "VCN reset fail : %d\n", r); 1722 /* Restore JPEG power gating state if it was originally gated */ 1723 if (pg_state) 1724 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG, 1725 AMD_PG_STATE_GATE); 1726 mutex_unlock(&adev->jpeg.jpeg_pg_lock); 1727 goto unlock; 1728 } 1729 1730 /* This flag is not set for VF, assumed to be disabled always */ 1731 if (RREG32_SOC15(VCN, GET_INST(VCN, 0), regVCN_RRMT_CNTL) & 0x100) 1732 adev->vcn.caps |= AMDGPU_VCN_CAPS(RRMT_ENABLED); 1733 vcn_v4_0_3_hw_init_inst(vinst); 1734 vcn_v4_0_3_start_dpg_mode(vinst, adev->vcn.inst[ring->me].indirect_sram); 1735 1736 r = amdgpu_ring_reset_helper_end(ring, timedout_fence); 1737 if (r) { 1738 if (pg_state) 1739 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG, 1740 AMD_PG_STATE_GATE); 1741 mutex_unlock(&adev->jpeg.jpeg_pg_lock); 1742 goto unlock; 1743 } 1744 1745 if (pg_state) 1746 amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG, 1747 AMD_PG_STATE_GATE); 1748 mutex_unlock(&adev->jpeg.jpeg_pg_lock); 1749 r = vcn_v4_0_3_reset_jpeg_post_helper(adev, ring->me); 1750 1751 unlock: 1752 mutex_unlock(&vinst->engine_reset_mutex); 1753 1754 return r; 1755 } 1756 1757 static const struct amdgpu_ring_funcs vcn_v4_0_3_unified_ring_vm_funcs = { 1758 .type = AMDGPU_RING_TYPE_VCN_ENC, 1759 .align_mask = 0x3f, 1760 .nop = VCN_ENC_CMD_NO_OP, 1761 .get_rptr = vcn_v4_0_3_unified_ring_get_rptr, 1762 .get_wptr = vcn_v4_0_3_unified_ring_get_wptr, 1763 .set_wptr = vcn_v4_0_3_unified_ring_set_wptr, 1764 .emit_frame_size = 1765 SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + 1766 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + 1767 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ 1768 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ 1769 1, /* vcn_v2_0_enc_ring_insert_end */ 1770 .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ 1771 .emit_ib = vcn_v2_0_enc_ring_emit_ib, 1772 .emit_fence = vcn_v2_0_enc_ring_emit_fence, 1773 .emit_vm_flush = vcn_v4_0_3_enc_ring_emit_vm_flush, 1774 .emit_hdp_flush = vcn_v4_0_3_ring_emit_hdp_flush, 1775 .test_ring = amdgpu_vcn_enc_ring_test_ring, 1776 .test_ib = amdgpu_vcn_unified_ring_test_ib, 1777 .insert_nop = amdgpu_ring_insert_nop, 1778 .insert_end = vcn_v2_0_enc_ring_insert_end, 1779 .pad_ib = amdgpu_ring_generic_pad_ib, 1780 .begin_use = amdgpu_vcn_ring_begin_use, 1781 .end_use = amdgpu_vcn_ring_end_use, 1782 .emit_wreg = vcn_v4_0_3_enc_ring_emit_wreg, 1783 .emit_reg_wait = vcn_v4_0_3_enc_ring_emit_reg_wait, 1784 .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, 1785 .reset = vcn_v4_0_3_ring_reset, 1786 }; 1787 1788 /** 1789 * vcn_v4_0_3_set_unified_ring_funcs - set unified ring functions 1790 * 1791 * @adev: amdgpu_device pointer 1792 * 1793 * Set unified ring functions 1794 */ 1795 static void vcn_v4_0_3_set_unified_ring_funcs(struct amdgpu_device *adev) 1796 { 1797 int i, vcn_inst; 1798 1799 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1800 adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_3_unified_ring_vm_funcs; 1801 adev->vcn.inst[i].ring_enc[0].me = i; 1802 vcn_inst = GET_INST(VCN, i); 1803 adev->vcn.inst[i].aid_id = 1804 vcn_inst / adev->vcn.num_inst_per_aid; 1805 } 1806 } 1807 1808 /** 1809 * vcn_v4_0_3_is_idle - check VCN block is idle 1810 * 1811 * @ip_block: Pointer to the amdgpu_ip_block structure 1812 * 1813 * Check whether VCN block is idle 1814 */ 1815 static bool vcn_v4_0_3_is_idle(struct amdgpu_ip_block *ip_block) 1816 { 1817 struct amdgpu_device *adev = ip_block->adev; 1818 int i, ret = 1; 1819 1820 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1821 ret &= (RREG32_SOC15(VCN, GET_INST(VCN, i), regUVD_STATUS) == 1822 UVD_STATUS__IDLE); 1823 } 1824 1825 return ret; 1826 } 1827 1828 /** 1829 * vcn_v4_0_3_wait_for_idle - wait for VCN block idle 1830 * 1831 * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. 1832 * 1833 * Wait for VCN block idle 1834 */ 1835 static int vcn_v4_0_3_wait_for_idle(struct amdgpu_ip_block *ip_block) 1836 { 1837 struct amdgpu_device *adev = ip_block->adev; 1838 int i, ret = 0; 1839 1840 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1841 ret = SOC15_WAIT_ON_RREG(VCN, GET_INST(VCN, i), regUVD_STATUS, 1842 UVD_STATUS__IDLE, UVD_STATUS__IDLE); 1843 if (ret) 1844 return ret; 1845 } 1846 1847 return ret; 1848 } 1849 1850 /* vcn_v4_0_3_set_clockgating_state - set VCN block clockgating state 1851 * 1852 * @ip_block: amdgpu_ip_block pointer 1853 * @state: clock gating state 1854 * 1855 * Set VCN block clockgating state 1856 */ 1857 static int vcn_v4_0_3_set_clockgating_state(struct amdgpu_ip_block *ip_block, 1858 enum amd_clockgating_state state) 1859 { 1860 struct amdgpu_device *adev = ip_block->adev; 1861 bool enable = state == AMD_CG_STATE_GATE; 1862 int i; 1863 1864 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1865 struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[i]; 1866 1867 if (enable) { 1868 if (RREG32_SOC15(VCN, GET_INST(VCN, i), 1869 regUVD_STATUS) != UVD_STATUS__IDLE) 1870 return -EBUSY; 1871 vcn_v4_0_3_enable_clock_gating(vinst); 1872 } else { 1873 vcn_v4_0_3_disable_clock_gating(vinst); 1874 } 1875 } 1876 return 0; 1877 } 1878 1879 static int vcn_v4_0_3_set_pg_state(struct amdgpu_vcn_inst *vinst, 1880 enum amd_powergating_state state) 1881 { 1882 struct amdgpu_device *adev = vinst->adev; 1883 int ret = 0; 1884 1885 /* for SRIOV, guest should not control VCN Power-gating 1886 * MMSCH FW should control Power-gating and clock-gating 1887 * guest should avoid touching CGC and PG 1888 */ 1889 if (amdgpu_sriov_vf(adev)) { 1890 vinst->cur_state = AMD_PG_STATE_UNGATE; 1891 return 0; 1892 } 1893 1894 if (state == vinst->cur_state) 1895 return 0; 1896 1897 if (state == AMD_PG_STATE_GATE) 1898 ret = vcn_v4_0_3_stop(vinst); 1899 else 1900 ret = vcn_v4_0_3_start(vinst); 1901 1902 if (!ret) 1903 vinst->cur_state = state; 1904 1905 return ret; 1906 } 1907 1908 /** 1909 * vcn_v4_0_3_set_interrupt_state - set VCN block interrupt state 1910 * 1911 * @adev: amdgpu_device pointer 1912 * @source: interrupt sources 1913 * @type: interrupt types 1914 * @state: interrupt states 1915 * 1916 * Set VCN block interrupt state 1917 */ 1918 static int vcn_v4_0_3_set_interrupt_state(struct amdgpu_device *adev, 1919 struct amdgpu_irq_src *source, 1920 unsigned int type, 1921 enum amdgpu_interrupt_state state) 1922 { 1923 return 0; 1924 } 1925 1926 /** 1927 * vcn_v4_0_3_process_interrupt - process VCN block interrupt 1928 * 1929 * @adev: amdgpu_device pointer 1930 * @source: interrupt sources 1931 * @entry: interrupt entry from clients and sources 1932 * 1933 * Process VCN block interrupt 1934 */ 1935 static int vcn_v4_0_3_process_interrupt(struct amdgpu_device *adev, 1936 struct amdgpu_irq_src *source, 1937 struct amdgpu_iv_entry *entry) 1938 { 1939 uint32_t i, inst; 1940 1941 i = node_id_to_phys_map[entry->node_id]; 1942 1943 DRM_DEV_DEBUG(adev->dev, "IH: VCN TRAP\n"); 1944 1945 for (inst = 0; inst < adev->vcn.num_vcn_inst; ++inst) 1946 if (adev->vcn.inst[inst].aid_id == i) 1947 break; 1948 1949 if (inst >= adev->vcn.num_vcn_inst) { 1950 dev_WARN_ONCE(adev->dev, 1, 1951 "Interrupt received for unknown VCN instance %d", 1952 entry->node_id); 1953 return 0; 1954 } 1955 1956 switch (entry->src_id) { 1957 case VCN_4_0__SRCID__UVD_ENC_GENERAL_PURPOSE: 1958 amdgpu_fence_process(&adev->vcn.inst[inst].ring_enc[0]); 1959 break; 1960 default: 1961 DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n", 1962 entry->src_id, entry->src_data[0]); 1963 break; 1964 } 1965 1966 return 0; 1967 } 1968 1969 static int vcn_v4_0_3_set_ras_interrupt_state(struct amdgpu_device *adev, 1970 struct amdgpu_irq_src *source, 1971 unsigned int type, 1972 enum amdgpu_interrupt_state state) 1973 { 1974 return 0; 1975 } 1976 1977 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_irq_funcs = { 1978 .set = vcn_v4_0_3_set_interrupt_state, 1979 .process = vcn_v4_0_3_process_interrupt, 1980 }; 1981 1982 static const struct amdgpu_irq_src_funcs vcn_v4_0_3_ras_irq_funcs = { 1983 .set = vcn_v4_0_3_set_ras_interrupt_state, 1984 .process = amdgpu_vcn_process_poison_irq, 1985 }; 1986 1987 /** 1988 * vcn_v4_0_3_set_irq_funcs - set VCN block interrupt irq functions 1989 * 1990 * @adev: amdgpu_device pointer 1991 * 1992 * Set VCN block interrupt irq functions 1993 */ 1994 static void vcn_v4_0_3_set_irq_funcs(struct amdgpu_device *adev) 1995 { 1996 int i; 1997 1998 for (i = 0; i < adev->vcn.num_vcn_inst; ++i) { 1999 adev->vcn.inst->irq.num_types++; 2000 } 2001 adev->vcn.inst->irq.funcs = &vcn_v4_0_3_irq_funcs; 2002 2003 adev->vcn.inst->ras_poison_irq.num_types = 1; 2004 adev->vcn.inst->ras_poison_irq.funcs = &vcn_v4_0_3_ras_irq_funcs; 2005 } 2006 2007 static const struct amd_ip_funcs vcn_v4_0_3_ip_funcs = { 2008 .name = "vcn_v4_0_3", 2009 .early_init = vcn_v4_0_3_early_init, 2010 .late_init = vcn_v4_0_3_late_init, 2011 .sw_init = vcn_v4_0_3_sw_init, 2012 .sw_fini = vcn_v4_0_3_sw_fini, 2013 .hw_init = vcn_v4_0_3_hw_init, 2014 .hw_fini = vcn_v4_0_3_hw_fini, 2015 .suspend = vcn_v4_0_3_suspend, 2016 .resume = vcn_v4_0_3_resume, 2017 .is_idle = vcn_v4_0_3_is_idle, 2018 .wait_for_idle = vcn_v4_0_3_wait_for_idle, 2019 .set_clockgating_state = vcn_v4_0_3_set_clockgating_state, 2020 .set_powergating_state = vcn_set_powergating_state, 2021 .dump_ip_state = amdgpu_vcn_dump_ip_state, 2022 .print_ip_state = amdgpu_vcn_print_ip_state, 2023 }; 2024 2025 const struct amdgpu_ip_block_version vcn_v4_0_3_ip_block = { 2026 .type = AMD_IP_BLOCK_TYPE_VCN, 2027 .major = 4, 2028 .minor = 0, 2029 .rev = 3, 2030 .funcs = &vcn_v4_0_3_ip_funcs, 2031 }; 2032 2033 static const struct amdgpu_ras_err_status_reg_entry vcn_v4_0_3_ue_reg_list[] = { 2034 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDD, regVCN_UE_ERR_STATUS_HI_VIDD), 2035 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDD"}, 2036 {AMDGPU_RAS_REG_ENTRY(VCN, 0, regVCN_UE_ERR_STATUS_LO_VIDV, regVCN_UE_ERR_STATUS_HI_VIDV), 2037 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "VIDV"}, 2038 }; 2039 2040 static void vcn_v4_0_3_inst_query_ras_error_count(struct amdgpu_device *adev, 2041 uint32_t vcn_inst, 2042 void *ras_err_status) 2043 { 2044 struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; 2045 2046 /* vcn v4_0_3 only support query uncorrectable errors */ 2047 amdgpu_ras_inst_query_ras_error_count(adev, 2048 vcn_v4_0_3_ue_reg_list, 2049 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 2050 NULL, 0, GET_INST(VCN, vcn_inst), 2051 AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 2052 &err_data->ue_count); 2053 } 2054 2055 static void vcn_v4_0_3_query_ras_error_count(struct amdgpu_device *adev, 2056 void *ras_err_status) 2057 { 2058 uint32_t i; 2059 2060 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 2061 dev_warn(adev->dev, "VCN RAS is not supported\n"); 2062 return; 2063 } 2064 2065 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 2066 vcn_v4_0_3_inst_query_ras_error_count(adev, i, ras_err_status); 2067 } 2068 2069 static void vcn_v4_0_3_inst_reset_ras_error_count(struct amdgpu_device *adev, 2070 uint32_t vcn_inst) 2071 { 2072 amdgpu_ras_inst_reset_ras_error_count(adev, 2073 vcn_v4_0_3_ue_reg_list, 2074 ARRAY_SIZE(vcn_v4_0_3_ue_reg_list), 2075 GET_INST(VCN, vcn_inst)); 2076 } 2077 2078 static void vcn_v4_0_3_reset_ras_error_count(struct amdgpu_device *adev) 2079 { 2080 uint32_t i; 2081 2082 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) { 2083 dev_warn(adev->dev, "VCN RAS is not supported\n"); 2084 return; 2085 } 2086 2087 for (i = 0; i < adev->vcn.num_vcn_inst; i++) 2088 vcn_v4_0_3_inst_reset_ras_error_count(adev, i); 2089 } 2090 2091 static uint32_t vcn_v4_0_3_query_poison_by_instance(struct amdgpu_device *adev, 2092 uint32_t instance, uint32_t sub_block) 2093 { 2094 uint32_t poison_stat = 0, reg_value = 0; 2095 2096 switch (sub_block) { 2097 case AMDGPU_VCN_V4_0_3_VCPU_VCODEC: 2098 reg_value = RREG32_SOC15(VCN, instance, regUVD_RAS_VCPU_VCODEC_STATUS); 2099 poison_stat = REG_GET_FIELD(reg_value, UVD_RAS_VCPU_VCODEC_STATUS, POISONED_PF); 2100 break; 2101 default: 2102 break; 2103 } 2104 2105 if (poison_stat) 2106 dev_info(adev->dev, "Poison detected in VCN%d, sub_block%d\n", 2107 instance, sub_block); 2108 2109 return poison_stat; 2110 } 2111 2112 static bool vcn_v4_0_3_query_poison_status(struct amdgpu_device *adev) 2113 { 2114 uint32_t inst, sub; 2115 uint32_t poison_stat = 0; 2116 2117 for (inst = 0; inst < adev->vcn.num_vcn_inst; inst++) 2118 for (sub = 0; sub < AMDGPU_VCN_V4_0_3_MAX_SUB_BLOCK; sub++) 2119 poison_stat += 2120 vcn_v4_0_3_query_poison_by_instance(adev, inst, sub); 2121 2122 return !!poison_stat; 2123 } 2124 2125 static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { 2126 .query_ras_error_count = vcn_v4_0_3_query_ras_error_count, 2127 .reset_ras_error_count = vcn_v4_0_3_reset_ras_error_count, 2128 .query_poison_status = vcn_v4_0_3_query_poison_status, 2129 }; 2130 2131 static int vcn_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, 2132 enum aca_smu_type type, void *data) 2133 { 2134 struct aca_bank_info info; 2135 u64 misc0; 2136 int ret; 2137 2138 ret = aca_bank_info_decode(bank, &info); 2139 if (ret) 2140 return ret; 2141 2142 misc0 = bank->regs[ACA_REG_IDX_MISC0]; 2143 switch (type) { 2144 case ACA_SMU_TYPE_UE: 2145 bank->aca_err_type = ACA_ERROR_TYPE_UE; 2146 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, 2147 1ULL); 2148 break; 2149 case ACA_SMU_TYPE_CE: 2150 bank->aca_err_type = ACA_ERROR_TYPE_CE; 2151 ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 2152 ACA_REG__MISC0__ERRCNT(misc0)); 2153 break; 2154 default: 2155 return -EINVAL; 2156 } 2157 2158 return ret; 2159 } 2160 2161 /* reference to smu driver if header file */ 2162 static int vcn_v4_0_3_err_codes[] = { 2163 14, 15, /* VCN */ 2164 }; 2165 2166 static bool vcn_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, 2167 enum aca_smu_type type, void *data) 2168 { 2169 u32 instlo; 2170 2171 instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); 2172 instlo &= GENMASK(31, 1); 2173 2174 if (instlo != mmSMNAID_AID0_MCA_SMU) 2175 return false; 2176 2177 if (aca_bank_check_error_codes(handle->adev, bank, 2178 vcn_v4_0_3_err_codes, 2179 ARRAY_SIZE(vcn_v4_0_3_err_codes))) 2180 return false; 2181 2182 return true; 2183 } 2184 2185 static const struct aca_bank_ops vcn_v4_0_3_aca_bank_ops = { 2186 .aca_bank_parser = vcn_v4_0_3_aca_bank_parser, 2187 .aca_bank_is_valid = vcn_v4_0_3_aca_bank_is_valid, 2188 }; 2189 2190 static const struct aca_info vcn_v4_0_3_aca_info = { 2191 .hwip = ACA_HWIP_TYPE_SMU, 2192 .mask = ACA_ERROR_UE_MASK, 2193 .bank_ops = &vcn_v4_0_3_aca_bank_ops, 2194 }; 2195 2196 static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) 2197 { 2198 int r; 2199 2200 r = amdgpu_ras_block_late_init(adev, ras_block); 2201 if (r) 2202 return r; 2203 2204 if (amdgpu_ras_is_supported(adev, ras_block->block) && 2205 adev->vcn.inst->ras_poison_irq.funcs) { 2206 r = amdgpu_irq_get(adev, &adev->vcn.inst->ras_poison_irq, 0); 2207 if (r) 2208 goto late_fini; 2209 } 2210 2211 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN, 2212 &vcn_v4_0_3_aca_info, NULL); 2213 if (r) 2214 goto late_fini; 2215 2216 return 0; 2217 2218 late_fini: 2219 amdgpu_ras_block_late_fini(adev, ras_block); 2220 2221 return r; 2222 } 2223 2224 static struct amdgpu_vcn_ras vcn_v4_0_3_ras = { 2225 .ras_block = { 2226 .hw_ops = &vcn_v4_0_3_ras_hw_ops, 2227 .ras_late_init = vcn_v4_0_3_ras_late_init, 2228 }, 2229 }; 2230 2231 static void vcn_v4_0_3_set_ras_funcs(struct amdgpu_device *adev) 2232 { 2233 adev->vcn.ras = &vcn_v4_0_3_ras; 2234 } 2235 2236 static void vcn_v4_0_3_enable_ras(struct amdgpu_device *adev, 2237 int inst_idx, bool indirect) 2238 { 2239 uint32_t tmp; 2240 2241 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__VCN)) 2242 return; 2243 2244 tmp = VCN_RAS_CNTL__VCPU_VCODEC_REARM_MASK | 2245 VCN_RAS_CNTL__VCPU_VCODEC_IH_EN_MASK | 2246 VCN_RAS_CNTL__VCPU_VCODEC_PMI_EN_MASK | 2247 VCN_RAS_CNTL__VCPU_VCODEC_STALL_EN_MASK; 2248 WREG32_SOC15_DPG_MODE(inst_idx, 2249 SOC15_DPG_MODE_OFFSET(VCN, 0, regVCN_RAS_CNTL), 2250 tmp, 0, indirect); 2251 2252 tmp = UVD_VCPU_INT_EN2__RASCNTL_VCPU_VCODEC_EN_MASK; 2253 WREG32_SOC15_DPG_MODE(inst_idx, 2254 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_VCPU_INT_EN2), 2255 tmp, 0, indirect); 2256 2257 tmp = UVD_SYS_INT_EN__RASCNTL_VCPU_VCODEC_EN_MASK; 2258 WREG32_SOC15_DPG_MODE(inst_idx, 2259 SOC15_DPG_MODE_OFFSET(VCN, 0, regUVD_SYS_INT_EN), 2260 tmp, 0, indirect); 2261 } 2262