40 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
43  * navi10_ih_init_register_offset - Initialize register offset for ih rings
49 static void navi10_ih_init_register_offset(struct amdgpu_device *adev)  in navi10_ih_init_register_offset()
53 	if (adev->irq.ih.ring_size) {  in navi10_ih_init_register_offset()
54 		ih_regs = &adev->irq.ih.ih_regs;  in navi10_ih_init_register_offset()
55 		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE);  in navi10_ih_init_register_offset()
56 		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI);  in navi10_ih_init_register_offset()
57 		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);  in navi10_ih_init_register_offset()
58 		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);  in navi10_ih_init_register_offset()
59 		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR);  in navi10_ih_init_register_offset()
60 		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR);  in navi10_ih_init_register_offset()
61 		ih_regs->ih_rb_wptr_addr_lo = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO);  in navi10_ih_init_register_offset()
62 		ih_regs->ih_rb_wptr_addr_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI);  in navi10_ih_init_register_offset()
63 		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL;  in navi10_ih_init_register_offset()
66 	if (adev->irq.ih1.ring_size) {  in navi10_ih_init_register_offset()
67 		ih_regs = &adev->irq.ih1.ih_regs;  in navi10_ih_init_register_offset()
68 		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING1);  in navi10_ih_init_register_offset()
69 		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING1);  in navi10_ih_init_register_offset()
70 		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1);  in navi10_ih_init_register_offset()
71 		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1);  in navi10_ih_init_register_offset()
72 		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1);  in navi10_ih_init_register_offset()
73 		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1);  in navi10_ih_init_register_offset()
74 		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING1;  in navi10_ih_init_register_offset()
77 	if (adev->irq.ih2.ring_size) {  in navi10_ih_init_register_offset()
78 		ih_regs = &adev->irq.ih2.ih_regs;  in navi10_ih_init_register_offset()
79 		ih_regs->ih_rb_base = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_RING2);  in navi10_ih_init_register_offset()
80 		ih_regs->ih_rb_base_hi = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_BASE_HI_RING2);  in navi10_ih_init_register_offset()
81 		ih_regs->ih_rb_cntl = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2);  in navi10_ih_init_register_offset()
82 		ih_regs->ih_rb_wptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2);  in navi10_ih_init_register_offset()
83 		ih_regs->ih_rb_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2);  in navi10_ih_init_register_offset()
84 		ih_regs->ih_doorbell_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2);  in navi10_ih_init_register_offset()
85 		ih_regs->psp_reg_id = PSP_REG_IH_RB_CNTL_RING2;  in navi10_ih_init_register_offset()
90  * force_update_wptr_for_self_int - Force update the wptr for self interrupt
95  * @enabled: Enable/disable timeout flush mechanism
104 static void
124 		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, ih_rb_cntl))  in force_update_wptr_for_self_int()
134 		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2, ih_rb_cntl))  in force_update_wptr_for_self_int()
144  * navi10_ih_toggle_ring_interrupts - toggle the interrupt ring buffer
148  * @enable: true - enable the interrupts, false - disable the interrupts
152 static int navi10_ih_toggle_ring_interrupts(struct amdgpu_device *adev,  in navi10_ih_toggle_ring_interrupts()
154 					    bool enable)  in navi10_ih_toggle_ring_interrupts()  argument
159 	ih_regs = &ih->ih_regs;  in navi10_ih_toggle_ring_interrupts()
161 	tmp = RREG32(ih_regs->ih_rb_cntl);  in navi10_ih_toggle_ring_interrupts()
162 	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0));  in navi10_ih_toggle_ring_interrupts()
165 	if (ih == &adev->irq.ih)  in navi10_ih_toggle_ring_interrupts()
166 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));  in navi10_ih_toggle_ring_interrupts()
169 		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp))  in navi10_ih_toggle_ring_interrupts()
170 			return -ETIMEDOUT;  in navi10_ih_toggle_ring_interrupts()
172 		WREG32(ih_regs->ih_rb_cntl, tmp);  in navi10_ih_toggle_ring_interrupts()
175 	if (enable) {  in navi10_ih_toggle_ring_interrupts()
176 		ih->enabled = true;  in navi10_ih_toggle_ring_interrupts()
179 		WREG32(ih_regs->ih_rb_rptr, 0);  in navi10_ih_toggle_ring_interrupts()
180 		WREG32(ih_regs->ih_rb_wptr, 0);  in navi10_ih_toggle_ring_interrupts()
181 		ih->enabled = false;  in navi10_ih_toggle_ring_interrupts()
182 		ih->rptr = 0;  in navi10_ih_toggle_ring_interrupts()
189  * navi10_ih_toggle_interrupts - Toggle all the available interrupt ring buffers
192  * @enable: enable or disable interrupt ring buffers
196 static int navi10_ih_toggle_interrupts(struct amdgpu_device *adev, bool enable)  in navi10_ih_toggle_interrupts()  argument
198 	struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2};  in navi10_ih_toggle_interrupts()
203 		if (ih[i]->ring_size) {  in navi10_ih_toggle_interrupts()
204 			r = navi10_ih_toggle_ring_interrupts(adev, ih[i], enable);  in navi10_ih_toggle_interrupts()
213 static uint32_t navi10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)  in navi10_ih_rb_cntl()
215 	int rb_bufsz = order_base_2(ih->ring_size / 4);  in navi10_ih_rb_cntl()
218 				   MC_SPACE, ih->use_bus_addr ? 1 : 4);  in navi10_ih_rb_cntl()
236 static uint32_t navi10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)  in navi10_ih_doorbell_rptr()
240 	if (ih->use_doorbell) {  in navi10_ih_doorbell_rptr()
243 						 ih->doorbell_index);  in navi10_ih_doorbell_rptr()
246 						 ENABLE, 1);  in navi10_ih_doorbell_rptr()
250 						 ENABLE, 0);  in navi10_ih_doorbell_rptr()
256  * navi10_ih_enable_ring - enable an ih ring buffer
261  * Enable an ih ring buffer (NAVI10)
263 static int navi10_ih_enable_ring(struct amdgpu_device *adev,  in navi10_ih_enable_ring()
269 	ih_regs = &ih->ih_regs;  in navi10_ih_enable_ring()
271 	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/  in navi10_ih_enable_ring()
272 	WREG32(ih_regs->ih_rb_base, ih->gpu_addr >> 8);  in navi10_ih_enable_ring()
273 	WREG32(ih_regs->ih_rb_base_hi, (ih->gpu_addr >> 40) & 0xff);  in navi10_ih_enable_ring()
275 	tmp = RREG32(ih_regs->ih_rb_cntl);  in navi10_ih_enable_ring()
277 	if (ih == &adev->irq.ih)  in navi10_ih_enable_ring()
278 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RPTR_REARM, !!adev->irq.msi_enabled);  in navi10_ih_enable_ring()
279 	if (ih == &adev->irq.ih1)  in navi10_ih_enable_ring()
283 		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) {  in navi10_ih_enable_ring()
285 			return -ETIMEDOUT;  in navi10_ih_enable_ring()
288 		WREG32(ih_regs->ih_rb_cntl, tmp);  in navi10_ih_enable_ring()
291 	if (ih == &adev->irq.ih) {  in navi10_ih_enable_ring()
293 		WREG32(ih_regs->ih_rb_wptr_addr_lo, lower_32_bits(ih->wptr_addr));  in navi10_ih_enable_ring()
294 		WREG32(ih_regs->ih_rb_wptr_addr_hi, upper_32_bits(ih->wptr_addr) & 0xFFFF);  in navi10_ih_enable_ring()
298 	WREG32(ih_regs->ih_rb_wptr, 0);  in navi10_ih_enable_ring()
299 	WREG32(ih_regs->ih_rb_rptr, 0);  in navi10_ih_enable_ring()
301 	WREG32(ih_regs->ih_doorbell_rptr, navi10_ih_doorbell_rptr(ih));  in navi10_ih_enable_ring()
307  * navi10_ih_irq_init - init and enable the interrupt ring
312  * enable the RLC, disable interrupts, enable the IH
313  * ring buffer and enable it (NAVI).
317 static int navi10_ih_irq_init(struct amdgpu_device *adev)  in navi10_ih_irq_init()
319 	struct amdgpu_ih_ring *ih[] = {&adev->irq.ih, &adev->irq.ih1, &adev->irq.ih2};  in navi10_ih_irq_init()
329 	adev->nbio.funcs->ih_control(adev);  in navi10_ih_irq_init()
331 	if (unlikely(adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT)) {  in navi10_ih_irq_init()
332 		if (ih[0]->use_bus_addr) {  in navi10_ih_irq_init()
353 		if (ih[i]->ring_size) {  in navi10_ih_irq_init()
361 	adev->nbio.funcs->ih_doorbell_range(adev, ih[0]->use_doorbell,  in navi10_ih_irq_init()
362 					    ih[0]->doorbell_index);  in navi10_ih_irq_init()
364 	pci_set_master(adev->pdev);  in navi10_ih_irq_init()
366 	/* enable interrupts */  in navi10_ih_irq_init()
370 	/* enable wptr force update for self int */  in navi10_ih_irq_init()
373 	if (adev->irq.ih_soft.ring_size)  in navi10_ih_irq_init()
374 		adev->irq.ih_soft.enabled = true;  in navi10_ih_irq_init()
380  * navi10_ih_irq_disable - disable interrupts
386 static void navi10_ih_irq_disable(struct amdgpu_device *adev)  in navi10_ih_irq_disable()
396  * navi10_ih_get_wptr - get the IH ring buffer wptr
406 static u32 navi10_ih_get_wptr(struct amdgpu_device *adev,  in navi10_ih_get_wptr()
412 	if (ih == &adev->irq.ih || ih == &adev->irq.ih_soft) {  in navi10_ih_get_wptr()
414 		 * to register-based code with overflow checking below.  in navi10_ih_get_wptr()
418 		wptr = le32_to_cpu(*ih->wptr_cpu);  in navi10_ih_get_wptr()
424 	ih_regs = &ih->ih_regs;  in navi10_ih_get_wptr()
427 	wptr = RREG32_NO_KIQ(ih_regs->ih_rb_wptr);  in navi10_ih_get_wptr()
436 	tmp = (wptr + 32) & ih->ptr_mask;  in navi10_ih_get_wptr()
437 	dev_warn(adev->dev, "%s ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",  in navi10_ih_get_wptr()
438 		 amdgpu_ih_ring_name(adev, ih), wptr, ih->rptr, tmp);  in navi10_ih_get_wptr()
439 	ih->rptr = tmp;  in navi10_ih_get_wptr()
441 	tmp = RREG32_NO_KIQ(ih_regs->ih_rb_cntl);  in navi10_ih_get_wptr()
443 	WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp);  in navi10_ih_get_wptr()
449 	WREG32_NO_KIQ(ih_regs->ih_rb_cntl, tmp);  in navi10_ih_get_wptr()
451 	return (wptr & ih->ptr_mask);  in navi10_ih_get_wptr()
455  * navi10_ih_irq_rearm - rearm IRQ if lost
461 static void navi10_ih_irq_rearm(struct amdgpu_device *adev,  in navi10_ih_irq_rearm()
468 	ih_regs = &ih->ih_regs;  in navi10_ih_irq_rearm()
470 	/* Rearm IRQ / re-write doorbell if doorbell write is lost */  in navi10_ih_irq_rearm()
472 		v = RREG32_NO_KIQ(ih_regs->ih_rb_rptr);  in navi10_ih_irq_rearm()
473 		if ((v < ih->ring_size) && (v != ih->rptr))  in navi10_ih_irq_rearm()
474 			WDOORBELL32(ih->doorbell_index, ih->rptr);  in navi10_ih_irq_rearm()
481  * navi10_ih_set_rptr - set the IH ring buffer rptr
488 static void navi10_ih_set_rptr(struct amdgpu_device *adev,  in navi10_ih_set_rptr()
493 	if (ih == &adev->irq.ih_soft)  in navi10_ih_set_rptr()
496 	if (ih->use_doorbell) {  in navi10_ih_set_rptr()
498 		*ih->rptr_cpu = ih->rptr;  in navi10_ih_set_rptr()
499 		WDOORBELL32(ih->doorbell_index, ih->rptr);  in navi10_ih_set_rptr()
504 		ih_regs = &ih->ih_regs;  in navi10_ih_set_rptr()
505 		WREG32(ih_regs->ih_rb_rptr, ih->rptr);  in navi10_ih_set_rptr()
510  * navi10_ih_self_irq - dispatch work for ring 1 and 2
518 static int navi10_ih_self_irq(struct amdgpu_device *adev,  in navi10_ih_self_irq()
522 	switch (entry->ring_id) {  in navi10_ih_self_irq()
524 		schedule_work(&adev->irq.ih1_work);  in navi10_ih_self_irq()
527 		schedule_work(&adev->irq.ih2_work);  in navi10_ih_self_irq()
534 static const struct amdgpu_irq_src_funcs navi10_ih_self_irq_funcs = {
538 static void navi10_ih_set_self_irq_funcs(struct amdgpu_device *adev)  in navi10_ih_set_self_irq_funcs()
540 	adev->irq.self_irq.num_types = 0;  in navi10_ih_set_self_irq_funcs()
541 	adev->irq.self_irq.funcs = &navi10_ih_self_irq_funcs;  in navi10_ih_set_self_irq_funcs()
544 static int navi10_ih_early_init(struct amdgpu_ip_block *ip_block)  in navi10_ih_early_init()
546 	struct amdgpu_device *adev = ip_block->adev;  in navi10_ih_early_init()
553 static int navi10_ih_sw_init(struct amdgpu_ip_block *ip_block)  in navi10_ih_sw_init()
556 	struct amdgpu_device *adev = ip_block->adev;  in navi10_ih_sw_init()
560 				&adev->irq.self_irq);  in navi10_ih_sw_init()
568 	if ((adev->flags & AMD_IS_APU) ||  in navi10_ih_sw_init()
569 	    (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP))  in navi10_ih_sw_init()
573 	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, IH_RING_SIZE, use_bus_addr);  in navi10_ih_sw_init()
577 	adev->irq.ih.use_doorbell = true;  in navi10_ih_sw_init()
578 	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;  in navi10_ih_sw_init()
580 	adev->irq.ih1.ring_size = 0;  in navi10_ih_sw_init()
581 	adev->irq.ih2.ring_size = 0;  in navi10_ih_sw_init()
586 	r = amdgpu_ih_ring_init(adev, &adev->irq.ih_soft, IH_SW_RING_SIZE, true);  in navi10_ih_sw_init()
595 static int navi10_ih_sw_fini(struct amdgpu_ip_block *ip_block)  in navi10_ih_sw_fini()
597 	struct amdgpu_device *adev = ip_block->adev;  in navi10_ih_sw_fini()
604 static int navi10_ih_hw_init(struct amdgpu_ip_block *ip_block)  in navi10_ih_hw_init()
606 	struct amdgpu_device *adev = ip_block->adev;  in navi10_ih_hw_init()
611 static int navi10_ih_hw_fini(struct amdgpu_ip_block *ip_block)  in navi10_ih_hw_fini()
613 	navi10_ih_irq_disable(ip_block->adev);  in navi10_ih_hw_fini()
618 static int navi10_ih_suspend(struct amdgpu_ip_block *ip_block)  in navi10_ih_suspend()
623 static int navi10_ih_resume(struct amdgpu_ip_block *ip_block)  in navi10_ih_resume()
628 static bool navi10_ih_is_idle(void *handle)  in navi10_ih_is_idle()
634 static int navi10_ih_wait_for_idle(struct amdgpu_ip_block *ip_block)  in navi10_ih_wait_for_idle()
637 	return -ETIMEDOUT;  in navi10_ih_wait_for_idle()
640 static int navi10_ih_soft_reset(struct amdgpu_ip_block *ip_block)  in navi10_ih_soft_reset()
646 static void navi10_ih_update_clockgating_state(struct amdgpu_device *adev,  in navi10_ih_update_clockgating_state()
647 					       bool enable)  in navi10_ih_update_clockgating_state()  argument
651 	if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {  in navi10_ih_update_clockgating_state()
653 		field_val = enable ? 0 : 1;  in navi10_ih_update_clockgating_state()
669 static int navi10_ih_set_clockgating_state(struct amdgpu_ip_block *ip_block,  in navi10_ih_set_clockgating_state()
672 	struct amdgpu_device *adev = ip_block->adev;  in navi10_ih_set_clockgating_state()
679 static int navi10_ih_set_powergating_state(struct amdgpu_ip_block *ip_block,  in navi10_ih_set_powergating_state()
685 static void navi10_ih_get_clockgating_state(void *handle, u64 *flags)  in navi10_ih_get_clockgating_state()
693 static const struct amd_ip_funcs navi10_ih_ip_funcs = {
710 static const struct amdgpu_ih_funcs navi10_ih_funcs = {
717 static void navi10_ih_set_interrupt_funcs(struct amdgpu_device *adev)  in navi10_ih_set_interrupt_funcs()
719 	if (adev->irq.ih_funcs == NULL)  in navi10_ih_set_interrupt_funcs()
720 		adev->irq.ih_funcs = &navi10_ih_funcs;  in navi10_ih_set_interrupt_funcs()