Lines Matching +full:sdm845 +full:- +full:gpi +full:- +full:dma
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
7 #include <dt-bindings/dma/qcom-gpi.h>
9 #include <linux/dma-mapping.h>
14 #include <linux/dma/qcom-gpi-dma.h>
18 #include "../virt-dma.h"
66 /* DMA TRE */
70 /* Register offsets from gpi-top */
183 /* GPII specific Global - Enable bit register */
188 /* GPII general interrupt - Enable bit register */
475 u32 max_gpii; /* maximum # of gpii instances available per gpi block */
502 void __iomem *regs; /* points to gpi top */
554 return ring->phys_addr + (addr - ring->base); in to_physical()
559 return ring->base + (addr - ring->phys_addr); in to_virtual()
575 void __iomem *addr = gpii->regs + offset; in gpi_update_reg()
586 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_disable_interrupts()
588 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_disable_interrupts()
590 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_disable_interrupts()
592 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_disable_interrupts()
594 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id), in gpi_disable_interrupts()
596 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id), in gpi_disable_interrupts()
598 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id), in gpi_disable_interrupts()
601 gpii->cntxt_type_irq_msk = 0; in gpi_disable_interrupts()
602 devm_free_irq(gpii->gpi_dev->dev, gpii->irq, gpii); in gpi_disable_interrupts()
603 gpii->configured_irq = false; in gpi_disable_interrupts()
616 if (!gpii->configured_irq) { in gpi_config_interrupts()
617 ret = devm_request_irq(gpii->gpi_dev->dev, gpii->irq, in gpi_config_interrupts()
619 "gpi-dma", gpii); in gpi_config_interrupts()
621 dev_err(gpii->gpi_dev->dev, "error request irq:%d ret:%d\n", in gpi_config_interrupts()
622 gpii->irq, ret); in gpi_config_interrupts()
633 gpii->cntxt_type_irq_msk |= GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB; in gpi_config_interrupts()
635 gpii->cntxt_type_irq_msk &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB); in gpi_config_interrupts()
636 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_config_interrupts()
637 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, gpii->cntxt_type_irq_msk); in gpi_config_interrupts()
639 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_config_interrupts()
641 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_config_interrupts()
644 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_config_interrupts()
647 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id), in gpi_config_interrupts()
650 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id), in gpi_config_interrupts()
653 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id), in gpi_config_interrupts()
655 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_LSB_OFFS(gpii->gpii_id), U32_MAX, 0); in gpi_config_interrupts()
656 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_MSB_OFFS(gpii->gpii_id), U32_MAX, 0); in gpi_config_interrupts()
657 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_0_OFFS(gpii->gpii_id), U32_MAX, 0); in gpi_config_interrupts()
658 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_1_OFFS(gpii->gpii_id), U32_MAX, 0); in gpi_config_interrupts()
659 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id), in gpi_config_interrupts()
661 gpi_update_reg(gpii, GPII_n_ERROR_LOG_OFFS(gpii->gpii_id), U32_MAX, 0); in gpi_config_interrupts()
663 gpii->cntxt_type_irq_msk = enable; in gpi_config_interrupts()
666 gpii->configured_irq = true; in gpi_config_interrupts()
680 return -EINVAL; in gpi_send_cmd()
682 chid = gchan->chid; in gpi_send_cmd()
684 dev_dbg(gpii->gpi_dev->dev, in gpi_send_cmd()
688 reinit_completion(&gpii->cmd_completion); in gpi_send_cmd()
689 gpii->gpi_cmd = gpi_cmd; in gpi_send_cmd()
691 cmd_reg = IS_CHAN_CMD(gpi_cmd) ? gchan->ch_cmd_reg : gpii->ev_cmd_reg; in gpi_send_cmd()
695 timeout = wait_for_completion_timeout(&gpii->cmd_completion, in gpi_send_cmd()
698 dev_err(gpii->gpi_dev->dev, "cmd: %s completion timeout:%u\n", in gpi_send_cmd()
700 return -EIO; in gpi_send_cmd()
707 if (IS_CHAN_CMD(gpi_cmd) && gchan->ch_state == gpi_cmd_info[gpi_cmd].state) in gpi_send_cmd()
710 if (!IS_CHAN_CMD(gpi_cmd) && gpii->ev_state == gpi_cmd_info[gpi_cmd].state) in gpi_send_cmd()
713 return -EIO; in gpi_send_cmd()
720 struct gpii *gpii = gchan->gpii; in gpi_write_ch_db()
724 gpi_write_reg(gpii, gchan->ch_cntxt_db_reg, p_wp); in gpi_write_ch_db()
733 p_wp = ring->phys_addr + (wp - ring->base); in gpi_write_ev_db()
734 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg, p_wp); in gpi_write_ev_db()
740 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0)); in gpi_process_ieob()
743 tasklet_hi_schedule(&gpii->ev_task); in gpi_process_ieob()
749 u32 gpii_id = gpii->gpii_id; in gpi_process_ch_ctrl_irq()
751 u32 ch_irq = gpi_read_reg(gpii, gpii->regs + offset); in gpi_process_ch_ctrl_irq()
757 gpi_write_reg(gpii, gpii->regs + offset, (u32)ch_irq); in gpi_process_ch_ctrl_irq()
763 gchan = &gpii->gchan[chid]; in gpi_process_ch_ctrl_irq()
764 state = gpi_read_reg(gpii, gchan->ch_cntxt_base_reg + in gpi_process_ch_ctrl_irq()
773 if (gpii->gpi_cmd == GPI_CH_CMD_DE_ALLOC) in gpi_process_ch_ctrl_irq()
775 gchan->ch_state = state; in gpi_process_ch_ctrl_irq()
782 if (gchan->ch_state != CH_STATE_STOP_IN_PROC) in gpi_process_ch_ctrl_irq()
783 complete_all(&gpii->cmd_completion); in gpi_process_ch_ctrl_irq()
787 /* processing gpi general error interrupts */
790 u32 gpii_id = gpii->gpii_id; in gpi_process_gen_err_irq()
792 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset); in gpi_process_gen_err_irq()
795 dev_dbg(gpii->gpi_dev->dev, "irq_stts:0x%x\n", irq_stts); in gpi_process_gen_err_irq()
799 gpi_write_reg(gpii, gpii->regs + offset, irq_stts); in gpi_process_gen_err_irq()
802 /* processing gpi level error interrupts */
805 u32 gpii_id = gpii->gpii_id; in gpi_process_glob_err_irq()
807 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset); in gpi_process_glob_err_irq()
810 gpi_write_reg(gpii, gpii->regs + offset, irq_stts); in gpi_process_glob_err_irq()
814 dev_err(gpii->gpi_dev->dev, "invalid error status:0x%x\n", irq_stts); in gpi_process_glob_err_irq()
819 gpi_write_reg(gpii, gpii->regs + offset, 0); in gpi_process_glob_err_irq()
826 u32 gpii_id = gpii->gpii_id; in gpi_handle_irq()
830 read_lock_irqsave(&gpii->pm_lock, flags); in gpi_handle_irq()
836 if (!REG_ACCESS_VALID(gpii->pm_state)) { in gpi_handle_irq()
837 dev_err(gpii->gpi_dev->dev, "receive interrupt while in %s state\n", in gpi_handle_irq()
838 TO_GPI_PM_STR(gpii->pm_state)); in gpi_handle_irq()
842 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id); in gpi_handle_irq()
843 type = gpi_read_reg(gpii, gpii->regs + offset); in gpi_handle_irq()
863 dev_dbg(gpii->gpi_dev->dev, in gpi_handle_irq()
866 ev_ch_irq = gpi_read_reg(gpii, gpii->regs + offset); in gpi_handle_irq()
870 gpi_write_reg(gpii, gpii->regs + offset, ev_ch_irq); in gpi_handle_irq()
871 ev_state = gpi_read_reg(gpii, gpii->ev_cntxt_base_reg + in gpi_handle_irq()
880 if (gpii->gpi_cmd == GPI_EV_CMD_DEALLOC) in gpi_handle_irq()
883 gpii->ev_state = ev_state; in gpi_handle_irq()
884 dev_dbg(gpii->gpi_dev->dev, "setting EV state to %s\n", in gpi_handle_irq()
885 TO_GPI_EV_STATE_STR(gpii->ev_state)); in gpi_handle_irq()
886 complete_all(&gpii->cmd_completion); in gpi_handle_irq()
892 dev_dbg(gpii->gpi_dev->dev, "process CH CTRL interrupts\n"); in gpi_handle_irq()
898 dev_err(gpii->gpi_dev->dev, "Unhandled interrupt status:0x%x\n", type); in gpi_handle_irq()
903 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id); in gpi_handle_irq()
904 type = gpi_read_reg(gpii, gpii->regs + offset); in gpi_handle_irq()
908 read_unlock_irqrestore(&gpii->pm_lock, flags); in gpi_handle_irq()
913 /* process DMA Immediate completion data events */
917 struct gpii *gpii = gchan->gpii; in gpi_process_imed_data_event()
918 struct gpi_ring *ch_ring = &gchan->ch_ring; in gpi_process_imed_data_event()
919 void *tre = ch_ring->base + (ch_ring->el_size * imed_event->tre_index); in gpi_process_imed_data_event()
929 if (gchan->pm_state != ACTIVE_STATE) { in gpi_process_imed_data_event()
930 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n", in gpi_process_imed_data_event()
931 TO_GPI_PM_STR(gchan->pm_state)); in gpi_process_imed_data_event()
935 spin_lock_irqsave(&gchan->vc.lock, flags); in gpi_process_imed_data_event()
936 vd = vchan_next_desc(&gchan->vc); in gpi_process_imed_data_event()
941 spin_unlock_irqrestore(&gchan->vc.lock, flags); in gpi_process_imed_data_event()
942 dev_dbg(gpii->gpi_dev->dev, "event without a pending descriptor!\n"); in gpi_process_imed_data_event()
944 dev_dbg(gpii->gpi_dev->dev, in gpi_process_imed_data_event()
946 gpi_ere->dword[0], gpi_ere->dword[1], in gpi_process_imed_data_event()
947 gpi_ere->dword[2], gpi_ere->dword[3]); in gpi_process_imed_data_event()
949 dev_dbg(gpii->gpi_dev->dev, in gpi_process_imed_data_event()
951 gpi_tre->dword[0], gpi_tre->dword[1], in gpi_process_imed_data_event()
952 gpi_tre->dword[2], gpi_tre->dword[3]); in gpi_process_imed_data_event()
956 spin_unlock_irqrestore(&gchan->vc.lock, flags); in gpi_process_imed_data_event()
962 tre += ch_ring->el_size; in gpi_process_imed_data_event()
963 if (tre >= (ch_ring->base + ch_ring->len)) in gpi_process_imed_data_event()
964 tre = ch_ring->base; in gpi_process_imed_data_event()
965 ch_ring->rp = tre; in gpi_process_imed_data_event()
970 chid = imed_event->chid; in gpi_process_imed_data_event()
971 if (imed_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) { in gpi_process_imed_data_event()
978 if (imed_event->code == MSM_GPI_TCE_UNEXP_ERR) in gpi_process_imed_data_event()
982 result.residue = gpi_desc->len - imed_event->length; in gpi_process_imed_data_event()
984 dma_cookie_complete(&vd->tx); in gpi_process_imed_data_event()
985 dmaengine_desc_get_callback_invoke(&vd->tx, &result); in gpi_process_imed_data_event()
988 spin_lock_irqsave(&gchan->vc.lock, flags); in gpi_process_imed_data_event()
989 list_del(&vd->node); in gpi_process_imed_data_event()
990 spin_unlock_irqrestore(&gchan->vc.lock, flags); in gpi_process_imed_data_event()
999 struct gpii *gpii = gchan->gpii; in gpi_process_xfer_compl_event()
1000 struct gpi_ring *ch_ring = &gchan->ch_ring; in gpi_process_xfer_compl_event()
1001 void *ev_rp = to_virtual(ch_ring, compl_event->ptr); in gpi_process_xfer_compl_event()
1009 if (unlikely(gchan->pm_state != ACTIVE_STATE)) { in gpi_process_xfer_compl_event()
1010 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n", in gpi_process_xfer_compl_event()
1011 TO_GPI_PM_STR(gchan->pm_state)); in gpi_process_xfer_compl_event()
1015 spin_lock_irqsave(&gchan->vc.lock, flags); in gpi_process_xfer_compl_event()
1016 vd = vchan_next_desc(&gchan->vc); in gpi_process_xfer_compl_event()
1020 spin_unlock_irqrestore(&gchan->vc.lock, flags); in gpi_process_xfer_compl_event()
1021 dev_err(gpii->gpi_dev->dev, "Event without a pending descriptor!\n"); in gpi_process_xfer_compl_event()
1023 dev_err(gpii->gpi_dev->dev, in gpi_process_xfer_compl_event()
1025 gpi_ere->dword[0], gpi_ere->dword[1], in gpi_process_xfer_compl_event()
1026 gpi_ere->dword[2], gpi_ere->dword[3]); in gpi_process_xfer_compl_event()
1031 spin_unlock_irqrestore(&gchan->vc.lock, flags); in gpi_process_xfer_compl_event()
1037 ev_rp += ch_ring->el_size; in gpi_process_xfer_compl_event()
1038 if (ev_rp >= (ch_ring->base + ch_ring->len)) in gpi_process_xfer_compl_event()
1039 ev_rp = ch_ring->base; in gpi_process_xfer_compl_event()
1040 ch_ring->rp = ev_rp; in gpi_process_xfer_compl_event()
1045 chid = compl_event->chid; in gpi_process_xfer_compl_event()
1046 if (compl_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) { in gpi_process_xfer_compl_event()
1053 if (compl_event->code == MSM_GPI_TCE_UNEXP_ERR) { in gpi_process_xfer_compl_event()
1054 dev_err(gpii->gpi_dev->dev, "Error in Transaction\n"); in gpi_process_xfer_compl_event()
1057 dev_dbg(gpii->gpi_dev->dev, "Transaction Success\n"); in gpi_process_xfer_compl_event()
1060 result.residue = gpi_desc->len - compl_event->length; in gpi_process_xfer_compl_event()
1061 dev_dbg(gpii->gpi_dev->dev, "Residue %d\n", result.residue); in gpi_process_xfer_compl_event()
1063 dma_cookie_complete(&vd->tx); in gpi_process_xfer_compl_event()
1064 dmaengine_desc_get_callback_invoke(&vd->tx, &result); in gpi_process_xfer_compl_event()
1067 spin_lock_irqsave(&gchan->vc.lock, flags); in gpi_process_xfer_compl_event()
1068 list_del(&vd->node); in gpi_process_xfer_compl_event()
1069 spin_unlock_irqrestore(&gchan->vc.lock, flags); in gpi_process_xfer_compl_event()
1077 struct gpi_ring *ev_ring = &gpii->ev_ring; in gpi_process_events()
1084 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); in gpi_process_events()
1088 while (rp != ev_ring->rp) { in gpi_process_events()
1089 gpi_event = ev_ring->rp; in gpi_process_events()
1090 chid = gpi_event->xfer_compl_event.chid; in gpi_process_events()
1091 type = gpi_event->xfer_compl_event.type; in gpi_process_events()
1093 dev_dbg(gpii->gpi_dev->dev, in gpi_process_events()
1095 chid, type, gpi_event->gpi_ere.dword[0], in gpi_process_events()
1096 gpi_event->gpi_ere.dword[1], gpi_event->gpi_ere.dword[2], in gpi_process_events()
1097 gpi_event->gpi_ere.dword[3]); in gpi_process_events()
1101 gchan = &gpii->gchan[chid]; in gpi_process_events()
1103 &gpi_event->xfer_compl_event); in gpi_process_events()
1106 dev_dbg(gpii->gpi_dev->dev, "stale event, not processing\n"); in gpi_process_events()
1109 gchan = &gpii->gchan[chid]; in gpi_process_events()
1111 &gpi_event->immediate_data_event); in gpi_process_events()
1114 dev_dbg(gpii->gpi_dev->dev, "QUP_NOTIF_EV_TYPE\n"); in gpi_process_events()
1117 dev_dbg(gpii->gpi_dev->dev, in gpi_process_events()
1122 gpi_write_ev_db(gpii, ev_ring, ev_ring->wp); in gpi_process_events()
1125 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0)); in gpi_process_events()
1127 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); in gpi_process_events()
1130 } while (rp != ev_ring->rp); in gpi_process_events()
1138 read_lock(&gpii->pm_lock); in gpi_ev_tasklet()
1139 if (!REG_ACCESS_VALID(gpii->pm_state)) { in gpi_ev_tasklet()
1140 read_unlock(&gpii->pm_lock); in gpi_ev_tasklet()
1141 dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n", in gpi_ev_tasklet()
1142 TO_GPI_PM_STR(gpii->pm_state)); in gpi_ev_tasklet()
1151 read_unlock(&gpii->pm_lock); in gpi_ev_tasklet()
1157 struct gpii *gpii = gchan->gpii; in gpi_mark_stale_events()
1158 struct gpi_ring *ev_ring = &gpii->ev_ring; in gpi_mark_stale_events()
1162 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); in gpi_mark_stale_events()
1164 ev_rp = ev_ring->rp; in gpi_mark_stale_events()
1168 u32 chid = gpi_event->xfer_compl_event.chid; in gpi_mark_stale_events()
1170 if (chid == gchan->chid) in gpi_mark_stale_events()
1171 gpi_event->xfer_compl_event.type = STALE_EV_TYPE; in gpi_mark_stale_events()
1172 ev_rp += ev_ring->el_size; in gpi_mark_stale_events()
1173 if (ev_rp >= (ev_ring->base + ev_ring->len)) in gpi_mark_stale_events()
1174 ev_rp = ev_ring->base; in gpi_mark_stale_events()
1175 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg); in gpi_mark_stale_events()
1180 /* reset sw state and issue channel reset or de-alloc */
1183 struct gpii *gpii = gchan->gpii; in gpi_reset_chan()
1184 struct gpi_ring *ch_ring = &gchan->ch_ring; in gpi_reset_chan()
1190 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", in gpi_reset_chan()
1196 ch_ring->rp = ch_ring->base; in gpi_reset_chan()
1197 ch_ring->wp = ch_ring->base; in gpi_reset_chan()
1203 write_lock_irq(&gpii->pm_lock); in gpi_reset_chan()
1207 spin_lock(&gchan->vc.lock); in gpi_reset_chan()
1208 vchan_get_all_descriptors(&gchan->vc, &list); in gpi_reset_chan()
1209 spin_unlock(&gchan->vc.lock); in gpi_reset_chan()
1210 write_unlock_irq(&gpii->pm_lock); in gpi_reset_chan()
1211 vchan_dma_desc_free_list(&gchan->vc, &list); in gpi_reset_chan()
1218 struct gpii *gpii = gchan->gpii; in gpi_start_chan()
1223 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", in gpi_start_chan()
1229 write_lock_irq(&gpii->pm_lock); in gpi_start_chan()
1230 gchan->pm_state = ACTIVE_STATE; in gpi_start_chan()
1231 write_unlock_irq(&gpii->pm_lock); in gpi_start_chan()
1238 struct gpii *gpii = gchan->gpii; in gpi_stop_chan()
1243 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", in gpi_stop_chan()
1254 struct gpii *gpii = chan->gpii; in gpi_alloc_chan()
1255 struct gpi_ring *ring = &chan->ch_ring; in gpi_alloc_chan()
1257 u32 id = gpii->gpii_id; in gpi_alloc_chan()
1258 u32 chid = chan->chid; in gpi_alloc_chan()
1264 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n", in gpi_alloc_chan()
1270 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_0_CONFIG, in gpi_alloc_chan()
1271 GPII_n_CH_k_CNTXT_0(ring->el_size, 0, chan->dir, GPI_CHTYPE_PROTO_GPI)); in gpi_alloc_chan()
1272 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_1_R_LENGTH, ring->len); in gpi_alloc_chan()
1273 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_2_RING_BASE_LSB, ring->phys_addr); in gpi_alloc_chan()
1274 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_3_RING_BASE_MSB, in gpi_alloc_chan()
1275 upper_32_bits(ring->phys_addr)); in gpi_alloc_chan()
1276 gpi_write_reg(gpii, chan->ch_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB, in gpi_alloc_chan()
1277 upper_32_bits(ring->phys_addr)); in gpi_alloc_chan()
1278 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_0_OFFS(id, chid), in gpi_alloc_chan()
1279 GPII_n_CH_k_SCRATCH_0(pair_chid, chan->protocol, chan->seid)); in gpi_alloc_chan()
1280 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_1_OFFS(id, chid), 0); in gpi_alloc_chan()
1281 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_2_OFFS(id, chid), 0); in gpi_alloc_chan()
1282 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_3_OFFS(id, chid), 0); in gpi_alloc_chan()
1283 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_QOS_OFFS(id, chid), 1); in gpi_alloc_chan()
1293 struct gpi_ring *ring = &gpii->ev_ring; in gpi_alloc_ev_chan()
1294 void __iomem *base = gpii->ev_cntxt_base_reg; in gpi_alloc_ev_chan()
1299 dev_err(gpii->gpi_dev->dev, "error with cmd:%s ret:%d\n", in gpi_alloc_ev_chan()
1306 GPII_n_EV_k_CNTXT_0(ring->el_size, GPI_INTTYPE_IRQ, GPI_CHTYPE_GPI_EV)); in gpi_alloc_ev_chan()
1307 gpi_write_reg(gpii, base + CNTXT_1_R_LENGTH, ring->len); in gpi_alloc_ev_chan()
1308 gpi_write_reg(gpii, base + CNTXT_2_RING_BASE_LSB, lower_32_bits(ring->phys_addr)); in gpi_alloc_ev_chan()
1309 gpi_write_reg(gpii, base + CNTXT_3_RING_BASE_MSB, upper_32_bits(ring->phys_addr)); in gpi_alloc_ev_chan()
1310 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB, in gpi_alloc_ev_chan()
1311 upper_32_bits(ring->phys_addr)); in gpi_alloc_ev_chan()
1320 ring->wp = (ring->base + ring->len - ring->el_size); in gpi_alloc_ev_chan()
1326 write_lock_irq(&gpii->pm_lock); in gpi_alloc_ev_chan()
1327 gpii->pm_state = ACTIVE_STATE; in gpi_alloc_ev_chan()
1328 write_unlock_irq(&gpii->pm_lock); in gpi_alloc_ev_chan()
1329 gpi_write_ev_db(gpii, ring, ring->wp); in gpi_alloc_ev_chan()
1339 if (ring->wp < ring->rp) { in gpi_ring_num_elements_avail()
1340 elements = ((ring->rp - ring->wp) / ring->el_size) - 1; in gpi_ring_num_elements_avail()
1342 elements = (ring->rp - ring->base) / ring->el_size; in gpi_ring_num_elements_avail()
1343 elements += ((ring->base + ring->len - ring->wp) / ring->el_size) - 1; in gpi_ring_num_elements_avail()
1352 return -ENOMEM; in gpi_ring_add_element()
1354 *wp = ring->wp; in gpi_ring_add_element()
1355 ring->wp += ring->el_size; in gpi_ring_add_element()
1356 if (ring->wp >= (ring->base + ring->len)) in gpi_ring_add_element()
1357 ring->wp = ring->base; in gpi_ring_add_element()
1368 ring->wp += ring->el_size; in gpi_ring_recycle_ev_element()
1369 if (ring->wp >= (ring->base + ring->len)) in gpi_ring_recycle_ev_element()
1370 ring->wp = ring->base; in gpi_ring_recycle_ev_element()
1373 ring->rp += ring->el_size; in gpi_ring_recycle_ev_element()
1374 if (ring->rp >= (ring->base + ring->len)) in gpi_ring_recycle_ev_element()
1375 ring->rp = ring->base; in gpi_ring_recycle_ev_element()
1384 dma_free_coherent(gpii->gpi_dev->dev, ring->alloc_size, in gpi_free_ring()
1385 ring->pre_aligned, ring->dma_handle); in gpi_free_ring()
1398 if (((1 << bit) - 1) & len) in gpi_alloc_ring()
1401 ring->alloc_size = (len + (len - 1)); in gpi_alloc_ring()
1402 dev_dbg(gpii->gpi_dev->dev, in gpi_alloc_ring()
1405 ring->alloc_size); in gpi_alloc_ring()
1407 ring->pre_aligned = dma_alloc_coherent(gpii->gpi_dev->dev, in gpi_alloc_ring()
1408 ring->alloc_size, in gpi_alloc_ring()
1409 &ring->dma_handle, GFP_KERNEL); in gpi_alloc_ring()
1410 if (!ring->pre_aligned) { in gpi_alloc_ring()
1411 dev_err(gpii->gpi_dev->dev, "could not alloc size:%zu mem for ring\n", in gpi_alloc_ring()
1412 ring->alloc_size); in gpi_alloc_ring()
1413 return -ENOMEM; in gpi_alloc_ring()
1417 ring->phys_addr = (ring->dma_handle + (len - 1)) & ~(len - 1); in gpi_alloc_ring()
1418 ring->base = ring->pre_aligned + (ring->phys_addr - ring->dma_handle); in gpi_alloc_ring()
1419 ring->rp = ring->base; in gpi_alloc_ring()
1420 ring->wp = ring->base; in gpi_alloc_ring()
1421 ring->len = len; in gpi_alloc_ring()
1422 ring->el_size = el_size; in gpi_alloc_ring()
1423 ring->elements = ring->len / ring->el_size; in gpi_alloc_ring()
1424 memset(ring->base, 0, ring->len); in gpi_alloc_ring()
1425 ring->configured = true; in gpi_alloc_ring()
1430 dev_dbg(gpii->gpi_dev->dev, in gpi_alloc_ring()
1432 &ring->dma_handle, &ring->phys_addr, ring->len, in gpi_alloc_ring()
1433 ring->el_size, ring->elements); in gpi_alloc_ring()
1446 ret = gpi_ring_add_element(&gchan->ch_ring, (void **)&ch_tre); in gpi_queue_xfer()
1448 dev_err(gpii->gpi_dev->dev, "Error adding ring element to xfer ring\n"); in gpi_queue_xfer()
1461 struct gpii *gpii = gchan->gpii; in gpi_terminate_all()
1465 mutex_lock(&gpii->ctrl_lock); in gpi_terminate_all()
1471 schid = (gchan->protocol == QCOM_GPI_UART) ? gchan->chid : 0; in gpi_terminate_all()
1472 echid = (gchan->protocol == QCOM_GPI_UART) ? schid + 1 : MAX_CHANNELS_PER_GPII; in gpi_terminate_all()
1476 gchan = &gpii->gchan[i]; in gpi_terminate_all()
1479 write_lock_irq(&gpii->pm_lock); in gpi_terminate_all()
1480 gchan->pm_state = PREPARE_TERMINATE; in gpi_terminate_all()
1481 write_unlock_irq(&gpii->pm_lock); in gpi_terminate_all()
1489 gchan = &gpii->gchan[i]; in gpi_terminate_all()
1493 dev_err(gpii->gpi_dev->dev, "Error resetting channel ret:%d\n", ret); in gpi_terminate_all()
1500 dev_err(gpii->gpi_dev->dev, "Error alloc_channel ret:%d\n", ret); in gpi_terminate_all()
1507 gchan = &gpii->gchan[i]; in gpi_terminate_all()
1511 dev_err(gpii->gpi_dev->dev, "Error Starting Channel ret:%d\n", ret); in gpi_terminate_all()
1517 mutex_unlock(&gpii->ctrl_lock); in gpi_terminate_all()
1521 /* pause dma transfer for all channels */
1525 struct gpii *gpii = gchan->gpii; in gpi_pause()
1528 mutex_lock(&gpii->ctrl_lock); in gpi_pause()
1534 if (gpii->pm_state == PAUSE_STATE) { in gpi_pause()
1535 dev_dbg(gpii->gpi_dev->dev, "channel is already paused\n"); in gpi_pause()
1536 mutex_unlock(&gpii->ctrl_lock); in gpi_pause()
1542 ret = gpi_stop_chan(&gpii->gchan[i]); in gpi_pause()
1544 mutex_unlock(&gpii->ctrl_lock); in gpi_pause()
1549 disable_irq(gpii->irq); in gpi_pause()
1552 tasklet_kill(&gpii->ev_task); in gpi_pause()
1554 write_lock_irq(&gpii->pm_lock); in gpi_pause()
1555 gpii->pm_state = PAUSE_STATE; in gpi_pause()
1556 write_unlock_irq(&gpii->pm_lock); in gpi_pause()
1557 mutex_unlock(&gpii->ctrl_lock); in gpi_pause()
1562 /* resume dma transfer */
1566 struct gpii *gpii = gchan->gpii; in gpi_resume()
1569 mutex_lock(&gpii->ctrl_lock); in gpi_resume()
1570 if (gpii->pm_state == ACTIVE_STATE) { in gpi_resume()
1571 dev_dbg(gpii->gpi_dev->dev, "channel is already active\n"); in gpi_resume()
1572 mutex_unlock(&gpii->ctrl_lock); in gpi_resume()
1576 enable_irq(gpii->irq); in gpi_resume()
1580 ret = gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_START); in gpi_resume()
1582 dev_err(gpii->gpi_dev->dev, "Error starting chan, ret:%d\n", ret); in gpi_resume()
1583 mutex_unlock(&gpii->ctrl_lock); in gpi_resume()
1588 write_lock_irq(&gpii->pm_lock); in gpi_resume()
1589 gpii->pm_state = ACTIVE_STATE; in gpi_resume()
1590 write_unlock_irq(&gpii->pm_lock); in gpi_resume()
1591 mutex_unlock(&gpii->ctrl_lock); in gpi_resume()
1609 if (!config->peripheral_config) in gpi_peripheral_config()
1610 return -EINVAL; in gpi_peripheral_config()
1612 gchan->config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT); in gpi_peripheral_config()
1613 if (!gchan->config) in gpi_peripheral_config()
1614 return -ENOMEM; in gpi_peripheral_config()
1616 memcpy(gchan->config, config->peripheral_config, config->peripheral_size); in gpi_peripheral_config()
1624 struct gpi_i2c_config *i2c = chan->config; in gpi_create_i2c_tre()
1625 struct device *dev = chan->gpii->gpi_dev->dev; in gpi_create_i2c_tre()
1632 if (i2c->set_config) { in gpi_create_i2c_tre()
1633 tre = &desc->tre[tre_idx]; in gpi_create_i2c_tre()
1636 tre->dword[0] = u32_encode_bits(i2c->low_count, TRE_I2C_C0_TLOW); in gpi_create_i2c_tre()
1637 tre->dword[0] |= u32_encode_bits(i2c->high_count, TRE_I2C_C0_THIGH); in gpi_create_i2c_tre()
1638 tre->dword[0] |= u32_encode_bits(i2c->cycle_count, TRE_I2C_C0_TCYL); in gpi_create_i2c_tre()
1639 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_TX_PACK); in gpi_create_i2c_tre()
1640 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_RX_PACK); in gpi_create_i2c_tre()
1642 tre->dword[1] = 0; in gpi_create_i2c_tre()
1644 tre->dword[2] = u32_encode_bits(i2c->clk_div, TRE_C0_CLK_DIV); in gpi_create_i2c_tre()
1646 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE); in gpi_create_i2c_tre()
1647 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); in gpi_create_i2c_tre()
1651 if (i2c->op == I2C_WRITE) { in gpi_create_i2c_tre()
1652 tre = &desc->tre[tre_idx]; in gpi_create_i2c_tre()
1655 if (i2c->multi_msg) in gpi_create_i2c_tre()
1656 tre->dword[0] = u32_encode_bits(I2C_READ, TRE_I2C_GO_CMD); in gpi_create_i2c_tre()
1658 tre->dword[0] = u32_encode_bits(i2c->op, TRE_I2C_GO_CMD); in gpi_create_i2c_tre()
1660 tre->dword[0] |= u32_encode_bits(i2c->addr, TRE_I2C_GO_ADDR); in gpi_create_i2c_tre()
1661 tre->dword[0] |= u32_encode_bits(i2c->stretch, TRE_I2C_GO_STRETCH); in gpi_create_i2c_tre()
1663 tre->dword[1] = 0; in gpi_create_i2c_tre()
1664 tre->dword[2] = u32_encode_bits(i2c->rx_len, TRE_RX_LEN); in gpi_create_i2c_tre()
1666 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE); in gpi_create_i2c_tre()
1668 if (i2c->multi_msg) in gpi_create_i2c_tre()
1669 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK); in gpi_create_i2c_tre()
1671 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); in gpi_create_i2c_tre()
1674 if (i2c->op == I2C_READ || i2c->multi_msg == false) { in gpi_create_i2c_tre()
1675 /* create the DMA TRE */ in gpi_create_i2c_tre()
1676 tre = &desc->tre[tre_idx]; in gpi_create_i2c_tre()
1680 tre->dword[0] = lower_32_bits(address); in gpi_create_i2c_tre()
1681 tre->dword[1] = upper_32_bits(address); in gpi_create_i2c_tre()
1683 tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN); in gpi_create_i2c_tre()
1685 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); in gpi_create_i2c_tre()
1686 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT); in gpi_create_i2c_tre()
1690 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], in gpi_create_i2c_tre()
1691 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]); in gpi_create_i2c_tre()
1699 struct gpi_spi_config *spi = chan->config; in gpi_create_spi_tre()
1700 struct device *dev = chan->gpii->gpi_dev->dev; in gpi_create_spi_tre()
1708 if (direction == DMA_MEM_TO_DEV && spi->set_config) { in gpi_create_spi_tre()
1709 tre = &desc->tre[tre_idx]; in gpi_create_spi_tre()
1712 tre->dword[0] = u32_encode_bits(spi->word_len, TRE_SPI_C0_WORD_SZ); in gpi_create_spi_tre()
1713 tre->dword[0] |= u32_encode_bits(spi->loopback_en, TRE_SPI_C0_LOOPBACK); in gpi_create_spi_tre()
1714 tre->dword[0] |= u32_encode_bits(spi->clock_pol_high, TRE_SPI_C0_CPOL); in gpi_create_spi_tre()
1715 tre->dword[0] |= u32_encode_bits(spi->data_pol_high, TRE_SPI_C0_CPHA); in gpi_create_spi_tre()
1716 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_TX_PACK); in gpi_create_spi_tre()
1717 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_RX_PACK); in gpi_create_spi_tre()
1719 tre->dword[1] = 0; in gpi_create_spi_tre()
1721 tre->dword[2] = u32_encode_bits(spi->clk_div, TRE_C0_CLK_DIV); in gpi_create_spi_tre()
1722 tre->dword[2] |= u32_encode_bits(spi->clk_src, TRE_C0_CLK_SRC); in gpi_create_spi_tre()
1724 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE); in gpi_create_spi_tre()
1725 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); in gpi_create_spi_tre()
1730 tre = &desc->tre[tre_idx]; in gpi_create_spi_tre()
1733 tre->dword[0] = u32_encode_bits(spi->fragmentation, TRE_SPI_GO_FRAG); in gpi_create_spi_tre()
1734 tre->dword[0] |= u32_encode_bits(spi->cs, TRE_SPI_GO_CS); in gpi_create_spi_tre()
1735 tre->dword[0] |= u32_encode_bits(spi->cmd, TRE_SPI_GO_CMD); in gpi_create_spi_tre()
1737 tre->dword[1] = 0; in gpi_create_spi_tre()
1739 tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN); in gpi_create_spi_tre()
1741 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE); in gpi_create_spi_tre()
1742 if (spi->cmd == SPI_RX) { in gpi_create_spi_tre()
1743 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB); in gpi_create_spi_tre()
1744 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK); in gpi_create_spi_tre()
1745 } else if (spi->cmd == SPI_TX) { in gpi_create_spi_tre()
1746 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); in gpi_create_spi_tre()
1748 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN); in gpi_create_spi_tre()
1749 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK); in gpi_create_spi_tre()
1753 /* create the dma tre */ in gpi_create_spi_tre()
1754 tre = &desc->tre[tre_idx]; in gpi_create_spi_tre()
1760 /* Support Immediate dma for write transfers for data length up to 8 bytes */ in gpi_create_spi_tre()
1761 if (direction == DMA_MEM_TO_DEV && len <= 2 * sizeof(tre->dword[0])) { in gpi_create_spi_tre()
1763 * For Immediate dma, data length may not always be length of 8 bytes, in gpi_create_spi_tre()
1766 tre->dword[0] = 0; in gpi_create_spi_tre()
1767 tre->dword[1] = 0; in gpi_create_spi_tre()
1768 memcpy(&tre->dword[0], sg_virt(sgl), len); in gpi_create_spi_tre()
1770 tre->dword[2] = u32_encode_bits(len, TRE_DMA_IMMEDIATE_LEN); in gpi_create_spi_tre()
1771 tre->dword[3] = u32_encode_bits(TRE_TYPE_IMMEDIATE_DMA, TRE_FLAGS_TYPE); in gpi_create_spi_tre()
1773 tre->dword[0] = lower_32_bits(address); in gpi_create_spi_tre()
1774 tre->dword[1] = upper_32_bits(address); in gpi_create_spi_tre()
1776 tre->dword[2] = u32_encode_bits(len, TRE_DMA_LEN); in gpi_create_spi_tre()
1777 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE); in gpi_create_spi_tre()
1780 tre->dword[3] |= u32_encode_bits(direction == DMA_MEM_TO_DEV, in gpi_create_spi_tre()
1784 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0], in gpi_create_spi_tre()
1785 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]); in gpi_create_spi_tre()
1797 struct gpii *gpii = gchan->gpii; in gpi_prep_slave_sg()
1798 struct device *dev = gpii->gpi_dev->dev; in gpi_prep_slave_sg()
1799 struct gpi_ring *ch_ring = &gchan->ch_ring; in gpi_prep_slave_sg()
1805 gpii->ieob_set = false; in gpi_prep_slave_sg()
1807 dev_err(gpii->gpi_dev->dev, "invalid dma direction: %d\n", direction); in gpi_prep_slave_sg()
1817 set_config = *(u32 *)gchan->config; in gpi_prep_slave_sg()
1835 if (gchan->protocol == QCOM_GPI_SPI) { in gpi_prep_slave_sg()
1837 } else if (gchan->protocol == QCOM_GPI_I2C) { in gpi_prep_slave_sg()
1840 dev_err(dev, "invalid peripheral: %d\n", gchan->protocol); in gpi_prep_slave_sg()
1846 gpi_desc->gchan = gchan; in gpi_prep_slave_sg()
1847 gpi_desc->len = sg_dma_len(sgl); in gpi_prep_slave_sg()
1848 gpi_desc->num_tre = i; in gpi_prep_slave_sg()
1850 return vchan_tx_prep(&gchan->vc, &gpi_desc->vd, flags); in gpi_prep_slave_sg()
1857 struct gpii *gpii = gchan->gpii; in gpi_issue_pending()
1861 struct gpi_ring *ch_ring = &gchan->ch_ring; in gpi_issue_pending()
1865 read_lock_irqsave(&gpii->pm_lock, pm_lock_flags); in gpi_issue_pending()
1868 spin_lock_irqsave(&gchan->vc.lock, flags); in gpi_issue_pending()
1869 if (vchan_issue_pending(&gchan->vc)) in gpi_issue_pending()
1870 vd = list_last_entry(&gchan->vc.desc_issued, in gpi_issue_pending()
1872 spin_unlock_irqrestore(&gchan->vc.lock, flags); in gpi_issue_pending()
1876 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags); in gpi_issue_pending()
1881 for (i = 0; i < gpi_desc->num_tre; i++) { in gpi_issue_pending()
1882 tre = &gpi_desc->tre[i]; in gpi_issue_pending()
1886 gpi_desc->db = ch_ring->wp; in gpi_issue_pending()
1887 gpi_write_ch_db(gchan, &gchan->ch_ring, gpi_desc->db); in gpi_issue_pending()
1888 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags); in gpi_issue_pending()
1893 struct gpii *gpii = gchan->gpii; in gpi_ch_init()
1894 const int ev_factor = gpii->gpi_dev->ev_factor; in gpi_ch_init()
1898 gchan->pm_state = CONFIG_STATE; in gpi_ch_init()
1902 if (gpii->gchan[i].pm_state != CONFIG_STATE) in gpi_ch_init()
1906 if (gpii->gchan[0].protocol != gpii->gchan[1].protocol) { in gpi_ch_init()
1907 dev_err(gpii->gpi_dev->dev, "protocol did not match protocol %u != %u\n", in gpi_ch_init()
1908 gpii->gchan[0].protocol, gpii->gchan[1].protocol); in gpi_ch_init()
1909 ret = -EINVAL; in gpi_ch_init()
1915 ret = gpi_alloc_ring(&gpii->ev_ring, elements, in gpi_ch_init()
1921 write_lock_irq(&gpii->pm_lock); in gpi_ch_init()
1922 gpii->pm_state = PREPARE_HARDWARE; in gpi_ch_init()
1923 write_unlock_irq(&gpii->pm_lock); in gpi_ch_init()
1926 dev_err(gpii->gpi_dev->dev, "error config. interrupts, ret:%d\n", ret); in gpi_ch_init()
1933 dev_err(gpii->gpi_dev->dev, "error alloc_ev_chan:%d\n", ret); in gpi_ch_init()
1939 ret = gpi_alloc_chan(&gpii->gchan[i], true); in gpi_ch_init()
1941 dev_err(gpii->gpi_dev->dev, "Error allocating chan:%d\n", ret); in gpi_ch_init()
1948 ret = gpi_start_chan(&gpii->gchan[i]); in gpi_ch_init()
1950 dev_err(gpii->gpi_dev->dev, "Error start chan:%d\n", ret); in gpi_ch_init()
1957 for (i = i - 1; i >= 0; i--) { in gpi_ch_init()
1958 gpi_stop_chan(&gpii->gchan[i]); in gpi_ch_init()
1963 for (i = i - 1; i >= 0; i--) in gpi_ch_init()
1968 gpi_free_ring(&gpii->ev_ring, gpii); in gpi_ch_init()
1977 struct gpii *gpii = gchan->gpii; in gpi_free_chan_resources()
1981 mutex_lock(&gpii->ctrl_lock); in gpi_free_chan_resources()
1983 cur_state = gchan->pm_state; in gpi_free_chan_resources()
1986 write_lock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
1987 gchan->pm_state = PREPARE_TERMINATE; in gpi_free_chan_resources()
1988 write_unlock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
1996 dev_err(gpii->gpi_dev->dev, "error resetting channel:%d\n", ret); in gpi_free_chan_resources()
2002 gpi_free_ring(&gchan->ch_ring, gpii); in gpi_free_chan_resources()
2003 vchan_free_chan_resources(&gchan->vc); in gpi_free_chan_resources()
2004 kfree(gchan->config); in gpi_free_chan_resources()
2006 write_lock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
2007 gchan->pm_state = DISABLE_STATE; in gpi_free_chan_resources()
2008 write_unlock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
2012 if (gpii->gchan[i].ch_ring.configured) in gpi_free_chan_resources()
2016 cur_state = gpii->pm_state; in gpi_free_chan_resources()
2017 write_lock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
2018 gpii->pm_state = PREPARE_TERMINATE; in gpi_free_chan_resources()
2019 write_unlock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
2022 tasklet_kill(&gpii->ev_task); in gpi_free_chan_resources()
2028 gpi_free_ring(&gpii->ev_ring, gpii); in gpi_free_chan_resources()
2035 write_lock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
2036 gpii->pm_state = DISABLE_STATE; in gpi_free_chan_resources()
2037 write_unlock_irq(&gpii->pm_lock); in gpi_free_chan_resources()
2040 mutex_unlock(&gpii->ctrl_lock); in gpi_free_chan_resources()
2047 struct gpii *gpii = gchan->gpii; in gpi_alloc_chan_resources()
2050 mutex_lock(&gpii->ctrl_lock); in gpi_alloc_chan_resources()
2053 ret = gpi_alloc_ring(&gchan->ch_ring, CHAN_TRES, in gpi_alloc_chan_resources()
2060 mutex_unlock(&gpii->ctrl_lock); in gpi_alloc_chan_resources()
2064 mutex_unlock(&gpii->ctrl_lock); in gpi_alloc_chan_resources()
2075 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) { in gpi_find_avail_gpii()
2076 if (!((1 << gpii) & gpi_dev->gpii_mask)) in gpi_find_avail_gpii()
2079 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN]; in gpi_find_avail_gpii()
2080 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN]; in gpi_find_avail_gpii()
2082 if (rx_chan->vc.chan.client_count && rx_chan->seid == seid) in gpi_find_avail_gpii()
2084 if (tx_chan->vc.chan.client_count && tx_chan->seid == seid) in gpi_find_avail_gpii()
2089 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) { in gpi_find_avail_gpii()
2090 if (!((1 << gpii) & gpi_dev->gpii_mask)) in gpi_find_avail_gpii()
2093 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN]; in gpi_find_avail_gpii()
2094 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN]; in gpi_find_avail_gpii()
2097 if (tx_chan->vc.chan.client_count || in gpi_find_avail_gpii()
2098 rx_chan->vc.chan.client_count) in gpi_find_avail_gpii()
2106 return -EIO; in gpi_find_avail_gpii()
2113 struct gpi_dev *gpi_dev = (struct gpi_dev *)of_dma->of_dma_data; in gpi_of_dma_xlate()
2118 if (args->args_count < 3) { in gpi_of_dma_xlate()
2119 dev_err(gpi_dev->dev, "gpii require minimum 2 args, client passed:%d args\n", in gpi_of_dma_xlate()
2120 args->args_count); in gpi_of_dma_xlate()
2124 chid = args->args[0]; in gpi_of_dma_xlate()
2126 dev_err(gpi_dev->dev, "gpii channel:%d not valid\n", chid); in gpi_of_dma_xlate()
2130 seid = args->args[1]; in gpi_of_dma_xlate()
2135 dev_err(gpi_dev->dev, "no available gpii instances\n"); in gpi_of_dma_xlate()
2139 gchan = &gpi_dev->gpiis[gpii].gchan[chid]; in gpi_of_dma_xlate()
2140 if (gchan->vc.chan.client_count) { in gpi_of_dma_xlate()
2141 dev_err(gpi_dev->dev, "gpii:%d chid:%d seid:%d already configured\n", in gpi_of_dma_xlate()
2142 gpii, chid, gchan->seid); in gpi_of_dma_xlate()
2146 gchan->seid = seid; in gpi_of_dma_xlate()
2147 gchan->protocol = args->args[2]; in gpi_of_dma_xlate()
2149 return dma_get_slave_channel(&gchan->vc.chan); in gpi_of_dma_xlate()
2159 gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL); in gpi_probe()
2161 return -ENOMEM; in gpi_probe()
2163 gpi_dev->dev = &pdev->dev; in gpi_probe()
2164 gpi_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &gpi_dev->res); in gpi_probe()
2165 if (IS_ERR(gpi_dev->regs)) in gpi_probe()
2166 return PTR_ERR(gpi_dev->regs); in gpi_probe()
2167 gpi_dev->ee_base = gpi_dev->regs; in gpi_probe()
2169 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channels", in gpi_probe()
2170 &gpi_dev->max_gpii); in gpi_probe()
2172 dev_err(gpi_dev->dev, "missing 'max-no-gpii' DT node\n"); in gpi_probe()
2176 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channel-mask", in gpi_probe()
2177 &gpi_dev->gpii_mask); in gpi_probe()
2179 dev_err(gpi_dev->dev, "missing 'gpii-mask' DT node\n"); in gpi_probe()
2183 ee_offset = (uintptr_t)device_get_match_data(gpi_dev->dev); in gpi_probe()
2184 gpi_dev->ee_base = gpi_dev->ee_base - ee_offset; in gpi_probe()
2186 gpi_dev->ev_factor = EV_FACTOR; in gpi_probe()
2188 ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64)); in gpi_probe()
2190 dev_err(gpi_dev->dev, "Error setting dma_mask to 64, ret:%d\n", ret); in gpi_probe()
2194 gpi_dev->gpiis = devm_kzalloc(gpi_dev->dev, sizeof(*gpi_dev->gpiis) * in gpi_probe()
2195 gpi_dev->max_gpii, GFP_KERNEL); in gpi_probe()
2196 if (!gpi_dev->gpiis) in gpi_probe()
2197 return -ENOMEM; in gpi_probe()
2200 INIT_LIST_HEAD(&gpi_dev->dma_device.channels); in gpi_probe()
2201 for (i = 0; i < gpi_dev->max_gpii; i++) { in gpi_probe()
2202 struct gpii *gpii = &gpi_dev->gpiis[i]; in gpi_probe()
2205 if (!((1 << i) & gpi_dev->gpii_mask)) in gpi_probe()
2209 gpii->ev_cntxt_base_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_CNTXT_0_OFFS(i, 0); in gpi_probe()
2210 gpii->ev_cntxt_db_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_DOORBELL_0_OFFS(i, 0); in gpi_probe()
2211 gpii->ev_ring_rp_lsb_reg = gpii->ev_cntxt_base_reg + CNTXT_4_RING_RP_LSB; in gpi_probe()
2212 gpii->ev_cmd_reg = gpi_dev->ee_base + GPII_n_EV_CH_CMD_OFFS(i); in gpi_probe()
2213 gpii->ieob_clr_reg = gpi_dev->ee_base + GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(i); in gpi_probe()
2219 gpii->irq = ret; in gpi_probe()
2223 struct gchan *gchan = &gpii->gchan[chan]; in gpi_probe()
2226 gchan->ch_cntxt_base_reg = gpi_dev->ee_base + in gpi_probe()
2228 gchan->ch_cntxt_db_reg = gpi_dev->ee_base + in gpi_probe()
2230 gchan->ch_cmd_reg = gpi_dev->ee_base + GPII_n_CH_CMD_OFFS(i); in gpi_probe()
2233 vchan_init(&gchan->vc, &gpi_dev->dma_device); in gpi_probe()
2234 gchan->vc.desc_free = gpi_desc_free; in gpi_probe()
2235 gchan->chid = chan; in gpi_probe()
2236 gchan->gpii = gpii; in gpi_probe()
2237 gchan->dir = GPII_CHAN_DIR[chan]; in gpi_probe()
2239 mutex_init(&gpii->ctrl_lock); in gpi_probe()
2240 rwlock_init(&gpii->pm_lock); in gpi_probe()
2241 tasklet_init(&gpii->ev_task, gpi_ev_tasklet, in gpi_probe()
2243 init_completion(&gpii->cmd_completion); in gpi_probe()
2244 gpii->gpii_id = i; in gpi_probe()
2245 gpii->regs = gpi_dev->ee_base; in gpi_probe()
2246 gpii->gpi_dev = gpi_dev; in gpi_probe()
2252 dma_cap_zero(gpi_dev->dma_device.cap_mask); in gpi_probe()
2253 dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask); in gpi_probe()
2256 gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in gpi_probe()
2257 gpi_dev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; in gpi_probe()
2258 gpi_dev->dma_device.src_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES; in gpi_probe()
2259 gpi_dev->dma_device.dst_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES; in gpi_probe()
2260 gpi_dev->dma_device.device_alloc_chan_resources = gpi_alloc_chan_resources; in gpi_probe()
2261 gpi_dev->dma_device.device_free_chan_resources = gpi_free_chan_resources; in gpi_probe()
2262 gpi_dev->dma_device.device_tx_status = dma_cookie_status; in gpi_probe()
2263 gpi_dev->dma_device.device_issue_pending = gpi_issue_pending; in gpi_probe()
2264 gpi_dev->dma_device.device_prep_slave_sg = gpi_prep_slave_sg; in gpi_probe()
2265 gpi_dev->dma_device.device_config = gpi_peripheral_config; in gpi_probe()
2266 gpi_dev->dma_device.device_terminate_all = gpi_terminate_all; in gpi_probe()
2267 gpi_dev->dma_device.dev = gpi_dev->dev; in gpi_probe()
2268 gpi_dev->dma_device.device_pause = gpi_pause; in gpi_probe()
2269 gpi_dev->dma_device.device_resume = gpi_resume; in gpi_probe()
2272 ret = dma_async_device_register(&gpi_dev->dma_device); in gpi_probe()
2274 dev_err(gpi_dev->dev, "async_device_register failed ret:%d", ret); in gpi_probe()
2278 ret = of_dma_controller_register(gpi_dev->dev->of_node, in gpi_probe()
2281 dev_err(gpi_dev->dev, "of_dma_controller_reg failed ret:%d", ret); in gpi_probe()
2289 { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 },
2290 { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 },
2293 * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma
2296 { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 },
2297 { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 },
2298 { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 },
2299 { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },
2300 { .compatible = "qcom,sm8450-gpi-dma", .data = (void *)0x10000 },
2319 MODULE_DESCRIPTION("QCOM GPI DMA engine driver");