Lines Matching +full:no +full:- +full:wp
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2017-2020, The Linux Foundation. All rights reserved.
7 #include <dt-bindings/dma/qcom-gpi.h>
9 #include <linux/dma-mapping.h>
14 #include <linux/dma/qcom-gpi-dma.h>
18 #include "../virt-dma.h"
70 /* Register offsets from gpi-top */
183 /* GPII specific Global - Enable bit register */
188 /* GPII general interrupt - Enable bit register */
360 * @DISABLE_STATE: no register access allowed
363 * however, no processing EVENTS
461 void *wp;
538 static int gpi_ring_add_element(struct gpi_ring *ring, void **wp);
554 return ring->phys_addr + (addr - ring->base);
559 return ring->base + (addr - ring->phys_addr);
572 /* gpi_write_reg_field - write to specific bit field */
586 void __iomem *addr = gpii->regs + offset;
597 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
599 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),
601 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),
603 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),
605 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),
607 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),
609 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),
612 gpii->cntxt_type_irq_msk = 0;
613 devm_free_irq(gpii->gpi_dev->dev, gpii->irq, gpii);
614 gpii->configured_irq = false;
627 if (!gpii->configured_irq) {
628 ret = devm_request_irq(gpii->gpi_dev->dev, gpii->irq,
630 "gpi-dma", gpii);
632 dev_err(gpii->gpi_dev->dev, "error request irq:%d ret:%d\n",
633 gpii->irq, ret);
644 gpii->cntxt_type_irq_msk |= GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB;
646 gpii->cntxt_type_irq_msk &= ~(GPII_n_CNTXT_TYPE_IRQ_MSK_IEOB);
647 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
648 GPII_n_CNTXT_TYPE_IRQ_MSK_BMSK, gpii->cntxt_type_irq_msk);
650 gpi_update_reg(gpii, GPII_n_CNTXT_TYPE_IRQ_MSK_OFFS(gpii->gpii_id),
652 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_IEOB_IRQ_MSK_OFFS(gpii->gpii_id),
655 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_CH_IRQ_MSK_OFFS(gpii->gpii_id),
658 gpi_update_reg(gpii, GPII_n_CNTXT_SRC_EV_CH_IRQ_MSK_OFFS(gpii->gpii_id),
661 gpi_update_reg(gpii, GPII_n_CNTXT_GLOB_IRQ_EN_OFFS(gpii->gpii_id),
664 gpi_update_reg(gpii, GPII_n_CNTXT_GPII_IRQ_EN_OFFS(gpii->gpii_id),
666 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_LSB_OFFS(gpii->gpii_id), U32_MAX, 0);
667 gpi_update_reg(gpii, GPII_n_CNTXT_MSI_BASE_MSB_OFFS(gpii->gpii_id), U32_MAX, 0);
668 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_0_OFFS(gpii->gpii_id), U32_MAX, 0);
669 gpi_update_reg(gpii, GPII_n_CNTXT_SCRATCH_1_OFFS(gpii->gpii_id), U32_MAX, 0);
670 gpi_update_reg(gpii, GPII_n_CNTXT_INTSET_OFFS(gpii->gpii_id),
672 gpi_update_reg(gpii, GPII_n_ERROR_LOG_OFFS(gpii->gpii_id), U32_MAX, 0);
674 gpii->cntxt_type_irq_msk = enable;
677 gpii->configured_irq = true;
691 return -EINVAL;
693 chid = gchan->chid;
695 dev_dbg(gpii->gpi_dev->dev,
699 reinit_completion(&gpii->cmd_completion);
700 gpii->gpi_cmd = gpi_cmd;
702 cmd_reg = IS_CHAN_CMD(gpi_cmd) ? gchan->ch_cmd_reg : gpii->ev_cmd_reg;
706 timeout = wait_for_completion_timeout(&gpii->cmd_completion,
709 dev_err(gpii->gpi_dev->dev, "cmd: %s completion timeout:%u\n",
711 return -EIO;
718 if (IS_CHAN_CMD(gpi_cmd) && gchan->ch_state == gpi_cmd_info[gpi_cmd].state)
721 if (!IS_CHAN_CMD(gpi_cmd) && gpii->ev_state == gpi_cmd_info[gpi_cmd].state)
724 return -EIO;
729 struct gpi_ring *ring, void *wp)
731 struct gpii *gpii = gchan->gpii;
734 p_wp = to_physical(ring, wp);
735 gpi_write_reg(gpii, gchan->ch_cntxt_db_reg, p_wp);
740 struct gpi_ring *ring, void *wp)
744 p_wp = ring->phys_addr + (wp - ring->base);
745 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg, p_wp);
751 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));
754 tasklet_hi_schedule(&gpii->ev_task);
760 u32 gpii_id = gpii->gpii_id;
762 u32 ch_irq = gpi_read_reg(gpii, gpii->regs + offset);
768 gpi_write_reg(gpii, gpii->regs + offset, (u32)ch_irq);
774 gchan = &gpii->gchan[chid];
775 state = gpi_read_reg(gpii, gchan->ch_cntxt_base_reg +
784 if (gpii->gpi_cmd == GPI_CH_CMD_DE_ALLOC)
786 gchan->ch_state = state;
793 if (gchan->ch_state != CH_STATE_STOP_IN_PROC)
794 complete_all(&gpii->cmd_completion);
801 u32 gpii_id = gpii->gpii_id;
803 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);
806 dev_dbg(gpii->gpi_dev->dev, "irq_stts:0x%x\n", irq_stts);
810 gpi_write_reg(gpii, gpii->regs + offset, irq_stts);
816 u32 gpii_id = gpii->gpii_id;
818 u32 irq_stts = gpi_read_reg(gpii, gpii->regs + offset);
821 gpi_write_reg(gpii, gpii->regs + offset, irq_stts);
825 dev_err(gpii->gpi_dev->dev, "invalid error status:0x%x\n", irq_stts);
830 gpi_write_reg(gpii, gpii->regs + offset, 0);
837 u32 gpii_id = gpii->gpii_id;
841 read_lock_irqsave(&gpii->pm_lock, flags);
847 if (!REG_ACCESS_VALID(gpii->pm_state)) {
848 dev_err(gpii->gpi_dev->dev, "receive interrupt while in %s state\n",
849 TO_GPI_PM_STR(gpii->pm_state));
853 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);
854 type = gpi_read_reg(gpii, gpii->regs + offset);
874 dev_dbg(gpii->gpi_dev->dev,
877 ev_ch_irq = gpi_read_reg(gpii, gpii->regs + offset);
881 gpi_write_reg(gpii, gpii->regs + offset, ev_ch_irq);
882 ev_state = gpi_read_reg(gpii, gpii->ev_cntxt_base_reg +
891 if (gpii->gpi_cmd == GPI_EV_CMD_DEALLOC)
894 gpii->ev_state = ev_state;
895 dev_dbg(gpii->gpi_dev->dev, "setting EV state to %s\n",
896 TO_GPI_EV_STATE_STR(gpii->ev_state));
897 complete_all(&gpii->cmd_completion);
903 dev_dbg(gpii->gpi_dev->dev, "process CH CTRL interrupts\n");
909 dev_err(gpii->gpi_dev->dev, "Unhandled interrupt status:0x%x\n", type);
914 offset = GPII_n_CNTXT_TYPE_IRQ_OFFS(gpii->gpii_id);
915 type = gpi_read_reg(gpii, gpii->regs + offset);
919 read_unlock_irqrestore(&gpii->pm_lock, flags);
928 struct gpii *gpii = gchan->gpii;
929 struct gpi_ring *ch_ring = &gchan->ch_ring;
930 void *tre = ch_ring->base + (ch_ring->el_size * imed_event->tre_index);
940 if (gchan->pm_state != ACTIVE_STATE) {
941 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",
942 TO_GPI_PM_STR(gchan->pm_state));
946 spin_lock_irqsave(&gchan->vc.lock, flags);
947 vd = vchan_next_desc(&gchan->vc);
952 spin_unlock_irqrestore(&gchan->vc.lock, flags);
953 dev_dbg(gpii->gpi_dev->dev, "event without a pending descriptor!\n");
955 dev_dbg(gpii->gpi_dev->dev,
957 gpi_ere->dword[0], gpi_ere->dword[1],
958 gpi_ere->dword[2], gpi_ere->dword[3]);
960 dev_dbg(gpii->gpi_dev->dev,
962 gpi_tre->dword[0], gpi_tre->dword[1],
963 gpi_tre->dword[2], gpi_tre->dword[3]);
967 spin_unlock_irqrestore(&gchan->vc.lock, flags);
973 tre += ch_ring->el_size;
974 if (tre >= (ch_ring->base + ch_ring->len))
975 tre = ch_ring->base;
976 ch_ring->rp = tre;
981 chid = imed_event->chid;
982 if (imed_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {
989 if (imed_event->code == MSM_GPI_TCE_UNEXP_ERR)
993 result.residue = gpi_desc->len - imed_event->length;
995 dma_cookie_complete(&vd->tx);
996 dmaengine_desc_get_callback_invoke(&vd->tx, &result);
999 spin_lock_irqsave(&gchan->vc.lock, flags);
1000 list_del(&vd->node);
1001 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1010 struct gpii *gpii = gchan->gpii;
1011 struct gpi_ring *ch_ring = &gchan->ch_ring;
1012 void *ev_rp = to_virtual(ch_ring, compl_event->ptr);
1020 if (unlikely(gchan->pm_state != ACTIVE_STATE)) {
1021 dev_err(gpii->gpi_dev->dev, "skipping processing event because ch @ %s state\n",
1022 TO_GPI_PM_STR(gchan->pm_state));
1026 spin_lock_irqsave(&gchan->vc.lock, flags);
1027 vd = vchan_next_desc(&gchan->vc);
1031 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1032 dev_err(gpii->gpi_dev->dev, "Event without a pending descriptor!\n");
1034 dev_err(gpii->gpi_dev->dev,
1036 gpi_ere->dword[0], gpi_ere->dword[1],
1037 gpi_ere->dword[2], gpi_ere->dword[3]);
1042 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1048 ev_rp += ch_ring->el_size;
1049 if (ev_rp >= (ch_ring->base + ch_ring->len))
1050 ev_rp = ch_ring->base;
1051 ch_ring->rp = ev_rp;
1056 chid = compl_event->chid;
1057 if (compl_event->code == MSM_GPI_TCE_EOT && gpii->ieob_set) {
1064 if (compl_event->code == MSM_GPI_TCE_UNEXP_ERR) {
1065 dev_err(gpii->gpi_dev->dev, "Error in Transaction\n");
1068 dev_dbg(gpii->gpi_dev->dev, "Transaction Success\n");
1071 result.residue = gpi_desc->len - compl_event->length;
1072 dev_dbg(gpii->gpi_dev->dev, "Residue %d\n", result.residue);
1074 dma_cookie_complete(&vd->tx);
1075 dmaengine_desc_get_callback_invoke(&vd->tx, &result);
1078 spin_lock_irqsave(&gchan->vc.lock, flags);
1079 list_del(&vd->node);
1080 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1088 struct gpi_ring *ev_ring = &gpii->ev_ring;
1095 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1099 while (rp != ev_ring->rp) {
1100 gpi_event = ev_ring->rp;
1101 chid = gpi_event->xfer_compl_event.chid;
1102 type = gpi_event->xfer_compl_event.type;
1104 dev_dbg(gpii->gpi_dev->dev,
1106 chid, type, gpi_event->gpi_ere.dword[0],
1107 gpi_event->gpi_ere.dword[1], gpi_event->gpi_ere.dword[2],
1108 gpi_event->gpi_ere.dword[3]);
1112 gchan = &gpii->gchan[chid];
1114 &gpi_event->xfer_compl_event);
1117 dev_dbg(gpii->gpi_dev->dev, "stale event, not processing\n");
1120 gchan = &gpii->gchan[chid];
1122 &gpi_event->immediate_data_event);
1125 dev_dbg(gpii->gpi_dev->dev, "QUP_NOTIF_EV_TYPE\n");
1128 dev_dbg(gpii->gpi_dev->dev,
1133 gpi_write_ev_db(gpii, ev_ring, ev_ring->wp);
1136 gpi_write_reg(gpii, gpii->ieob_clr_reg, BIT(0));
1138 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1141 } while (rp != ev_ring->rp);
1149 read_lock(&gpii->pm_lock);
1150 if (!REG_ACCESS_VALID(gpii->pm_state)) {
1151 read_unlock(&gpii->pm_lock);
1152 dev_err(gpii->gpi_dev->dev, "not processing any events, pm_state:%s\n",
1153 TO_GPI_PM_STR(gpii->pm_state));
1162 read_unlock(&gpii->pm_lock);
1168 struct gpii *gpii = gchan->gpii;
1169 struct gpi_ring *ev_ring = &gpii->ev_ring;
1173 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1175 ev_rp = ev_ring->rp;
1179 u32 chid = gpi_event->xfer_compl_event.chid;
1181 if (chid == gchan->chid)
1182 gpi_event->xfer_compl_event.type = STALE_EV_TYPE;
1183 ev_rp += ev_ring->el_size;
1184 if (ev_rp >= (ev_ring->base + ev_ring->len))
1185 ev_rp = ev_ring->base;
1186 cntxt_rp = gpi_read_reg(gpii, gpii->ev_ring_rp_lsb_reg);
1191 /* reset sw state and issue channel reset or de-alloc */
1194 struct gpii *gpii = gchan->gpii;
1195 struct gpi_ring *ch_ring = &gchan->ch_ring;
1201 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1207 ch_ring->rp = ch_ring->base;
1208 ch_ring->wp = ch_ring->base;
1214 write_lock_irq(&gpii->pm_lock);
1218 spin_lock(&gchan->vc.lock);
1219 vchan_get_all_descriptors(&gchan->vc, &list);
1220 spin_unlock(&gchan->vc.lock);
1221 write_unlock_irq(&gpii->pm_lock);
1222 vchan_dma_desc_free_list(&gchan->vc, &list);
1229 struct gpii *gpii = gchan->gpii;
1234 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1240 write_lock_irq(&gpii->pm_lock);
1241 gchan->pm_state = ACTIVE_STATE;
1242 write_unlock_irq(&gpii->pm_lock);
1249 struct gpii *gpii = gchan->gpii;
1254 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1265 struct gpii *gpii = chan->gpii;
1266 struct gpi_ring *ring = &chan->ch_ring;
1268 u32 id = gpii->gpii_id;
1269 u32 chid = chan->chid;
1275 dev_err(gpii->gpi_dev->dev, "Error with cmd:%s ret:%d\n",
1281 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_0_CONFIG,
1282 GPII_n_CH_k_CNTXT_0(ring->el_size, 0, chan->dir, GPI_CHTYPE_PROTO_GPI));
1283 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_1_R_LENGTH, ring->len);
1284 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_2_RING_BASE_LSB, ring->phys_addr);
1285 gpi_write_reg(gpii, chan->ch_cntxt_base_reg + CNTXT_3_RING_BASE_MSB,
1286 upper_32_bits(ring->phys_addr));
1287 gpi_write_reg(gpii, chan->ch_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,
1288 upper_32_bits(ring->phys_addr));
1289 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_0_OFFS(id, chid),
1290 GPII_n_CH_k_SCRATCH_0(pair_chid, chan->protocol, chan->seid));
1291 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_1_OFFS(id, chid), 0);
1292 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_2_OFFS(id, chid), 0);
1293 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_SCRATCH_3_OFFS(id, chid), 0);
1294 gpi_write_reg(gpii, gpii->regs + GPII_n_CH_k_QOS_OFFS(id, chid), 1);
1304 struct gpi_ring *ring = &gpii->ev_ring;
1305 void __iomem *base = gpii->ev_cntxt_base_reg;
1310 dev_err(gpii->gpi_dev->dev, "error with cmd:%s ret:%d\n",
1317 GPII_n_EV_k_CNTXT_0(ring->el_size, GPI_INTTYPE_IRQ, GPI_CHTYPE_GPI_EV));
1318 gpi_write_reg(gpii, base + CNTXT_1_R_LENGTH, ring->len);
1319 gpi_write_reg(gpii, base + CNTXT_2_RING_BASE_LSB, lower_32_bits(ring->phys_addr));
1320 gpi_write_reg(gpii, base + CNTXT_3_RING_BASE_MSB, upper_32_bits(ring->phys_addr));
1321 gpi_write_reg(gpii, gpii->ev_cntxt_db_reg + CNTXT_5_RING_RP_MSB - CNTXT_4_RING_RP_LSB,
1322 upper_32_bits(ring->phys_addr));
1331 ring->wp = (ring->base + ring->len - ring->el_size);
1337 write_lock_irq(&gpii->pm_lock);
1338 gpii->pm_state = ACTIVE_STATE;
1339 write_unlock_irq(&gpii->pm_lock);
1340 gpi_write_ev_db(gpii, ring, ring->wp);
1350 if (ring->wp < ring->rp) {
1351 elements = ((ring->rp - ring->wp) / ring->el_size) - 1;
1353 elements = (ring->rp - ring->base) / ring->el_size;
1354 elements += ((ring->base + ring->len - ring->wp) / ring->el_size) - 1;
1360 static int gpi_ring_add_element(struct gpi_ring *ring, void **wp)
1363 return -ENOMEM;
1365 *wp = ring->wp;
1366 ring->wp += ring->el_size;
1367 if (ring->wp >= (ring->base + ring->len))
1368 ring->wp = ring->base;
1378 /* Update the WP */
1379 ring->wp += ring->el_size;
1380 if (ring->wp >= (ring->base + ring->len))
1381 ring->wp = ring->base;
1384 ring->rp += ring->el_size;
1385 if (ring->rp >= (ring->base + ring->len))
1386 ring->rp = ring->base;
1395 dma_free_coherent(gpii->gpi_dev->dev, ring->alloc_size,
1396 ring->pre_aligned, ring->dma_handle);
1409 if (((1 << bit) - 1) & len)
1412 ring->alloc_size = (len + (len - 1));
1413 dev_dbg(gpii->gpi_dev->dev,
1416 ring->alloc_size);
1418 ring->pre_aligned = dma_alloc_coherent(gpii->gpi_dev->dev,
1419 ring->alloc_size,
1420 &ring->dma_handle, GFP_KERNEL);
1421 if (!ring->pre_aligned) {
1422 dev_err(gpii->gpi_dev->dev, "could not alloc size:%zu mem for ring\n",
1423 ring->alloc_size);
1424 return -ENOMEM;
1428 ring->phys_addr = (ring->dma_handle + (len - 1)) & ~(len - 1);
1429 ring->base = ring->pre_aligned + (ring->phys_addr - ring->dma_handle);
1430 ring->rp = ring->base;
1431 ring->wp = ring->base;
1432 ring->len = len;
1433 ring->el_size = el_size;
1434 ring->elements = ring->len / ring->el_size;
1435 memset(ring->base, 0, ring->len);
1436 ring->configured = true;
1441 dev_dbg(gpii->gpi_dev->dev,
1443 &ring->dma_handle, &ring->phys_addr, ring->len,
1444 ring->el_size, ring->elements);
1451 struct gpi_tre *gpi_tre, void **wp)
1457 ret = gpi_ring_add_element(&gchan->ch_ring, (void **)&ch_tre);
1459 dev_err(gpii->gpi_dev->dev, "Error adding ring element to xfer ring\n");
1465 *wp = ch_tre;
1472 struct gpii *gpii = gchan->gpii;
1476 mutex_lock(&gpii->ctrl_lock);
1482 schid = (gchan->protocol == QCOM_GPI_UART) ? gchan->chid : 0;
1483 echid = (gchan->protocol == QCOM_GPI_UART) ? schid + 1 : MAX_CHANNELS_PER_GPII;
1487 gchan = &gpii->gchan[i];
1489 /* disable ch state so no more TRE processing */
1490 write_lock_irq(&gpii->pm_lock);
1491 gchan->pm_state = PREPARE_TERMINATE;
1492 write_unlock_irq(&gpii->pm_lock);
1500 gchan = &gpii->gchan[i];
1504 dev_err(gpii->gpi_dev->dev, "Error resetting channel ret:%d\n", ret);
1511 dev_err(gpii->gpi_dev->dev, "Error alloc_channel ret:%d\n", ret);
1518 gchan = &gpii->gchan[i];
1522 dev_err(gpii->gpi_dev->dev, "Error Starting Channel ret:%d\n", ret);
1528 mutex_unlock(&gpii->ctrl_lock);
1536 struct gpii *gpii = gchan->gpii;
1539 mutex_lock(&gpii->ctrl_lock);
1545 if (gpii->pm_state == PAUSE_STATE) {
1546 dev_dbg(gpii->gpi_dev->dev, "channel is already paused\n");
1547 mutex_unlock(&gpii->ctrl_lock);
1553 ret = gpi_stop_chan(&gpii->gchan[i]);
1555 mutex_unlock(&gpii->ctrl_lock);
1560 disable_irq(gpii->irq);
1563 tasklet_kill(&gpii->ev_task);
1565 write_lock_irq(&gpii->pm_lock);
1566 gpii->pm_state = PAUSE_STATE;
1567 write_unlock_irq(&gpii->pm_lock);
1568 mutex_unlock(&gpii->ctrl_lock);
1577 struct gpii *gpii = gchan->gpii;
1580 mutex_lock(&gpii->ctrl_lock);
1581 if (gpii->pm_state == ACTIVE_STATE) {
1582 dev_dbg(gpii->gpi_dev->dev, "channel is already active\n");
1583 mutex_unlock(&gpii->ctrl_lock);
1587 enable_irq(gpii->irq);
1591 ret = gpi_send_cmd(gpii, &gpii->gchan[i], GPI_CH_CMD_START);
1593 dev_err(gpii->gpi_dev->dev, "Error starting chan, ret:%d\n", ret);
1594 mutex_unlock(&gpii->ctrl_lock);
1599 write_lock_irq(&gpii->pm_lock);
1600 gpii->pm_state = ACTIVE_STATE;
1601 write_unlock_irq(&gpii->pm_lock);
1602 mutex_unlock(&gpii->ctrl_lock);
1620 if (!config->peripheral_config)
1621 return -EINVAL;
1623 gchan->config = krealloc(gchan->config, config->peripheral_size, GFP_NOWAIT);
1624 if (!gchan->config)
1625 return -ENOMEM;
1627 memcpy(gchan->config, config->peripheral_config, config->peripheral_size);
1635 struct gpi_i2c_config *i2c = chan->config;
1636 struct device *dev = chan->gpii->gpi_dev->dev;
1643 if (i2c->set_config) {
1644 tre = &desc->tre[tre_idx];
1647 tre->dword[0] = u32_encode_bits(i2c->low_count, TRE_I2C_C0_TLOW);
1648 tre->dword[0] |= u32_encode_bits(i2c->high_count, TRE_I2C_C0_THIGH);
1649 tre->dword[0] |= u32_encode_bits(i2c->cycle_count, TRE_I2C_C0_TCYL);
1650 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_TX_PACK);
1651 tre->dword[0] |= u32_encode_bits(i2c->pack_enable, TRE_I2C_C0_RX_PACK);
1653 tre->dword[1] = 0;
1655 tre->dword[2] = u32_encode_bits(i2c->clk_div, TRE_C0_CLK_DIV);
1657 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);
1658 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1662 if (i2c->op == I2C_WRITE) {
1663 tre = &desc->tre[tre_idx];
1666 if (i2c->multi_msg)
1667 tre->dword[0] = u32_encode_bits(I2C_READ, TRE_I2C_GO_CMD);
1669 tre->dword[0] = u32_encode_bits(i2c->op, TRE_I2C_GO_CMD);
1671 tre->dword[0] |= u32_encode_bits(i2c->addr, TRE_I2C_GO_ADDR);
1672 tre->dword[0] |= u32_encode_bits(i2c->stretch, TRE_I2C_GO_STRETCH);
1674 tre->dword[1] = 0;
1675 tre->dword[2] = u32_encode_bits(i2c->rx_len, TRE_RX_LEN);
1677 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);
1679 if (i2c->multi_msg)
1680 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1682 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1685 if (i2c->op == I2C_READ || i2c->multi_msg == false) {
1687 tre = &desc->tre[tre_idx];
1691 tre->dword[0] = lower_32_bits(address);
1692 tre->dword[1] = upper_32_bits(address);
1694 tre->dword[2] = u32_encode_bits(sg_dma_len(sgl), TRE_DMA_LEN);
1696 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);
1697 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOT);
1701 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],
1702 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);
1710 struct gpi_spi_config *spi = chan->config;
1711 struct device *dev = chan->gpii->gpi_dev->dev;
1719 if (direction == DMA_MEM_TO_DEV && spi->set_config) {
1720 tre = &desc->tre[tre_idx];
1723 tre->dword[0] = u32_encode_bits(spi->word_len, TRE_SPI_C0_WORD_SZ);
1724 tre->dword[0] |= u32_encode_bits(spi->loopback_en, TRE_SPI_C0_LOOPBACK);
1725 tre->dword[0] |= u32_encode_bits(spi->clock_pol_high, TRE_SPI_C0_CPOL);
1726 tre->dword[0] |= u32_encode_bits(spi->data_pol_high, TRE_SPI_C0_CPHA);
1727 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_TX_PACK);
1728 tre->dword[0] |= u32_encode_bits(spi->pack_en, TRE_SPI_C0_RX_PACK);
1730 tre->dword[1] = 0;
1732 tre->dword[2] = u32_encode_bits(spi->clk_div, TRE_C0_CLK_DIV);
1733 tre->dword[2] |= u32_encode_bits(spi->clk_src, TRE_C0_CLK_SRC);
1735 tre->dword[3] = u32_encode_bits(TRE_TYPE_CONFIG0, TRE_FLAGS_TYPE);
1736 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1741 tre = &desc->tre[tre_idx];
1744 tre->dword[0] = u32_encode_bits(spi->fragmentation, TRE_SPI_GO_FRAG);
1745 tre->dword[0] |= u32_encode_bits(spi->cs, TRE_SPI_GO_CS);
1746 tre->dword[0] |= u32_encode_bits(spi->cmd, TRE_SPI_GO_CMD);
1748 tre->dword[1] = 0;
1750 tre->dword[2] = u32_encode_bits(spi->rx_len, TRE_RX_LEN);
1752 tre->dword[3] = u32_encode_bits(TRE_TYPE_GO, TRE_FLAGS_TYPE);
1753 if (spi->cmd == SPI_RX) {
1754 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_IEOB);
1755 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1756 } else if (spi->cmd == SPI_TX) {
1757 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1759 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_CHAIN);
1760 tre->dword[3] |= u32_encode_bits(1, TRE_FLAGS_LINK);
1765 tre = &desc->tre[tre_idx];
1772 if (direction == DMA_MEM_TO_DEV && len <= 2 * sizeof(tre->dword[0])) {
1777 tre->dword[0] = 0;
1778 tre->dword[1] = 0;
1779 memcpy(&tre->dword[0], sg_virt(sgl), len);
1781 tre->dword[2] = u32_encode_bits(len, TRE_DMA_IMMEDIATE_LEN);
1782 tre->dword[3] = u32_encode_bits(TRE_TYPE_IMMEDIATE_DMA, TRE_FLAGS_TYPE);
1784 tre->dword[0] = lower_32_bits(address);
1785 tre->dword[1] = upper_32_bits(address);
1787 tre->dword[2] = u32_encode_bits(len, TRE_DMA_LEN);
1788 tre->dword[3] = u32_encode_bits(TRE_TYPE_DMA, TRE_FLAGS_TYPE);
1791 tre->dword[3] |= u32_encode_bits(direction == DMA_MEM_TO_DEV,
1795 dev_dbg(dev, "TRE:%d %x:%x:%x:%x\n", i, desc->tre[i].dword[0],
1796 desc->tre[i].dword[1], desc->tre[i].dword[2], desc->tre[i].dword[3]);
1808 struct gpii *gpii = gchan->gpii;
1809 struct device *dev = gpii->gpi_dev->dev;
1810 struct gpi_ring *ch_ring = &gchan->ch_ring;
1816 gpii->ieob_set = false;
1818 dev_err(gpii->gpi_dev->dev, "invalid dma direction: %d\n", direction);
1828 set_config = *(u32 *)gchan->config;
1846 if (gchan->protocol == QCOM_GPI_SPI) {
1848 } else if (gchan->protocol == QCOM_GPI_I2C) {
1851 dev_err(dev, "invalid peripheral: %d\n", gchan->protocol);
1857 gpi_desc->gchan = gchan;
1858 gpi_desc->len = sg_dma_len(sgl);
1859 gpi_desc->num_tre = i;
1861 return vchan_tx_prep(&gchan->vc, &gpi_desc->vd, flags);
1868 struct gpii *gpii = gchan->gpii;
1872 struct gpi_ring *ch_ring = &gchan->ch_ring;
1873 void *tre, *wp = NULL;
1876 read_lock_irqsave(&gpii->pm_lock, pm_lock_flags);
1879 spin_lock_irqsave(&gchan->vc.lock, flags);
1880 if (vchan_issue_pending(&gchan->vc))
1881 vd = list_last_entry(&gchan->vc.desc_issued,
1883 spin_unlock_irqrestore(&gchan->vc.lock, flags);
1887 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);
1892 for (i = 0; i < gpi_desc->num_tre; i++) {
1893 tre = &gpi_desc->tre[i];
1894 gpi_queue_xfer(gpii, gchan, tre, &wp);
1897 gpi_desc->db = ch_ring->wp;
1898 gpi_write_ch_db(gchan, &gchan->ch_ring, gpi_desc->db);
1899 read_unlock_irqrestore(&gpii->pm_lock, pm_lock_flags);
1904 struct gpii *gpii = gchan->gpii;
1905 const int ev_factor = gpii->gpi_dev->ev_factor;
1909 gchan->pm_state = CONFIG_STATE;
1913 if (gpii->gchan[i].pm_state != CONFIG_STATE)
1917 if (gpii->gchan[0].protocol != gpii->gchan[1].protocol) {
1918 dev_err(gpii->gpi_dev->dev, "protocol did not match protocol %u != %u\n",
1919 gpii->gchan[0].protocol, gpii->gchan[1].protocol);
1920 ret = -EINVAL;
1926 ret = gpi_alloc_ring(&gpii->ev_ring, elements,
1932 write_lock_irq(&gpii->pm_lock);
1933 gpii->pm_state = PREPARE_HARDWARE;
1934 write_unlock_irq(&gpii->pm_lock);
1937 dev_err(gpii->gpi_dev->dev, "error config. interrupts, ret:%d\n", ret);
1944 dev_err(gpii->gpi_dev->dev, "error alloc_ev_chan:%d\n", ret);
1950 ret = gpi_alloc_chan(&gpii->gchan[i], true);
1952 dev_err(gpii->gpi_dev->dev, "Error allocating chan:%d\n", ret);
1959 ret = gpi_start_chan(&gpii->gchan[i]);
1961 dev_err(gpii->gpi_dev->dev, "Error start chan:%d\n", ret);
1968 for (i = i - 1; i >= 0; i--) {
1969 gpi_stop_chan(&gpii->gchan[i]);
1974 for (i = i - 1; i >= 0; i--)
1979 gpi_free_ring(&gpii->ev_ring, gpii);
1988 struct gpii *gpii = gchan->gpii;
1992 mutex_lock(&gpii->ctrl_lock);
1994 cur_state = gchan->pm_state;
1996 /* disable ch state so no more TRE processing for this channel */
1997 write_lock_irq(&gpii->pm_lock);
1998 gchan->pm_state = PREPARE_TERMINATE;
1999 write_unlock_irq(&gpii->pm_lock);
2007 dev_err(gpii->gpi_dev->dev, "error resetting channel:%d\n", ret);
2013 gpi_free_ring(&gchan->ch_ring, gpii);
2014 vchan_free_chan_resources(&gchan->vc);
2015 kfree(gchan->config);
2017 write_lock_irq(&gpii->pm_lock);
2018 gchan->pm_state = DISABLE_STATE;
2019 write_unlock_irq(&gpii->pm_lock);
2023 if (gpii->gchan[i].ch_ring.configured)
2027 cur_state = gpii->pm_state;
2028 write_lock_irq(&gpii->pm_lock);
2029 gpii->pm_state = PREPARE_TERMINATE;
2030 write_unlock_irq(&gpii->pm_lock);
2033 tasklet_kill(&gpii->ev_task);
2039 gpi_free_ring(&gpii->ev_ring, gpii);
2046 write_lock_irq(&gpii->pm_lock);
2047 gpii->pm_state = DISABLE_STATE;
2048 write_unlock_irq(&gpii->pm_lock);
2051 mutex_unlock(&gpii->ctrl_lock);
2058 struct gpii *gpii = gchan->gpii;
2061 mutex_lock(&gpii->ctrl_lock);
2064 ret = gpi_alloc_ring(&gchan->ch_ring, CHAN_TRES,
2071 mutex_unlock(&gpii->ctrl_lock);
2075 mutex_unlock(&gpii->ctrl_lock);
2086 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {
2087 if (!((1 << gpii) & gpi_dev->gpii_mask))
2090 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];
2091 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];
2093 if (rx_chan->vc.chan.client_count && rx_chan->seid == seid)
2095 if (tx_chan->vc.chan.client_count && tx_chan->seid == seid)
2099 /* no channels configured with same seid, return next avail gpii */
2100 for (gpii = 0; gpii < gpi_dev->max_gpii; gpii++) {
2101 if (!((1 << gpii) & gpi_dev->gpii_mask))
2104 tx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_TX_CHAN];
2105 rx_chan = &gpi_dev->gpiis[gpii].gchan[GPI_RX_CHAN];
2108 if (tx_chan->vc.chan.client_count ||
2109 rx_chan->vc.chan.client_count)
2116 /* no gpii instance available to use */
2117 return -EIO;
2124 struct gpi_dev *gpi_dev = (struct gpi_dev *)of_dma->of_dma_data;
2129 if (args->args_count < 3) {
2130 dev_err(gpi_dev->dev, "gpii require minimum 2 args, client passed:%d args\n",
2131 args->args_count);
2135 chid = args->args[0];
2137 dev_err(gpi_dev->dev, "gpii channel:%d not valid\n", chid);
2141 seid = args->args[1];
2146 dev_err(gpi_dev->dev, "no available gpii instances\n");
2150 gchan = &gpi_dev->gpiis[gpii].gchan[chid];
2151 if (gchan->vc.chan.client_count) {
2152 dev_err(gpi_dev->dev, "gpii:%d chid:%d seid:%d already configured\n",
2153 gpii, chid, gchan->seid);
2157 gchan->seid = seid;
2158 gchan->protocol = args->args[2];
2160 return dma_get_slave_channel(&gchan->vc.chan);
2170 gpi_dev = devm_kzalloc(&pdev->dev, sizeof(*gpi_dev), GFP_KERNEL);
2172 return -ENOMEM;
2174 gpi_dev->dev = &pdev->dev;
2175 gpi_dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &gpi_dev->res);
2176 if (IS_ERR(gpi_dev->regs))
2177 return PTR_ERR(gpi_dev->regs);
2178 gpi_dev->ee_base = gpi_dev->regs;
2180 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channels",
2181 &gpi_dev->max_gpii);
2183 dev_err(gpi_dev->dev, "missing 'max-no-gpii' DT node\n");
2187 ret = of_property_read_u32(gpi_dev->dev->of_node, "dma-channel-mask",
2188 &gpi_dev->gpii_mask);
2190 dev_err(gpi_dev->dev, "missing 'gpii-mask' DT node\n");
2194 ee_offset = (uintptr_t)device_get_match_data(gpi_dev->dev);
2195 gpi_dev->ee_base = gpi_dev->ee_base - ee_offset;
2197 gpi_dev->ev_factor = EV_FACTOR;
2199 ret = dma_set_mask(gpi_dev->dev, DMA_BIT_MASK(64));
2201 dev_err(gpi_dev->dev, "Error setting dma_mask to 64, ret:%d\n", ret);
2205 gpi_dev->gpiis = devm_kzalloc(gpi_dev->dev, sizeof(*gpi_dev->gpiis) *
2206 gpi_dev->max_gpii, GFP_KERNEL);
2207 if (!gpi_dev->gpiis)
2208 return -ENOMEM;
2211 INIT_LIST_HEAD(&gpi_dev->dma_device.channels);
2212 for (i = 0; i < gpi_dev->max_gpii; i++) {
2213 struct gpii *gpii = &gpi_dev->gpiis[i];
2216 if (!((1 << i) & gpi_dev->gpii_mask))
2220 gpii->ev_cntxt_base_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_CNTXT_0_OFFS(i, 0);
2221 gpii->ev_cntxt_db_reg = gpi_dev->ee_base + GPII_n_EV_CH_k_DOORBELL_0_OFFS(i, 0);
2222 gpii->ev_ring_rp_lsb_reg = gpii->ev_cntxt_base_reg + CNTXT_4_RING_RP_LSB;
2223 gpii->ev_cmd_reg = gpi_dev->ee_base + GPII_n_EV_CH_CMD_OFFS(i);
2224 gpii->ieob_clr_reg = gpi_dev->ee_base + GPII_n_CNTXT_SRC_IEOB_IRQ_CLR_OFFS(i);
2230 gpii->irq = ret;
2234 struct gchan *gchan = &gpii->gchan[chan];
2237 gchan->ch_cntxt_base_reg = gpi_dev->ee_base +
2239 gchan->ch_cntxt_db_reg = gpi_dev->ee_base +
2241 gchan->ch_cmd_reg = gpi_dev->ee_base + GPII_n_CH_CMD_OFFS(i);
2244 vchan_init(&gchan->vc, &gpi_dev->dma_device);
2245 gchan->vc.desc_free = gpi_desc_free;
2246 gchan->chid = chan;
2247 gchan->gpii = gpii;
2248 gchan->dir = GPII_CHAN_DIR[chan];
2250 mutex_init(&gpii->ctrl_lock);
2251 rwlock_init(&gpii->pm_lock);
2252 tasklet_init(&gpii->ev_task, gpi_ev_tasklet,
2254 init_completion(&gpii->cmd_completion);
2255 gpii->gpii_id = i;
2256 gpii->regs = gpi_dev->ee_base;
2257 gpii->gpi_dev = gpi_dev;
2263 dma_cap_zero(gpi_dev->dma_device.cap_mask);
2264 dma_cap_set(DMA_SLAVE, gpi_dev->dma_device.cap_mask);
2267 gpi_dev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2268 gpi_dev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
2269 gpi_dev->dma_device.src_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;
2270 gpi_dev->dma_device.dst_addr_widths = DMA_SLAVE_BUSWIDTH_8_BYTES;
2271 gpi_dev->dma_device.device_alloc_chan_resources = gpi_alloc_chan_resources;
2272 gpi_dev->dma_device.device_free_chan_resources = gpi_free_chan_resources;
2273 gpi_dev->dma_device.device_tx_status = dma_cookie_status;
2274 gpi_dev->dma_device.device_issue_pending = gpi_issue_pending;
2275 gpi_dev->dma_device.device_prep_slave_sg = gpi_prep_slave_sg;
2276 gpi_dev->dma_device.device_config = gpi_peripheral_config;
2277 gpi_dev->dma_device.device_terminate_all = gpi_terminate_all;
2278 gpi_dev->dma_device.dev = gpi_dev->dev;
2279 gpi_dev->dma_device.device_pause = gpi_pause;
2280 gpi_dev->dma_device.device_resume = gpi_resume;
2283 ret = dma_async_device_register(&gpi_dev->dma_device);
2285 dev_err(gpi_dev->dev, "async_device_register failed ret:%d", ret);
2289 ret = of_dma_controller_register(gpi_dev->dev->of_node,
2292 dev_err(gpi_dev->dev, "of_dma_controller_reg failed ret:%d", ret);
2300 { .compatible = "qcom,sdm845-gpi-dma", .data = (void *)0x0 },
2301 { .compatible = "qcom,sm6350-gpi-dma", .data = (void *)0x10000 },
2304 * qcom,sdm845-gpi-dma (for ee_offset = 0x0) or qcom,sm6350-gpi-dma
2307 { .compatible = "qcom,sc7280-gpi-dma", .data = (void *)0x10000 },
2308 { .compatible = "qcom,sm8150-gpi-dma", .data = (void *)0x0 },
2309 { .compatible = "qcom,sm8250-gpi-dma", .data = (void *)0x0 },
2310 { .compatible = "qcom,sm8350-gpi-dma", .data = (void *)0x10000 },
2311 { .compatible = "qcom,sm8450-gpi-dma", .data = (void *)0x10000 },