Lines Matching +full:bd +full:- +full:address
1 // SPDX-License-Identifier: GPL-2.0-or-later
4 * Copyright (c) 2010 ST-Ericsson SA
27 * - CH_CONFIG register at different offset,
28 * - separate CH_CONTROL2 register for transfer size,
29 * - bigger maximum transfer size,
30 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word,
31 * - no support for peripheral flow control.
45 * (Bursts are irrelevant for mem to mem transfers - there are no burst
51 * - DMAC flow control: the transfer size defines the number of transfers
58 * - Peripheral flow control: the transfer size is ignored (and should be
70 #include <linux/dma-mapping.h>
83 #include "virt-dma.h"
97 * struct vendor_data - vendor-specific config parameters for PL08x derivatives
123 * struct pl08x_bus_data - information of source or destination
125 * @addr: current address
135 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth)
138 * struct pl08x_phy_chan - holder for the physical channels
140 * @base: memory base address for this physical channel
141 * @reg_config: configuration address for this physical channel
142 * @reg_control: control address for this physical channel
143 * @reg_src: transfer source address register
144 * @reg_dst: transfer destination address register
145 * @reg_lli: transfer LLI address register
146 * @reg_busy: if the variant has a special per-channel busy register,
173 * struct pl08x_sg - structure containing data per sg
174 * @src_addr: src address of sg
175 * @dst_addr: dst address of sg
187 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor
190 * @llis_bus: DMA memory address (physical) start for the LLIs
191 * @llis_va: virtual memory address start for the LLIs
215 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel
233 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel
263 * struct pl08x_driver_data - the local state holder for the PL08x
335 const struct pl08x_platform_data *pd = plchan->host->pd; in pl08x_request_mux()
338 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { in pl08x_request_mux()
339 ret = pd->get_xfer_signal(plchan->cd); in pl08x_request_mux()
341 plchan->mux_use = 0; in pl08x_request_mux()
345 plchan->signal = ret; in pl08x_request_mux()
352 const struct pl08x_platform_data *pd = plchan->host->pd; in pl08x_release_mux()
354 if (plchan->signal >= 0) { in pl08x_release_mux()
355 WARN_ON(plchan->mux_use == 0); in pl08x_release_mux()
357 if (--plchan->mux_use == 0 && pd->put_xfer_signal) { in pl08x_release_mux()
358 pd->put_xfer_signal(plchan->cd, plchan->signal); in pl08x_release_mux()
359 plchan->signal = -1; in pl08x_release_mux()
374 if (ch->reg_busy) { in pl08x_phy_channel_busy()
375 val = readl(ch->reg_busy); in pl08x_phy_channel_busy()
376 return !!(val & BIT(ch->id)); in pl08x_phy_channel_busy()
378 val = readl(ch->reg_config); in pl08x_phy_channel_busy()
383 * pl08x_write_lli() - Write an LLI into the DMA controller.
394 if (pl08x->vd->pl080s) in pl08x_write_lli()
395 dev_vdbg(&pl08x->adev->dev, in pl08x_write_lli()
398 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], in pl08x_write_lli()
402 dev_vdbg(&pl08x->adev->dev, in pl08x_write_lli()
405 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], in pl08x_write_lli()
408 writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src); in pl08x_write_lli()
409 writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst); in pl08x_write_lli()
410 writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli); in pl08x_write_lli()
418 if (phychan->ftdmac020) { in pl08x_write_lli()
424 phychan->base + FTDMAC020_CH_SIZE); in pl08x_write_lli()
429 * Bit 28: TC_MSK - mask on all except last LLI in pl08x_write_lli()
440 (FTDMAC020_LLI_SRC_WIDTH_SHIFT - in pl08x_write_lli()
443 (FTDMAC020_LLI_DST_WIDTH_SHIFT - in pl08x_write_lli()
446 (FTDMAC020_LLI_SRCAD_CTL_SHIFT - in pl08x_write_lli()
449 (FTDMAC020_LLI_DSTAD_CTL_SHIFT - in pl08x_write_lli()
463 switch (pl08x->pd->memcpy_burst_size) { in pl08x_write_lli()
500 if (pl08x->pd->memcpy_prot_buff) in pl08x_write_lli()
502 if (pl08x->pd->memcpy_prot_cache) in pl08x_write_lli()
507 writel_relaxed(val, phychan->reg_control); in pl08x_write_lli()
510 writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control); in pl08x_write_lli()
514 if (pl08x->vd->pl080s) in pl08x_write_lli()
516 phychan->base + PL080S_CH_CONTROL2); in pl08x_write_lli()
518 writel(ccfg, phychan->reg_config); in pl08x_write_lli()
529 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_start_next_txd()
530 struct pl08x_phy_chan *phychan = plchan->phychan; in pl08x_start_next_txd()
531 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); in pl08x_start_next_txd()
532 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); in pl08x_start_next_txd()
535 list_del(&txd->vd.node); in pl08x_start_next_txd()
537 plchan->at = txd; in pl08x_start_next_txd()
543 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); in pl08x_start_next_txd()
547 while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id)) in pl08x_start_next_txd()
551 if (phychan->ftdmac020) { in pl08x_start_next_txd()
552 val = readl(phychan->reg_config); in pl08x_start_next_txd()
554 val = readl(phychan->reg_config); in pl08x_start_next_txd()
556 val = readl(phychan->reg_control); in pl08x_start_next_txd()
558 val = readl(phychan->reg_control); in pl08x_start_next_txd()
561 phychan->reg_control); in pl08x_start_next_txd()
563 val = readl(phychan->reg_config); in pl08x_start_next_txd()
566 val = readl(phychan->reg_config); in pl08x_start_next_txd()
568 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); in pl08x_start_next_txd()
575 * For M->P transfers, pause the DMAC first and then stop the peripheral -
579 * For P->M transfers, disable the peripheral first to stop it filling
587 if (ch->ftdmac020) { in pl08x_pause_phy_chan()
589 val = readl(ch->reg_control); in pl08x_pause_phy_chan()
591 writel(val, ch->reg_control); in pl08x_pause_phy_chan()
596 val = readl(ch->reg_config); in pl08x_pause_phy_chan()
598 writel(val, ch->reg_config); in pl08x_pause_phy_chan()
601 for (timeout = 1000; timeout; timeout--) { in pl08x_pause_phy_chan()
607 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); in pl08x_pause_phy_chan()
615 if (ch->ftdmac020) { in pl08x_resume_phy_chan()
616 val = readl(ch->reg_control); in pl08x_resume_phy_chan()
618 writel(val, ch->reg_control); in pl08x_resume_phy_chan()
623 val = readl(ch->reg_config); in pl08x_resume_phy_chan()
625 writel(val, ch->reg_config); in pl08x_resume_phy_chan()
631 * an on-going transfer, but as a method of shutting down a channel
640 if (ch->ftdmac020) { in pl08x_terminate_phy_chan()
642 val = readl(ch->reg_config); in pl08x_terminate_phy_chan()
646 writel(val, ch->reg_config); in pl08x_terminate_phy_chan()
649 val = readl(ch->reg_control); in pl08x_terminate_phy_chan()
652 writel(val, ch->reg_control); in pl08x_terminate_phy_chan()
655 writel(BIT(ch->id) | BIT(ch->id + 16), in pl08x_terminate_phy_chan()
656 pl08x->base + PL080_ERR_CLEAR); in pl08x_terminate_phy_chan()
657 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); in pl08x_terminate_phy_chan()
662 val = readl(ch->reg_config); in pl08x_terminate_phy_chan()
665 writel(val, ch->reg_config); in pl08x_terminate_phy_chan()
667 writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR); in pl08x_terminate_phy_chan()
668 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); in pl08x_terminate_phy_chan()
676 if (ch->ftdmac020) { in get_bytes_in_phy_channel()
677 bytes = readl(ch->base + FTDMAC020_CH_SIZE); in get_bytes_in_phy_channel()
679 val = readl(ch->reg_control); in get_bytes_in_phy_channel()
682 } else if (ch->pl080s) { in get_bytes_in_phy_channel()
683 val = readl(ch->base + PL080S_CH_CONTROL2); in get_bytes_in_phy_channel()
686 val = readl(ch->reg_control); in get_bytes_in_phy_channel()
691 val = readl(ch->reg_control); in get_bytes_in_phy_channel()
716 if (ch->ftdmac020) { in get_bytes_in_lli()
723 } else if (ch->pl080s) { in get_bytes_in_lli()
755 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_getbytes_chan()
764 ch = plchan->phychan; in pl08x_getbytes_chan()
765 txd = plchan->at; in pl08x_getbytes_chan()
774 clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2; in pl08x_getbytes_chan()
782 llis_va = txd->llis_va; in pl08x_getbytes_chan()
783 llis_bus = txd->llis_bus; in pl08x_getbytes_chan()
785 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; in pl08x_getbytes_chan()
790 * Locate the next LLI - as this is an array, in pl08x_getbytes_chan()
793 llis_va += (clli - llis_bus) / sizeof(u32); in pl08x_getbytes_chan()
797 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { in pl08x_getbytes_chan()
825 for (i = 0; i < pl08x->vd->channels; i++) { in pl08x_get_phy_channel()
826 ch = &pl08x->phy_chans[i]; in pl08x_get_phy_channel()
828 spin_lock_irqsave(&ch->lock, flags); in pl08x_get_phy_channel()
830 if (!ch->locked && !ch->serving) { in pl08x_get_phy_channel()
831 ch->serving = virt_chan; in pl08x_get_phy_channel()
832 spin_unlock_irqrestore(&ch->lock, flags); in pl08x_get_phy_channel()
836 spin_unlock_irqrestore(&ch->lock, flags); in pl08x_get_phy_channel()
839 if (i == pl08x->vd->channels) { in pl08x_get_phy_channel()
851 ch->serving = NULL; in pl08x_put_phy_channel()
861 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_phy_alloc_and_start()
866 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); in pl08x_phy_alloc_and_start()
867 plchan->state = PL08X_CHAN_WAITING; in pl08x_phy_alloc_and_start()
868 plchan->waiting_at = jiffies; in pl08x_phy_alloc_and_start()
872 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", in pl08x_phy_alloc_and_start()
873 ch->id, plchan->name); in pl08x_phy_alloc_and_start()
875 plchan->phychan = ch; in pl08x_phy_alloc_and_start()
876 plchan->state = PL08X_CHAN_RUNNING; in pl08x_phy_alloc_and_start()
883 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_phy_reassign_start()
885 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", in pl08x_phy_reassign_start()
886 ch->id, plchan->name); in pl08x_phy_reassign_start()
891 * that this will only be called when it _already_ is non-NULL. in pl08x_phy_reassign_start()
893 ch->serving = plchan; in pl08x_phy_reassign_start()
894 plchan->phychan = ch; in pl08x_phy_reassign_start()
895 plchan->state = PL08X_CHAN_RUNNING; in pl08x_phy_reassign_start()
905 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_phy_free()
917 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) in pl08x_phy_free()
918 if (p->state == PL08X_CHAN_WAITING && in pl08x_phy_free()
919 p->waiting_at <= waiting_at) { in pl08x_phy_free()
921 waiting_at = p->waiting_at; in pl08x_phy_free()
924 if (!next && pl08x->has_slave) { in pl08x_phy_free()
925 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) in pl08x_phy_free()
926 if (p->state == PL08X_CHAN_WAITING && in pl08x_phy_free()
927 p->waiting_at <= waiting_at) { in pl08x_phy_free()
929 waiting_at = p->waiting_at; in pl08x_phy_free()
934 pl08x_terminate_phy_chan(pl08x, plchan->phychan); in pl08x_phy_free()
943 spin_lock(&next->vc.lock); in pl08x_phy_free()
944 /* Re-check the state now that we have the lock */ in pl08x_phy_free()
945 success = next->state == PL08X_CHAN_WAITING; in pl08x_phy_free()
947 pl08x_phy_reassign_start(plchan->phychan, next); in pl08x_phy_free()
948 spin_unlock(&next->vc.lock); in pl08x_phy_free()
955 pl08x_put_phy_channel(pl08x, plchan->phychan); in pl08x_phy_free()
958 plchan->phychan = NULL; in pl08x_phy_free()
959 plchan->state = PL08X_CHAN_IDLE; in pl08x_phy_free()
973 if (pl08x->vd->ftdmac020) { in pl08x_get_bytes_for_lli()
1015 if (pl08x->vd->ftdmac020) { in pl08x_lli_control_bits()
1117 * masters address with width requirements of transfer (by sending few byte by
1120 * - prefers the destination bus if both available
1121 * - prefers bus with fixed address (i.e. peripheral)
1124 struct pl08x_lli_build_data *bd, in pl08x_choose_master_bus() argument
1133 * The FTDMAC020 only supports memory-to-memory transfer, so in pl08x_choose_master_bus()
1136 if (pl08x->vd->ftdmac020) { in pl08x_choose_master_bus()
1149 *mbus = &bd->dstbus; in pl08x_choose_master_bus()
1150 *sbus = &bd->srcbus; in pl08x_choose_master_bus()
1152 *mbus = &bd->srcbus; in pl08x_choose_master_bus()
1153 *sbus = &bd->dstbus; in pl08x_choose_master_bus()
1155 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { in pl08x_choose_master_bus()
1156 *mbus = &bd->dstbus; in pl08x_choose_master_bus()
1157 *sbus = &bd->srcbus; in pl08x_choose_master_bus()
1159 *mbus = &bd->srcbus; in pl08x_choose_master_bus()
1160 *sbus = &bd->dstbus; in pl08x_choose_master_bus()
1169 struct pl08x_lli_build_data *bd, in pl08x_fill_lli_for_desc() argument
1172 u32 offset = num_llis * pl08x->lli_words; in pl08x_fill_lli_for_desc()
1173 u32 *llis_va = bd->txd->llis_va + offset; in pl08x_fill_lli_for_desc()
1174 dma_addr_t llis_bus = bd->txd->llis_bus; in pl08x_fill_lli_for_desc()
1179 offset += pl08x->lli_words; in pl08x_fill_lli_for_desc()
1181 llis_va[PL080_LLI_SRC] = bd->srcbus.addr; in pl08x_fill_lli_for_desc()
1182 llis_va[PL080_LLI_DST] = bd->dstbus.addr; in pl08x_fill_lli_for_desc()
1184 llis_va[PL080_LLI_LLI] |= bd->lli_bus; in pl08x_fill_lli_for_desc()
1186 if (pl08x->vd->pl080s) in pl08x_fill_lli_for_desc()
1189 if (pl08x->vd->ftdmac020) { in pl08x_fill_lli_for_desc()
1191 bd->srcbus.addr += len; in pl08x_fill_lli_for_desc()
1192 bd->dstbus.addr += len; in pl08x_fill_lli_for_desc()
1195 bd->srcbus.addr += len; in pl08x_fill_lli_for_desc()
1197 bd->dstbus.addr += len; in pl08x_fill_lli_for_desc()
1200 BUG_ON(bd->remainder < len); in pl08x_fill_lli_for_desc()
1202 bd->remainder -= len; in pl08x_fill_lli_for_desc()
1206 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len, in prep_byte_width_lli() argument
1210 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len); in prep_byte_width_lli()
1220 if (pl08x->vd->pl080s) { in pl08x_dump_lli()
1221 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1222 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", in pl08x_dump_lli()
1225 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1231 llis_va += pl08x->lli_words; in pl08x_dump_lli()
1234 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1235 "%-3s %-9s %-10s %-10s %-10s %s\n", in pl08x_dump_lli()
1238 dev_vdbg(&pl08x->adev->dev, in pl08x_dump_lli()
1243 llis_va += pl08x->lli_words; in pl08x_dump_lli()
1261 struct pl08x_lli_build_data bd; in pl08x_fill_llis_for_desc() local
1268 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); in pl08x_fill_llis_for_desc()
1269 if (!txd->llis_va) { in pl08x_fill_llis_for_desc()
1270 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); in pl08x_fill_llis_for_desc()
1274 bd.txd = txd; in pl08x_fill_llis_for_desc()
1275 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; in pl08x_fill_llis_for_desc()
1276 cctl = txd->cctl; in pl08x_fill_llis_for_desc()
1279 bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true); in pl08x_fill_llis_for_desc()
1282 bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false); in pl08x_fill_llis_for_desc()
1284 list_for_each_entry(dsg, &txd->dsg_list, node) { in pl08x_fill_llis_for_desc()
1286 cctl = txd->cctl; in pl08x_fill_llis_for_desc()
1288 bd.srcbus.addr = dsg->src_addr; in pl08x_fill_llis_for_desc()
1289 bd.dstbus.addr = dsg->dst_addr; in pl08x_fill_llis_for_desc()
1290 bd.remainder = dsg->len; in pl08x_fill_llis_for_desc()
1291 bd.srcbus.buswidth = bd.srcbus.maxwidth; in pl08x_fill_llis_for_desc()
1292 bd.dstbus.buswidth = bd.dstbus.maxwidth; in pl08x_fill_llis_for_desc()
1294 pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl); in pl08x_fill_llis_for_desc()
1296 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1298 (u64)bd.srcbus.addr, in pl08x_fill_llis_for_desc()
1300 bd.srcbus.buswidth, in pl08x_fill_llis_for_desc()
1301 (u64)bd.dstbus.addr, in pl08x_fill_llis_for_desc()
1303 bd.dstbus.buswidth, in pl08x_fill_llis_for_desc()
1304 bd.remainder); in pl08x_fill_llis_for_desc()
1305 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", in pl08x_fill_llis_for_desc()
1306 mbus == &bd.srcbus ? "src" : "dst", in pl08x_fill_llis_for_desc()
1307 sbus == &bd.srcbus ? "src" : "dst"); in pl08x_fill_llis_for_desc()
1312 * - flow controller is peripheral. in pl08x_fill_llis_for_desc()
1313 * - src.addr is aligned to src.width in pl08x_fill_llis_for_desc()
1314 * - dst.addr is aligned to dst.width in pl08x_fill_llis_for_desc()
1318 * - Memory addresses are contiguous and are not scattered. in pl08x_fill_llis_for_desc()
1320 * memory address and zero length. We pass this to controller in pl08x_fill_llis_for_desc()
1324 * - Memory addresses are scattered and are not contiguous. in pl08x_fill_llis_for_desc()
1330 if (!bd.remainder) { in pl08x_fill_llis_for_desc()
1333 /* FTDMAC020 only does memory-to-memory */ in pl08x_fill_llis_for_desc()
1334 if (pl08x->vd->ftdmac020) in pl08x_fill_llis_for_desc()
1337 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> in pl08x_fill_llis_for_desc()
1341 dev_err(&pl08x->adev->dev, "%s sg len can't be zero", in pl08x_fill_llis_for_desc()
1346 if (!IS_BUS_ALIGNED(&bd.srcbus) || in pl08x_fill_llis_for_desc()
1347 !IS_BUS_ALIGNED(&bd.dstbus)) { in pl08x_fill_llis_for_desc()
1348 dev_err(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1349 "%s src & dst address must be aligned to src" in pl08x_fill_llis_for_desc()
1356 bd.srcbus.buswidth, bd.dstbus.buswidth, in pl08x_fill_llis_for_desc()
1358 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, in pl08x_fill_llis_for_desc()
1365 * - Less than a bus width available in pl08x_fill_llis_for_desc()
1366 * - until master bus is aligned in pl08x_fill_llis_for_desc()
1368 if (bd.remainder < mbus->buswidth) in pl08x_fill_llis_for_desc()
1369 early_bytes = bd.remainder; in pl08x_fill_llis_for_desc()
1371 early_bytes = mbus->buswidth - in pl08x_fill_llis_for_desc()
1372 (mbus->addr & (mbus->buswidth - 1)); in pl08x_fill_llis_for_desc()
1373 if ((bd.remainder - early_bytes) < mbus->buswidth) in pl08x_fill_llis_for_desc()
1374 early_bytes = bd.remainder; in pl08x_fill_llis_for_desc()
1378 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1380 __func__, bd.remainder); in pl08x_fill_llis_for_desc()
1381 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes, in pl08x_fill_llis_for_desc()
1385 if (bd.remainder) { in pl08x_fill_llis_for_desc()
1388 * - if slave is not then we must set its width down in pl08x_fill_llis_for_desc()
1391 dev_dbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1395 sbus->buswidth = 1; in pl08x_fill_llis_for_desc()
1402 max_bytes_per_lli = bd.srcbus.buswidth * in pl08x_fill_llis_for_desc()
1403 pl08x->vd->max_transfer_size; in pl08x_fill_llis_for_desc()
1404 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1412 while (bd.remainder > (mbus->buswidth - 1)) { in pl08x_fill_llis_for_desc()
1419 lli_len = min(bd.remainder, max_bytes_per_lli); in pl08x_fill_llis_for_desc()
1425 * highest bus width - 1 in pl08x_fill_llis_for_desc()
1427 width = max(mbus->buswidth, sbus->buswidth); in pl08x_fill_llis_for_desc()
1429 tsize = lli_len / bd.srcbus.buswidth; in pl08x_fill_llis_for_desc()
1431 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1434 __func__, lli_len, bd.remainder); in pl08x_fill_llis_for_desc()
1437 bd.srcbus.buswidth, bd.dstbus.buswidth, in pl08x_fill_llis_for_desc()
1439 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, in pl08x_fill_llis_for_desc()
1447 if (bd.remainder) { in pl08x_fill_llis_for_desc()
1448 dev_vdbg(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1450 __func__, bd.remainder); in pl08x_fill_llis_for_desc()
1451 prep_byte_width_lli(pl08x, &bd, &cctl, in pl08x_fill_llis_for_desc()
1452 bd.remainder, num_llis++, &total_bytes); in pl08x_fill_llis_for_desc()
1456 if (total_bytes != dsg->len) { in pl08x_fill_llis_for_desc()
1457 dev_err(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1459 __func__, total_bytes, dsg->len); in pl08x_fill_llis_for_desc()
1464 dev_err(&pl08x->adev->dev, in pl08x_fill_llis_for_desc()
1471 llis_va = txd->llis_va; in pl08x_fill_llis_for_desc()
1472 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; in pl08x_fill_llis_for_desc()
1474 if (txd->cyclic) { in pl08x_fill_llis_for_desc()
1476 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; in pl08x_fill_llis_for_desc()
1481 if (pl08x->vd->ftdmac020) in pl08x_fill_llis_for_desc()
1497 if (txd->llis_va) in pl08x_free_txd()
1498 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); in pl08x_free_txd()
1500 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { in pl08x_free_txd()
1501 list_del(&dsg->node); in pl08x_free_txd()
1510 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); in pl08x_desc_free()
1511 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); in pl08x_desc_free()
1513 dma_descriptor_unmap(&vd->tx); in pl08x_desc_free()
1514 if (!txd->done) in pl08x_desc_free()
1517 pl08x_free_txd(plchan->host, txd); in pl08x_desc_free()
1525 vchan_get_all_descriptors(&plchan->vc, &head); in pl08x_free_txd_list()
1526 vchan_dma_desc_free_list(&plchan->vc, &head); in pl08x_free_txd_list()
1561 if (plchan->state == PL08X_CHAN_PAUSED) in pl08x_dma_tx_status()
1566 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_dma_tx_status()
1569 vd = vchan_find_desc(&plchan->vc, cookie); in pl08x_dma_tx_status()
1572 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); in pl08x_dma_tx_status()
1575 list_for_each_entry(dsg, &txd->dsg_list, node) in pl08x_dma_tx_status()
1576 bytes += dsg->len; in pl08x_dma_tx_status()
1581 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_dma_tx_status()
1589 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) in pl08x_dma_tx_status()
1671 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ in pl08x_cctl()
1717 if (plchan->cd->single) in pl08x_get_cctl()
1736 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_issue_pending()
1737 if (vchan_issue_pending(&plchan->vc)) { in pl08x_issue_pending()
1738 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) in pl08x_issue_pending()
1741 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_issue_pending()
1749 INIT_LIST_HEAD(&txd->dsg_list); in pl08x_get_txd()
1758 switch (pl08x->pd->memcpy_burst_size) { in pl08x_memcpy_cctl()
1760 dev_err(&pl08x->adev->dev, in pl08x_memcpy_cctl()
1797 switch (pl08x->pd->memcpy_bus_width) { in pl08x_memcpy_cctl()
1799 dev_err(&pl08x->adev->dev, in pl08x_memcpy_cctl()
1817 if (pl08x->pd->memcpy_prot_buff) in pl08x_memcpy_cctl()
1819 if (pl08x->pd->memcpy_prot_cache) in pl08x_memcpy_cctl()
1828 if (pl08x->vd->dualmaster) in pl08x_memcpy_cctl()
1830 pl08x->mem_buses, in pl08x_memcpy_cctl()
1831 pl08x->mem_buses); in pl08x_memcpy_cctl()
1841 switch (pl08x->pd->memcpy_bus_width) { in pl08x_ftdmac020_memcpy_cctl()
1843 dev_err(&pl08x->adev->dev, in pl08x_ftdmac020_memcpy_cctl()
1870 if (pl08x->vd->dualmaster) in pl08x_ftdmac020_memcpy_cctl()
1872 pl08x->mem_buses, in pl08x_ftdmac020_memcpy_cctl()
1873 pl08x->mem_buses); in pl08x_ftdmac020_memcpy_cctl()
1886 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_prep_dma_memcpy()
1893 dev_err(&pl08x->adev->dev, in pl08x_prep_dma_memcpy()
1903 list_add_tail(&dsg->node, &txd->dsg_list); in pl08x_prep_dma_memcpy()
1905 dsg->src_addr = src; in pl08x_prep_dma_memcpy()
1906 dsg->dst_addr = dest; in pl08x_prep_dma_memcpy()
1907 dsg->len = len; in pl08x_prep_dma_memcpy()
1908 if (pl08x->vd->ftdmac020) { in pl08x_prep_dma_memcpy()
1910 txd->ccfg = 0; in pl08x_prep_dma_memcpy()
1911 txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x); in pl08x_prep_dma_memcpy()
1913 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | in pl08x_prep_dma_memcpy()
1916 txd->cctl = pl08x_memcpy_cctl(pl08x); in pl08x_prep_dma_memcpy()
1919 ret = pl08x_fill_llis_for_desc(plchan->host, txd); in pl08x_prep_dma_memcpy()
1925 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); in pl08x_prep_dma_memcpy()
1934 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_init_txd()
1943 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); in pl08x_init_txd()
1948 * Set up addresses, the PrimeCell configured address in pl08x_init_txd()
1950 * channel target address dynamically at runtime. in pl08x_init_txd()
1954 *slave_addr = plchan->cfg.dst_addr; in pl08x_init_txd()
1955 addr_width = plchan->cfg.dst_addr_width; in pl08x_init_txd()
1956 maxburst = plchan->cfg.dst_maxburst; in pl08x_init_txd()
1957 src_buses = pl08x->mem_buses; in pl08x_init_txd()
1958 dst_buses = plchan->cd->periph_buses; in pl08x_init_txd()
1961 *slave_addr = plchan->cfg.src_addr; in pl08x_init_txd()
1962 addr_width = plchan->cfg.src_addr_width; in pl08x_init_txd()
1963 maxburst = plchan->cfg.src_maxburst; in pl08x_init_txd()
1964 src_buses = plchan->cd->periph_buses; in pl08x_init_txd()
1965 dst_buses = pl08x->mem_buses; in pl08x_init_txd()
1968 dev_err(&pl08x->adev->dev, in pl08x_init_txd()
1976 dev_err(&pl08x->adev->dev, in pl08x_init_txd()
1981 txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses); in pl08x_init_txd()
1983 if (plchan->cfg.device_fc) in pl08x_init_txd()
1990 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | in pl08x_init_txd()
1997 dev_dbg(&pl08x->adev->dev, in pl08x_init_txd()
1999 plchan->name); in pl08x_init_txd()
2003 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", in pl08x_init_txd()
2004 plchan->signal, plchan->name); in pl08x_init_txd()
2008 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; in pl08x_init_txd()
2010 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; in pl08x_init_txd()
2025 return -ENOMEM; in pl08x_tx_add_sg()
2027 list_add_tail(&dsg->node, &txd->dsg_list); in pl08x_tx_add_sg()
2029 dsg->len = len; in pl08x_tx_add_sg()
2031 dsg->src_addr = buf_addr; in pl08x_tx_add_sg()
2032 dsg->dst_addr = slave_addr; in pl08x_tx_add_sg()
2034 dsg->src_addr = slave_addr; in pl08x_tx_add_sg()
2035 dsg->dst_addr = buf_addr; in pl08x_tx_add_sg()
2047 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_prep_slave_sg()
2053 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", in pl08x_prep_slave_sg()
2054 __func__, sg_dma_len(sgl), plchan->name); in pl08x_prep_slave_sg()
2067 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", in pl08x_prep_slave_sg()
2073 ret = pl08x_fill_llis_for_desc(plchan->host, txd); in pl08x_prep_slave_sg()
2080 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); in pl08x_prep_slave_sg()
2089 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_prep_dma_cyclic()
2094 dev_dbg(&pl08x->adev->dev, in pl08x_prep_dma_cyclic()
2098 plchan->name); in pl08x_prep_dma_cyclic()
2104 txd->cyclic = true; in pl08x_prep_dma_cyclic()
2105 txd->cctl |= PL080_CONTROL_TC_IRQ_EN; in pl08x_prep_dma_cyclic()
2116 ret = pl08x_fill_llis_for_desc(plchan->host, txd); in pl08x_prep_dma_cyclic()
2123 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); in pl08x_prep_dma_cyclic()
2130 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_config()
2132 if (!plchan->slave) in pl08x_config()
2133 return -EINVAL; in pl08x_config()
2136 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || in pl08x_config()
2137 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) in pl08x_config()
2138 return -EINVAL; in pl08x_config()
2140 if (config->device_fc && pl08x->vd->pl080s) { in pl08x_config()
2141 dev_err(&pl08x->adev->dev, in pl08x_config()
2144 return -EINVAL; in pl08x_config()
2147 plchan->cfg = *config; in pl08x_config()
2155 struct pl08x_driver_data *pl08x = plchan->host; in pl08x_terminate_all()
2158 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_terminate_all()
2159 if (!plchan->phychan && !plchan->at) { in pl08x_terminate_all()
2160 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_terminate_all()
2164 plchan->state = PL08X_CHAN_IDLE; in pl08x_terminate_all()
2166 if (plchan->phychan) { in pl08x_terminate_all()
2174 if (plchan->at) { in pl08x_terminate_all()
2175 vchan_terminate_vdesc(&plchan->at->vd); in pl08x_terminate_all()
2176 plchan->at = NULL; in pl08x_terminate_all()
2181 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_terminate_all()
2190 vchan_synchronize(&plchan->vc); in pl08x_synchronize()
2202 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_pause()
2203 if (!plchan->phychan && !plchan->at) { in pl08x_pause()
2204 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_pause()
2208 pl08x_pause_phy_chan(plchan->phychan); in pl08x_pause()
2209 plchan->state = PL08X_CHAN_PAUSED; in pl08x_pause()
2211 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_pause()
2225 spin_lock_irqsave(&plchan->vc.lock, flags); in pl08x_resume()
2226 if (!plchan->phychan && !plchan->at) { in pl08x_resume()
2227 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_resume()
2231 pl08x_resume_phy_chan(plchan->phychan); in pl08x_resume()
2232 plchan->state = PL08X_CHAN_RUNNING; in pl08x_resume()
2234 spin_unlock_irqrestore(&plchan->vc.lock, flags); in pl08x_resume()
2245 if (chan->device->dev->driver != &pl08x_amba_driver.drv) in pl08x_filter_id()
2251 if (!strcmp(plchan->name, name)) in pl08x_filter_id()
2262 return plchan->cd == chan_id; in pl08x_filter_fn()
2274 if (pl08x->vd->nomadik) in pl08x_ensure_on()
2277 if (pl08x->vd->ftdmac020) { in pl08x_ensure_on()
2278 writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR); in pl08x_ensure_on()
2281 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); in pl08x_ensure_on()
2289 /* check & clear - ERR & TC interrupts */ in pl08x_irq()
2290 err = readl(pl08x->base + PL080_ERR_STATUS); in pl08x_irq()
2292 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", in pl08x_irq()
2294 writel(err, pl08x->base + PL080_ERR_CLEAR); in pl08x_irq()
2296 tc = readl(pl08x->base + PL080_TC_STATUS); in pl08x_irq()
2298 writel(tc, pl08x->base + PL080_TC_CLEAR); in pl08x_irq()
2303 for (i = 0; i < pl08x->vd->channels; i++) { in pl08x_irq()
2306 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; in pl08x_irq()
2307 struct pl08x_dma_chan *plchan = phychan->serving; in pl08x_irq()
2311 dev_err(&pl08x->adev->dev, in pl08x_irq()
2317 spin_lock(&plchan->vc.lock); in pl08x_irq()
2318 tx = plchan->at; in pl08x_irq()
2319 if (tx && tx->cyclic) { in pl08x_irq()
2320 vchan_cyclic_callback(&tx->vd); in pl08x_irq()
2322 plchan->at = NULL; in pl08x_irq()
2328 tx->done = true; in pl08x_irq()
2329 vchan_cookie_complete(&tx->vd); in pl08x_irq()
2335 if (vchan_next_desc(&plchan->vc)) in pl08x_irq()
2340 spin_unlock(&plchan->vc.lock); in pl08x_irq()
2351 chan->slave = true; in pl08x_dma_slave_init()
2352 chan->name = chan->cd->bus_id; in pl08x_dma_slave_init()
2353 chan->cfg.src_addr = chan->cd->addr; in pl08x_dma_slave_init()
2354 chan->cfg.dst_addr = chan->cd->addr; in pl08x_dma_slave_init()
2367 INIT_LIST_HEAD(&dmadev->channels); in pl08x_dma_init_virtual_channels()
2377 return -ENOMEM; in pl08x_dma_init_virtual_channels()
2379 chan->host = pl08x; in pl08x_dma_init_virtual_channels()
2380 chan->state = PL08X_CHAN_IDLE; in pl08x_dma_init_virtual_channels()
2381 chan->signal = -1; in pl08x_dma_init_virtual_channels()
2384 chan->cd = &pl08x->pd->slave_channels[i]; in pl08x_dma_init_virtual_channels()
2390 chan->signal = i; in pl08x_dma_init_virtual_channels()
2393 chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL); in pl08x_dma_init_virtual_channels()
2394 if (!chan->cd) { in pl08x_dma_init_virtual_channels()
2396 return -ENOMEM; in pl08x_dma_init_virtual_channels()
2398 chan->cd->bus_id = "memcpy"; in pl08x_dma_init_virtual_channels()
2399 chan->cd->periph_buses = pl08x->pd->mem_buses; in pl08x_dma_init_virtual_channels()
2400 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); in pl08x_dma_init_virtual_channels()
2401 if (!chan->name) { in pl08x_dma_init_virtual_channels()
2402 kfree(chan->cd); in pl08x_dma_init_virtual_channels()
2404 return -ENOMEM; in pl08x_dma_init_virtual_channels()
2407 dev_dbg(&pl08x->adev->dev, in pl08x_dma_init_virtual_channels()
2409 chan->name); in pl08x_dma_init_virtual_channels()
2411 chan->vc.desc_free = pl08x_desc_free; in pl08x_dma_init_virtual_channels()
2412 vchan_init(&chan->vc, dmadev); in pl08x_dma_init_virtual_channels()
2414 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", in pl08x_dma_init_virtual_channels()
2425 next, &dmadev->channels, vc.chan.device_node) { in pl08x_free_virtual_channels()
2426 list_del(&chan->vc.chan.device_node); in pl08x_free_virtual_channels()
2451 struct pl08x_driver_data *pl08x = s->private; in pl08x_debugfs_show()
2459 seq_printf(s, "--------\t-----\n"); in pl08x_debugfs_show()
2460 for (i = 0; i < pl08x->vd->channels; i++) { in pl08x_debugfs_show()
2463 ch = &pl08x->phy_chans[i]; in pl08x_debugfs_show()
2465 spin_lock_irqsave(&ch->lock, flags); in pl08x_debugfs_show()
2466 virt_chan = ch->serving; in pl08x_debugfs_show()
2469 ch->id, in pl08x_debugfs_show()
2470 virt_chan ? virt_chan->name : "(none)", in pl08x_debugfs_show()
2471 ch->locked ? " LOCKED" : ""); in pl08x_debugfs_show()
2473 spin_unlock_irqrestore(&ch->lock, flags); in pl08x_debugfs_show()
2478 seq_printf(s, "--------\t------\n"); in pl08x_debugfs_show()
2479 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { in pl08x_debugfs_show()
2480 seq_printf(s, "%s\t\t%s\n", chan->name, in pl08x_debugfs_show()
2481 pl08x_state_str(chan->state)); in pl08x_debugfs_show()
2484 if (pl08x->has_slave) { in pl08x_debugfs_show()
2487 seq_printf(s, "--------\t------\n"); in pl08x_debugfs_show()
2488 list_for_each_entry(chan, &pl08x->slave.channels, in pl08x_debugfs_show()
2490 seq_printf(s, "%s\t\t%s\n", chan->name, in pl08x_debugfs_show()
2491 pl08x_state_str(chan->state)); in pl08x_debugfs_show()
2503 debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, in init_pl08x_debugfs()
2520 if (!pl08x->has_slave) in pl08x_find_chan_id()
2523 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { in pl08x_find_chan_id()
2524 if (chan->signal == id) in pl08x_find_chan_id()
2525 return &chan->vc.chan; in pl08x_find_chan_id()
2534 struct pl08x_driver_data *pl08x = ofdma->of_dma_data; in pl08x_of_xlate()
2541 if (dma_spec->args_count != 2) { in pl08x_of_xlate()
2542 dev_err(&pl08x->adev->dev, in pl08x_of_xlate()
2547 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); in pl08x_of_xlate()
2549 dev_err(&pl08x->adev->dev, in pl08x_of_xlate()
2555 dev_dbg(&pl08x->adev->dev, in pl08x_of_xlate()
2557 dma_spec->args[0]); in pl08x_of_xlate()
2560 plchan->cd->periph_buses = dma_spec->args[1]; in pl08x_of_xlate()
2574 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); in pl08x_of_probe()
2576 return -ENOMEM; in pl08x_of_probe()
2579 if (of_property_read_bool(np, "lli-bus-interface-ahb1")) in pl08x_of_probe()
2580 pd->lli_buses |= PL08X_AHB1; in pl08x_of_probe()
2581 if (of_property_read_bool(np, "lli-bus-interface-ahb2")) in pl08x_of_probe()
2582 pd->lli_buses |= PL08X_AHB2; in pl08x_of_probe()
2583 if (!pd->lli_buses) { in pl08x_of_probe()
2584 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); in pl08x_of_probe()
2585 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; in pl08x_of_probe()
2589 if (of_property_read_bool(np, "mem-bus-interface-ahb1")) in pl08x_of_probe()
2590 pd->mem_buses |= PL08X_AHB1; in pl08x_of_probe()
2591 if (of_property_read_bool(np, "mem-bus-interface-ahb2")) in pl08x_of_probe()
2592 pd->mem_buses |= PL08X_AHB2; in pl08x_of_probe()
2593 if (!pd->mem_buses) { in pl08x_of_probe()
2594 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); in pl08x_of_probe()
2595 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; in pl08x_of_probe()
2599 ret = of_property_read_u32(np, "memcpy-burst-size", &val); in pl08x_of_probe()
2601 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); in pl08x_of_probe()
2606 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); in pl08x_of_probe()
2609 pd->memcpy_burst_size = PL08X_BURST_SZ_1; in pl08x_of_probe()
2612 pd->memcpy_burst_size = PL08X_BURST_SZ_4; in pl08x_of_probe()
2615 pd->memcpy_burst_size = PL08X_BURST_SZ_8; in pl08x_of_probe()
2618 pd->memcpy_burst_size = PL08X_BURST_SZ_16; in pl08x_of_probe()
2621 pd->memcpy_burst_size = PL08X_BURST_SZ_32; in pl08x_of_probe()
2624 pd->memcpy_burst_size = PL08X_BURST_SZ_64; in pl08x_of_probe()
2627 pd->memcpy_burst_size = PL08X_BURST_SZ_128; in pl08x_of_probe()
2630 pd->memcpy_burst_size = PL08X_BURST_SZ_256; in pl08x_of_probe()
2634 ret = of_property_read_u32(np, "memcpy-bus-width", &val); in pl08x_of_probe()
2636 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); in pl08x_of_probe()
2641 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); in pl08x_of_probe()
2644 pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS; in pl08x_of_probe()
2647 pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS; in pl08x_of_probe()
2650 pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS; in pl08x_of_probe()
2660 if (pl08x->vd->signals) { in pl08x_of_probe()
2661 chanp = devm_kcalloc(&adev->dev, in pl08x_of_probe()
2662 pl08x->vd->signals, in pl08x_of_probe()
2666 return -ENOMEM; in pl08x_of_probe()
2668 pd->slave_channels = chanp; in pl08x_of_probe()
2669 for (i = 0; i < pl08x->vd->signals; i++) { in pl08x_of_probe()
2671 * chanp->periph_buses will be assigned at translation in pl08x_of_probe()
2673 chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i); in pl08x_of_probe()
2676 pd->num_slave_channels = pl08x->vd->signals; in pl08x_of_probe()
2679 pl08x->pd = pd; in pl08x_of_probe()
2681 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, in pl08x_of_probe()
2689 return -EINVAL; in pl08x_of_probe()
2696 struct vendor_data *vd = id->data; in pl08x_probe()
2697 struct device_node *np = adev->dev.of_node; in pl08x_probe()
2707 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); in pl08x_probe()
2714 ret = -ENOMEM; in pl08x_probe()
2719 pl08x->adev = adev; in pl08x_probe()
2720 pl08x->vd = vd; in pl08x_probe()
2722 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res)); in pl08x_probe()
2723 if (!pl08x->base) { in pl08x_probe()
2724 ret = -ENOMEM; in pl08x_probe()
2728 if (vd->ftdmac020) { in pl08x_probe()
2731 val = readl(pl08x->base + FTDMAC020_REVISION); in pl08x_probe()
2732 dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n", in pl08x_probe()
2734 val = readl(pl08x->base + FTDMAC020_FEATURE); in pl08x_probe()
2735 dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, " in pl08x_probe()
2736 "%s built-in bridge, %s, %s linked lists\n", in pl08x_probe()
2744 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2746 vd->channels = (val >> 12) & 0x0f; in pl08x_probe()
2747 vd->dualmaster = !!(val & BIT(9)); in pl08x_probe()
2751 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); in pl08x_probe()
2752 pl08x->memcpy.dev = &adev->dev; in pl08x_probe()
2753 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; in pl08x_probe()
2754 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; in pl08x_probe()
2755 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; in pl08x_probe()
2756 pl08x->memcpy.device_issue_pending = pl08x_issue_pending; in pl08x_probe()
2757 pl08x->memcpy.device_config = pl08x_config; in pl08x_probe()
2758 pl08x->memcpy.device_pause = pl08x_pause; in pl08x_probe()
2759 pl08x->memcpy.device_resume = pl08x_resume; in pl08x_probe()
2760 pl08x->memcpy.device_terminate_all = pl08x_terminate_all; in pl08x_probe()
2761 pl08x->memcpy.device_synchronize = pl08x_synchronize; in pl08x_probe()
2762 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2763 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2764 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); in pl08x_probe()
2765 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; in pl08x_probe()
2766 if (vd->ftdmac020) in pl08x_probe()
2767 pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; in pl08x_probe()
2774 if (vd->signals) { in pl08x_probe()
2775 pl08x->has_slave = true; in pl08x_probe()
2776 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); in pl08x_probe()
2777 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); in pl08x_probe()
2778 pl08x->slave.dev = &adev->dev; in pl08x_probe()
2779 pl08x->slave.device_free_chan_resources = in pl08x_probe()
2781 pl08x->slave.device_tx_status = pl08x_dma_tx_status; in pl08x_probe()
2782 pl08x->slave.device_issue_pending = pl08x_issue_pending; in pl08x_probe()
2783 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; in pl08x_probe()
2784 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; in pl08x_probe()
2785 pl08x->slave.device_config = pl08x_config; in pl08x_probe()
2786 pl08x->slave.device_pause = pl08x_pause; in pl08x_probe()
2787 pl08x->slave.device_resume = pl08x_resume; in pl08x_probe()
2788 pl08x->slave.device_terminate_all = pl08x_terminate_all; in pl08x_probe()
2789 pl08x->slave.device_synchronize = pl08x_synchronize; in pl08x_probe()
2790 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2791 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; in pl08x_probe()
2792 pl08x->slave.directions = in pl08x_probe()
2794 pl08x->slave.residue_granularity = in pl08x_probe()
2799 pl08x->pd = dev_get_platdata(&adev->dev); in pl08x_probe()
2800 if (!pl08x->pd) { in pl08x_probe()
2806 dev_err(&adev->dev, "no platform data supplied\n"); in pl08x_probe()
2807 ret = -EINVAL; in pl08x_probe()
2811 pl08x->slave.filter.map = pl08x->pd->slave_map; in pl08x_probe()
2812 pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len; in pl08x_probe()
2813 pl08x->slave.filter.fn = pl08x_filter_fn; in pl08x_probe()
2817 pl08x->lli_buses = PL08X_AHB1; in pl08x_probe()
2818 pl08x->mem_buses = PL08X_AHB1; in pl08x_probe()
2819 if (pl08x->vd->dualmaster) { in pl08x_probe()
2820 pl08x->lli_buses = pl08x->pd->lli_buses; in pl08x_probe()
2821 pl08x->mem_buses = pl08x->pd->mem_buses; in pl08x_probe()
2824 if (vd->pl080s) in pl08x_probe()
2825 pl08x->lli_words = PL080S_LLI_WORDS; in pl08x_probe()
2827 pl08x->lli_words = PL080_LLI_WORDS; in pl08x_probe()
2828 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); in pl08x_probe()
2830 /* A DMA memory pool for LLIs, align on 1-byte boundary */ in pl08x_probe()
2831 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, in pl08x_probe()
2833 if (!pl08x->pool) { in pl08x_probe()
2834 ret = -ENOMEM; in pl08x_probe()
2842 if (vd->ftdmac020) in pl08x_probe()
2843 /* This variant has error IRQs in bits 16-19 */ in pl08x_probe()
2844 writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR); in pl08x_probe()
2846 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); in pl08x_probe()
2847 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); in pl08x_probe()
2850 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); in pl08x_probe()
2852 dev_err(&adev->dev, "%s failed to request interrupt %d\n", in pl08x_probe()
2853 __func__, adev->irq[0]); in pl08x_probe()
2858 pl08x->phy_chans = kcalloc(vd->channels, sizeof(*pl08x->phy_chans), in pl08x_probe()
2860 if (!pl08x->phy_chans) { in pl08x_probe()
2861 ret = -ENOMEM; in pl08x_probe()
2865 for (i = 0; i < vd->channels; i++) { in pl08x_probe()
2866 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; in pl08x_probe()
2868 ch->id = i; in pl08x_probe()
2869 ch->base = pl08x->base + PL080_Cx_BASE(i); in pl08x_probe()
2870 if (vd->ftdmac020) { in pl08x_probe()
2872 ch->reg_busy = ch->base + FTDMAC020_CH_BUSY; in pl08x_probe()
2873 ch->reg_config = ch->base + FTDMAC020_CH_CFG; in pl08x_probe()
2874 ch->reg_control = ch->base + FTDMAC020_CH_CSR; in pl08x_probe()
2875 ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR; in pl08x_probe()
2876 ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR; in pl08x_probe()
2877 ch->reg_lli = ch->base + FTDMAC020_CH_LLP; in pl08x_probe()
2878 ch->ftdmac020 = true; in pl08x_probe()
2880 ch->reg_config = ch->base + vd->config_offset; in pl08x_probe()
2881 ch->reg_control = ch->base + PL080_CH_CONTROL; in pl08x_probe()
2882 ch->reg_src = ch->base + PL080_CH_SRC_ADDR; in pl08x_probe()
2883 ch->reg_dst = ch->base + PL080_CH_DST_ADDR; in pl08x_probe()
2884 ch->reg_lli = ch->base + PL080_CH_LLI; in pl08x_probe()
2886 if (vd->pl080s) in pl08x_probe()
2887 ch->pl080s = true; in pl08x_probe()
2889 spin_lock_init(&ch->lock); in pl08x_probe()
2896 if (vd->nomadik) { in pl08x_probe()
2899 val = readl(ch->reg_config); in pl08x_probe()
2901 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i); in pl08x_probe()
2902 ch->locked = true; in pl08x_probe()
2906 dev_dbg(&adev->dev, "physical channel %d is %s\n", in pl08x_probe()
2911 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, in pl08x_probe()
2912 pl08x->vd->channels, false); in pl08x_probe()
2914 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2915 "%s failed to enumerate memcpy channels - %d\n", in pl08x_probe()
2921 if (pl08x->has_slave) { in pl08x_probe()
2922 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, in pl08x_probe()
2923 pl08x->pd->num_slave_channels, true); in pl08x_probe()
2925 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2926 "%s failed to enumerate slave channels - %d\n", in pl08x_probe()
2932 ret = dma_async_device_register(&pl08x->memcpy); in pl08x_probe()
2934 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2935 "%s failed to register memcpy as an async device - %d\n", in pl08x_probe()
2940 if (pl08x->has_slave) { in pl08x_probe()
2941 ret = dma_async_device_register(&pl08x->slave); in pl08x_probe()
2943 dev_warn(&pl08x->adev->dev, in pl08x_probe()
2944 "%s failed to register slave as an async device - %d\n", in pl08x_probe()
2952 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n", in pl08x_probe()
2953 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev), in pl08x_probe()
2954 (unsigned long long)adev->res.start, adev->irq[0]); in pl08x_probe()
2959 dma_async_device_unregister(&pl08x->memcpy); in pl08x_probe()
2961 if (pl08x->has_slave) in pl08x_probe()
2962 pl08x_free_virtual_channels(&pl08x->slave); in pl08x_probe()
2964 pl08x_free_virtual_channels(&pl08x->memcpy); in pl08x_probe()
2966 kfree(pl08x->phy_chans); in pl08x_probe()
2968 free_irq(adev->irq[0], pl08x); in pl08x_probe()
2970 dma_pool_destroy(pl08x->pool); in pl08x_probe()
2973 iounmap(pl08x->base); in pl08x_probe()