1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (c) 2006 ARM Ltd. 4 * Copyright (c) 2010 ST-Ericsson SA 5 * Copyirght (c) 2017 Linaro Ltd. 6 * 7 * Author: Peter Pearse <peter.pearse@arm.com> 8 * Author: Linus Walleij <linus.walleij@linaro.org> 9 * 10 * Documentation: ARM DDI 0196G == PL080 11 * Documentation: ARM DDI 0218E == PL081 12 * Documentation: S3C6410 User's Manual == PL080S 13 * 14 * PL080 & PL081 both have 16 sets of DMA signals that can be routed to any 15 * channel. 16 * 17 * The PL080 has 8 channels available for simultaneous use, and the PL081 18 * has only two channels. So on these DMA controllers the number of channels 19 * and the number of incoming DMA signals are two totally different things. 20 * It is usually not possible to theoretically handle all physical signals, 21 * so a multiplexing scheme with possible denial of use is necessary. 22 * 23 * The PL080 has a dual bus master, PL081 has a single master. 24 * 25 * PL080S is a version modified by Samsung and used in S3C64xx SoCs. 26 * It differs in following aspects: 27 * - CH_CONFIG register at different offset, 28 * - separate CH_CONTROL2 register for transfer size, 29 * - bigger maximum transfer size, 30 * - 8-word aligned LLI, instead of 4-word, due to extra CCTL2 word, 31 * - no support for peripheral flow control. 32 * 33 * Memory to peripheral transfer may be visualized as 34 * Get data from memory to DMAC 35 * Until no data left 36 * On burst request from peripheral 37 * Destination burst from DMAC to peripheral 38 * Clear burst request 39 * Raise terminal count interrupt 40 * 41 * For peripherals with a FIFO: 42 * Source burst size == half the depth of the peripheral FIFO 43 * Destination burst size == the depth of the peripheral FIFO 44 * 45 * (Bursts are irrelevant for mem to mem transfers - there are no burst 46 * signals, the DMA controller will simply facilitate its AHB master.) 47 * 48 * ASSUMES default (little) endianness for DMA transfers 49 * 50 * The PL08x has two flow control settings: 51 * - DMAC flow control: the transfer size defines the number of transfers 52 * which occur for the current LLI entry, and the DMAC raises TC at the 53 * end of every LLI entry. Observed behaviour shows the DMAC listening 54 * to both the BREQ and SREQ signals (contrary to documented), 55 * transferring data if either is active. The LBREQ and LSREQ signals 56 * are ignored. 57 * 58 * - Peripheral flow control: the transfer size is ignored (and should be 59 * zero). The data is transferred from the current LLI entry, until 60 * after the final transfer signalled by LBREQ or LSREQ. The DMAC 61 * will then move to the next LLI entry. Unsupported by PL080S. 62 */ 63 #include <linux/amba/bus.h> 64 #include <linux/amba/pl08x.h> 65 #include <linux/debugfs.h> 66 #include <linux/delay.h> 67 #include <linux/device.h> 68 #include <linux/dmaengine.h> 69 #include <linux/dmapool.h> 70 #include <linux/dma-mapping.h> 71 #include <linux/export.h> 72 #include <linux/init.h> 73 #include <linux/interrupt.h> 74 #include <linux/module.h> 75 #include <linux/of.h> 76 #include <linux/of_dma.h> 77 #include <linux/pm_runtime.h> 78 #include <linux/seq_file.h> 79 #include <linux/slab.h> 80 #include <linux/amba/pl080.h> 81 82 #include "dmaengine.h" 83 #include "virt-dma.h" 84 85 #define DRIVER_NAME "pl08xdmac" 86 87 #define PL80X_DMA_BUSWIDTHS \ 88 BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ 89 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ 90 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ 91 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) 92 93 static struct amba_driver pl08x_amba_driver; 94 struct pl08x_driver_data; 95 96 /** 97 * struct vendor_data - vendor-specific config parameters for PL08x derivatives 98 * @config_offset: offset to the configuration register 99 * @channels: the number of channels available in this variant 100 * @signals: the number of request signals available from the hardware 101 * @dualmaster: whether this version supports dual AHB masters or not. 102 * @nomadik: whether this variant is a ST Microelectronics Nomadik, where the 103 * channels have Nomadik security extension bits that need to be checked 104 * for permission before use and some registers are missing 105 * @pl080s: whether this variant is a Samsung PL080S, which has separate 106 * register and LLI word for transfer size. 107 * @ftdmac020: whether this variant is a Faraday Technology FTDMAC020 108 * @max_transfer_size: the maximum single element transfer size for this 109 * PL08x variant. 110 */ 111 struct vendor_data { 112 u8 config_offset; 113 u8 channels; 114 u8 signals; 115 bool dualmaster; 116 bool nomadik; 117 bool pl080s; 118 bool ftdmac020; 119 u32 max_transfer_size; 120 }; 121 122 /** 123 * struct pl08x_bus_data - information of source or destination 124 * busses for a transfer 125 * @addr: current address 126 * @maxwidth: the maximum width of a transfer on this bus 127 * @buswidth: the width of this bus in bytes: 1, 2 or 4 128 */ 129 struct pl08x_bus_data { 130 dma_addr_t addr; 131 u8 maxwidth; 132 u8 buswidth; 133 }; 134 135 #define IS_BUS_ALIGNED(bus) IS_ALIGNED((bus)->addr, (bus)->buswidth) 136 137 /** 138 * struct pl08x_phy_chan - holder for the physical channels 139 * @id: physical index to this channel 140 * @base: memory base address for this physical channel 141 * @reg_config: configuration address for this physical channel 142 * @reg_control: control address for this physical channel 143 * @reg_src: transfer source address register 144 * @reg_dst: transfer destination address register 145 * @reg_lli: transfer LLI address register 146 * @reg_busy: if the variant has a special per-channel busy register, 147 * this contains a pointer to it 148 * @lock: a lock to use when altering an instance of this struct 149 * @serving: the virtual channel currently being served by this physical 150 * channel 151 * @locked: channel unavailable for the system, e.g. dedicated to secure 152 * world 153 * @ftdmac020: channel is on a FTDMAC020 154 * @pl080s: channel is on a PL08s 155 */ 156 struct pl08x_phy_chan { 157 unsigned int id; 158 void __iomem *base; 159 void __iomem *reg_config; 160 void __iomem *reg_control; 161 void __iomem *reg_src; 162 void __iomem *reg_dst; 163 void __iomem *reg_lli; 164 void __iomem *reg_busy; 165 spinlock_t lock; 166 struct pl08x_dma_chan *serving; 167 bool locked; 168 bool ftdmac020; 169 bool pl080s; 170 }; 171 172 /** 173 * struct pl08x_sg - structure containing data per sg 174 * @src_addr: src address of sg 175 * @dst_addr: dst address of sg 176 * @len: transfer len in bytes 177 * @node: node for txd's dsg_list 178 */ 179 struct pl08x_sg { 180 dma_addr_t src_addr; 181 dma_addr_t dst_addr; 182 size_t len; 183 struct list_head node; 184 }; 185 186 /** 187 * struct pl08x_txd - wrapper for struct dma_async_tx_descriptor 188 * @vd: virtual DMA descriptor 189 * @dsg_list: list of children sg's 190 * @llis_bus: DMA memory address (physical) start for the LLIs 191 * @llis_va: virtual memory address start for the LLIs 192 * @cctl: control reg values for current txd 193 * @ccfg: config reg values for current txd 194 * @done: this marks completed descriptors, which should not have their 195 * mux released. 196 * @cyclic: indicate cyclic transfers 197 */ 198 struct pl08x_txd { 199 struct virt_dma_desc vd; 200 struct list_head dsg_list; 201 dma_addr_t llis_bus; 202 u32 *llis_va; 203 /* Default cctl value for LLIs */ 204 u32 cctl; 205 /* 206 * Settings to be put into the physical channel when we 207 * trigger this txd. Other registers are in llis_va[0]. 208 */ 209 u32 ccfg; 210 bool done; 211 bool cyclic; 212 }; 213 214 /** 215 * enum pl08x_dma_chan_state - holds the PL08x specific virtual channel 216 * states 217 * @PL08X_CHAN_IDLE: the channel is idle 218 * @PL08X_CHAN_RUNNING: the channel has allocated a physical transport 219 * channel and is running a transfer on it 220 * @PL08X_CHAN_PAUSED: the channel has allocated a physical transport 221 * channel, but the transfer is currently paused 222 * @PL08X_CHAN_WAITING: the channel is waiting for a physical transport 223 * channel to become available (only pertains to memcpy channels) 224 */ 225 enum pl08x_dma_chan_state { 226 PL08X_CHAN_IDLE, 227 PL08X_CHAN_RUNNING, 228 PL08X_CHAN_PAUSED, 229 PL08X_CHAN_WAITING, 230 }; 231 232 /** 233 * struct pl08x_dma_chan - this structure wraps a DMA ENGINE channel 234 * @vc: wrapped virtual channel 235 * @phychan: the physical channel utilized by this channel, if there is one 236 * @name: name of channel 237 * @cd: channel platform data 238 * @cfg: slave configuration 239 * @at: active transaction on this channel 240 * @host: a pointer to the host (internal use) 241 * @state: whether the channel is idle, paused, running etc 242 * @slave: whether this channel is a device (slave) or for memcpy 243 * @signal: the physical DMA request signal which this channel is using 244 * @mux_use: count of descriptors using this DMA request signal setting 245 * @waiting_at: time in jiffies when this channel moved to waiting state 246 */ 247 struct pl08x_dma_chan { 248 struct virt_dma_chan vc; 249 struct pl08x_phy_chan *phychan; 250 const char *name; 251 struct pl08x_channel_data *cd; 252 struct dma_slave_config cfg; 253 struct pl08x_txd *at; 254 struct pl08x_driver_data *host; 255 enum pl08x_dma_chan_state state; 256 bool slave; 257 int signal; 258 unsigned mux_use; 259 unsigned long waiting_at; 260 }; 261 262 /** 263 * struct pl08x_driver_data - the local state holder for the PL08x 264 * @slave: optional slave engine for this instance 265 * @memcpy: memcpy engine for this instance 266 * @has_slave: the PL08x has a slave engine (routed signals) 267 * @base: virtual memory base (remapped) for the PL08x 268 * @adev: the corresponding AMBA (PrimeCell) bus entry 269 * @vd: vendor data for this PL08x variant 270 * @pd: platform data passed in from the platform/machine 271 * @phy_chans: array of data for the physical channels 272 * @pool: a pool for the LLI descriptors 273 * @lli_buses: bitmask to or in to LLI pointer selecting AHB port for LLI 274 * fetches 275 * @mem_buses: set to indicate memory transfers on AHB2. 276 * @lli_words: how many words are used in each LLI item for this variant 277 */ 278 struct pl08x_driver_data { 279 struct dma_device slave; 280 struct dma_device memcpy; 281 bool has_slave; 282 void __iomem *base; 283 struct amba_device *adev; 284 const struct vendor_data *vd; 285 struct pl08x_platform_data *pd; 286 struct pl08x_phy_chan *phy_chans; 287 struct dma_pool *pool; 288 u8 lli_buses; 289 u8 mem_buses; 290 u8 lli_words; 291 }; 292 293 /* 294 * PL08X specific defines 295 */ 296 297 /* The order of words in an LLI. */ 298 #define PL080_LLI_SRC 0 299 #define PL080_LLI_DST 1 300 #define PL080_LLI_LLI 2 301 #define PL080_LLI_CCTL 3 302 #define PL080S_LLI_CCTL2 4 303 304 /* Total words in an LLI. */ 305 #define PL080_LLI_WORDS 4 306 #define PL080S_LLI_WORDS 8 307 308 /* 309 * Number of LLIs in each LLI buffer allocated for one transfer 310 * (maximum times we call dma_pool_alloc on this pool without freeing) 311 */ 312 #define MAX_NUM_TSFR_LLIS 512 313 #define PL08X_ALIGN 8 314 315 static inline struct pl08x_dma_chan *to_pl08x_chan(struct dma_chan *chan) 316 { 317 return container_of(chan, struct pl08x_dma_chan, vc.chan); 318 } 319 320 static inline struct pl08x_txd *to_pl08x_txd(struct dma_async_tx_descriptor *tx) 321 { 322 return container_of(tx, struct pl08x_txd, vd.tx); 323 } 324 325 /* 326 * Mux handling. 327 * 328 * This gives us the DMA request input to the PL08x primecell which the 329 * peripheral described by the channel data will be routed to, possibly 330 * via a board/SoC specific external MUX. One important point to note 331 * here is that this does not depend on the physical channel. 332 */ 333 static int pl08x_request_mux(struct pl08x_dma_chan *plchan) 334 { 335 const struct pl08x_platform_data *pd = plchan->host->pd; 336 int ret; 337 338 if (plchan->mux_use++ == 0 && pd->get_xfer_signal) { 339 ret = pd->get_xfer_signal(plchan->cd); 340 if (ret < 0) { 341 plchan->mux_use = 0; 342 return ret; 343 } 344 345 plchan->signal = ret; 346 } 347 return 0; 348 } 349 350 static void pl08x_release_mux(struct pl08x_dma_chan *plchan) 351 { 352 const struct pl08x_platform_data *pd = plchan->host->pd; 353 354 if (plchan->signal >= 0) { 355 WARN_ON(plchan->mux_use == 0); 356 357 if (--plchan->mux_use == 0 && pd->put_xfer_signal) { 358 pd->put_xfer_signal(plchan->cd, plchan->signal); 359 plchan->signal = -1; 360 } 361 } 362 } 363 364 /* 365 * Physical channel handling 366 */ 367 368 /* Whether a certain channel is busy or not */ 369 static int pl08x_phy_channel_busy(struct pl08x_phy_chan *ch) 370 { 371 unsigned int val; 372 373 /* If we have a special busy register, take a shortcut */ 374 if (ch->reg_busy) { 375 val = readl(ch->reg_busy); 376 return !!(val & BIT(ch->id)); 377 } 378 val = readl(ch->reg_config); 379 return val & PL080_CONFIG_ACTIVE; 380 } 381 382 /* 383 * pl08x_write_lli() - Write an LLI into the DMA controller. 384 * 385 * The PL08x derivatives support linked lists, but the first item of the 386 * list containing the source, destination, control word and next LLI is 387 * ignored. Instead the driver has to write those values directly into the 388 * SRC, DST, LLI and control registers. On FTDMAC020 also the SIZE 389 * register need to be set up for the first transfer. 390 */ 391 static void pl08x_write_lli(struct pl08x_driver_data *pl08x, 392 struct pl08x_phy_chan *phychan, const u32 *lli, u32 ccfg) 393 { 394 if (pl08x->vd->pl080s) 395 dev_vdbg(&pl08x->adev->dev, 396 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 397 "clli=0x%08x, cctl=0x%08x, cctl2=0x%08x, ccfg=0x%08x\n", 398 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 399 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], 400 lli[PL080S_LLI_CCTL2], ccfg); 401 else 402 dev_vdbg(&pl08x->adev->dev, 403 "WRITE channel %d: csrc=0x%08x, cdst=0x%08x, " 404 "clli=0x%08x, cctl=0x%08x, ccfg=0x%08x\n", 405 phychan->id, lli[PL080_LLI_SRC], lli[PL080_LLI_DST], 406 lli[PL080_LLI_LLI], lli[PL080_LLI_CCTL], ccfg); 407 408 writel_relaxed(lli[PL080_LLI_SRC], phychan->reg_src); 409 writel_relaxed(lli[PL080_LLI_DST], phychan->reg_dst); 410 writel_relaxed(lli[PL080_LLI_LLI], phychan->reg_lli); 411 412 /* 413 * The FTMAC020 has a different layout in the CCTL word of the LLI 414 * and the CCTL register which is split in CSR and SIZE registers. 415 * Convert the LLI item CCTL into the proper values to write into 416 * the CSR and SIZE registers. 417 */ 418 if (phychan->ftdmac020) { 419 u32 llictl = lli[PL080_LLI_CCTL]; 420 u32 val = 0; 421 422 /* Write the transfer size (12 bits) to the size register */ 423 writel_relaxed(llictl & FTDMAC020_LLI_TRANSFER_SIZE_MASK, 424 phychan->base + FTDMAC020_CH_SIZE); 425 /* 426 * Then write the control bits 28..16 to the control register 427 * by shuffleing the bits around to where they are in the 428 * main register. The mapping is as follows: 429 * Bit 28: TC_MSK - mask on all except last LLI 430 * Bit 27..25: SRC_WIDTH 431 * Bit 24..22: DST_WIDTH 432 * Bit 21..20: SRCAD_CTRL 433 * Bit 19..17: DSTAD_CTRL 434 * Bit 17: SRC_SEL 435 * Bit 16: DST_SEL 436 */ 437 if (llictl & FTDMAC020_LLI_TC_MSK) 438 val |= FTDMAC020_CH_CSR_TC_MSK; 439 val |= ((llictl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> 440 (FTDMAC020_LLI_SRC_WIDTH_SHIFT - 441 FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT)); 442 val |= ((llictl & FTDMAC020_LLI_DST_WIDTH_MSK) >> 443 (FTDMAC020_LLI_DST_WIDTH_SHIFT - 444 FTDMAC020_CH_CSR_DST_WIDTH_SHIFT)); 445 val |= ((llictl & FTDMAC020_LLI_SRCAD_CTL_MSK) >> 446 (FTDMAC020_LLI_SRCAD_CTL_SHIFT - 447 FTDMAC020_CH_CSR_SRCAD_CTL_SHIFT)); 448 val |= ((llictl & FTDMAC020_LLI_DSTAD_CTL_MSK) >> 449 (FTDMAC020_LLI_DSTAD_CTL_SHIFT - 450 FTDMAC020_CH_CSR_DSTAD_CTL_SHIFT)); 451 if (llictl & FTDMAC020_LLI_SRC_SEL) 452 val |= FTDMAC020_CH_CSR_SRC_SEL; 453 if (llictl & FTDMAC020_LLI_DST_SEL) 454 val |= FTDMAC020_CH_CSR_DST_SEL; 455 456 /* 457 * Set up the bits that exist in the CSR but are not 458 * part the LLI, i.e. only gets written to the control 459 * register right here. 460 * 461 * FIXME: do not just handle memcpy, also handle slave DMA. 462 */ 463 switch (pl08x->pd->memcpy_burst_size) { 464 default: 465 case PL08X_BURST_SZ_1: 466 val |= PL080_BSIZE_1 << 467 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 468 break; 469 case PL08X_BURST_SZ_4: 470 val |= PL080_BSIZE_4 << 471 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 472 break; 473 case PL08X_BURST_SZ_8: 474 val |= PL080_BSIZE_8 << 475 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 476 break; 477 case PL08X_BURST_SZ_16: 478 val |= PL080_BSIZE_16 << 479 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 480 break; 481 case PL08X_BURST_SZ_32: 482 val |= PL080_BSIZE_32 << 483 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 484 break; 485 case PL08X_BURST_SZ_64: 486 val |= PL080_BSIZE_64 << 487 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 488 break; 489 case PL08X_BURST_SZ_128: 490 val |= PL080_BSIZE_128 << 491 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 492 break; 493 case PL08X_BURST_SZ_256: 494 val |= PL080_BSIZE_256 << 495 FTDMAC020_CH_CSR_SRC_SIZE_SHIFT; 496 break; 497 } 498 499 /* Protection flags */ 500 if (pl08x->pd->memcpy_prot_buff) 501 val |= FTDMAC020_CH_CSR_PROT2; 502 if (pl08x->pd->memcpy_prot_cache) 503 val |= FTDMAC020_CH_CSR_PROT3; 504 /* We are the kernel, so we are in privileged mode */ 505 val |= FTDMAC020_CH_CSR_PROT1; 506 507 writel_relaxed(val, phychan->reg_control); 508 } else { 509 /* Bits are just identical */ 510 writel_relaxed(lli[PL080_LLI_CCTL], phychan->reg_control); 511 } 512 513 /* Second control word on the PL080s */ 514 if (pl08x->vd->pl080s) 515 writel_relaxed(lli[PL080S_LLI_CCTL2], 516 phychan->base + PL080S_CH_CONTROL2); 517 518 writel(ccfg, phychan->reg_config); 519 } 520 521 /* 522 * Set the initial DMA register values i.e. those for the first LLI 523 * The next LLI pointer and the configuration interrupt bit have 524 * been set when the LLIs were constructed. Poke them into the hardware 525 * and start the transfer. 526 */ 527 static void pl08x_start_next_txd(struct pl08x_dma_chan *plchan) 528 { 529 struct pl08x_driver_data *pl08x = plchan->host; 530 struct pl08x_phy_chan *phychan = plchan->phychan; 531 struct virt_dma_desc *vd = vchan_next_desc(&plchan->vc); 532 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 533 u32 val; 534 535 list_del(&txd->vd.node); 536 537 plchan->at = txd; 538 539 /* Wait for channel inactive */ 540 while (pl08x_phy_channel_busy(phychan)) 541 cpu_relax(); 542 543 pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg); 544 545 /* Enable the DMA channel */ 546 /* Do not access config register until channel shows as disabled */ 547 while (readl(pl08x->base + PL080_EN_CHAN) & BIT(phychan->id)) 548 cpu_relax(); 549 550 /* Do not access config register until channel shows as inactive */ 551 if (phychan->ftdmac020) { 552 val = readl(phychan->reg_config); 553 while (val & FTDMAC020_CH_CFG_BUSY) 554 val = readl(phychan->reg_config); 555 556 val = readl(phychan->reg_control); 557 while (val & FTDMAC020_CH_CSR_EN) 558 val = readl(phychan->reg_control); 559 560 writel(val | FTDMAC020_CH_CSR_EN, 561 phychan->reg_control); 562 } else { 563 val = readl(phychan->reg_config); 564 while ((val & PL080_CONFIG_ACTIVE) || 565 (val & PL080_CONFIG_ENABLE)) 566 val = readl(phychan->reg_config); 567 568 writel(val | PL080_CONFIG_ENABLE, phychan->reg_config); 569 } 570 } 571 572 /* 573 * Pause the channel by setting the HALT bit. 574 * 575 * For M->P transfers, pause the DMAC first and then stop the peripheral - 576 * the FIFO can only drain if the peripheral is still requesting data. 577 * (note: this can still timeout if the DMAC FIFO never drains of data.) 578 * 579 * For P->M transfers, disable the peripheral first to stop it filling 580 * the DMAC FIFO, and then pause the DMAC. 581 */ 582 static void pl08x_pause_phy_chan(struct pl08x_phy_chan *ch) 583 { 584 u32 val; 585 int timeout; 586 587 if (ch->ftdmac020) { 588 /* Use the enable bit on the FTDMAC020 */ 589 val = readl(ch->reg_control); 590 val &= ~FTDMAC020_CH_CSR_EN; 591 writel(val, ch->reg_control); 592 return; 593 } 594 595 /* Set the HALT bit and wait for the FIFO to drain */ 596 val = readl(ch->reg_config); 597 val |= PL080_CONFIG_HALT; 598 writel(val, ch->reg_config); 599 600 /* Wait for channel inactive */ 601 for (timeout = 1000; timeout; timeout--) { 602 if (!pl08x_phy_channel_busy(ch)) 603 break; 604 udelay(1); 605 } 606 if (pl08x_phy_channel_busy(ch)) 607 pr_err("pl08x: channel%u timeout waiting for pause\n", ch->id); 608 } 609 610 static void pl08x_resume_phy_chan(struct pl08x_phy_chan *ch) 611 { 612 u32 val; 613 614 /* Use the enable bit on the FTDMAC020 */ 615 if (ch->ftdmac020) { 616 val = readl(ch->reg_control); 617 val |= FTDMAC020_CH_CSR_EN; 618 writel(val, ch->reg_control); 619 return; 620 } 621 622 /* Clear the HALT bit */ 623 val = readl(ch->reg_config); 624 val &= ~PL080_CONFIG_HALT; 625 writel(val, ch->reg_config); 626 } 627 628 /* 629 * pl08x_terminate_phy_chan() stops the channel, clears the FIFO and 630 * clears any pending interrupt status. This should not be used for 631 * an on-going transfer, but as a method of shutting down a channel 632 * (eg, when it's no longer used) or terminating a transfer. 633 */ 634 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x, 635 struct pl08x_phy_chan *ch) 636 { 637 u32 val; 638 639 /* The layout for the FTDMAC020 is different */ 640 if (ch->ftdmac020) { 641 /* Disable all interrupts */ 642 val = readl(ch->reg_config); 643 val |= (FTDMAC020_CH_CFG_INT_ABT_MASK | 644 FTDMAC020_CH_CFG_INT_ERR_MASK | 645 FTDMAC020_CH_CFG_INT_TC_MASK); 646 writel(val, ch->reg_config); 647 648 /* Abort and disable channel */ 649 val = readl(ch->reg_control); 650 val &= ~FTDMAC020_CH_CSR_EN; 651 val |= FTDMAC020_CH_CSR_ABT; 652 writel(val, ch->reg_control); 653 654 /* Clear ABT and ERR interrupt flags */ 655 writel(BIT(ch->id) | BIT(ch->id + 16), 656 pl08x->base + PL080_ERR_CLEAR); 657 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); 658 659 return; 660 } 661 662 val = readl(ch->reg_config); 663 val &= ~(PL080_CONFIG_ENABLE | PL080_CONFIG_ERR_IRQ_MASK | 664 PL080_CONFIG_TC_IRQ_MASK); 665 writel(val, ch->reg_config); 666 667 writel(BIT(ch->id), pl08x->base + PL080_ERR_CLEAR); 668 writel(BIT(ch->id), pl08x->base + PL080_TC_CLEAR); 669 } 670 671 static u32 get_bytes_in_phy_channel(struct pl08x_phy_chan *ch) 672 { 673 u32 val; 674 u32 bytes; 675 676 if (ch->ftdmac020) { 677 bytes = readl(ch->base + FTDMAC020_CH_SIZE); 678 679 val = readl(ch->reg_control); 680 val &= FTDMAC020_CH_CSR_SRC_WIDTH_MSK; 681 val >>= FTDMAC020_CH_CSR_SRC_WIDTH_SHIFT; 682 } else if (ch->pl080s) { 683 val = readl(ch->base + PL080S_CH_CONTROL2); 684 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; 685 686 val = readl(ch->reg_control); 687 val &= PL080_CONTROL_SWIDTH_MASK; 688 val >>= PL080_CONTROL_SWIDTH_SHIFT; 689 } else { 690 /* Plain PL08x */ 691 val = readl(ch->reg_control); 692 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; 693 694 val &= PL080_CONTROL_SWIDTH_MASK; 695 val >>= PL080_CONTROL_SWIDTH_SHIFT; 696 } 697 698 switch (val) { 699 case PL080_WIDTH_8BIT: 700 break; 701 case PL080_WIDTH_16BIT: 702 bytes *= 2; 703 break; 704 case PL080_WIDTH_32BIT: 705 bytes *= 4; 706 break; 707 } 708 return bytes; 709 } 710 711 static u32 get_bytes_in_lli(struct pl08x_phy_chan *ch, const u32 *llis_va) 712 { 713 u32 val; 714 u32 bytes; 715 716 if (ch->ftdmac020) { 717 val = llis_va[PL080_LLI_CCTL]; 718 bytes = val & FTDMAC020_LLI_TRANSFER_SIZE_MASK; 719 720 val = llis_va[PL080_LLI_CCTL]; 721 val &= FTDMAC020_LLI_SRC_WIDTH_MSK; 722 val >>= FTDMAC020_LLI_SRC_WIDTH_SHIFT; 723 } else if (ch->pl080s) { 724 val = llis_va[PL080S_LLI_CCTL2]; 725 bytes = val & PL080S_CONTROL_TRANSFER_SIZE_MASK; 726 727 val = llis_va[PL080_LLI_CCTL]; 728 val &= PL080_CONTROL_SWIDTH_MASK; 729 val >>= PL080_CONTROL_SWIDTH_SHIFT; 730 } else { 731 /* Plain PL08x */ 732 val = llis_va[PL080_LLI_CCTL]; 733 bytes = val & PL080_CONTROL_TRANSFER_SIZE_MASK; 734 735 val &= PL080_CONTROL_SWIDTH_MASK; 736 val >>= PL080_CONTROL_SWIDTH_SHIFT; 737 } 738 739 switch (val) { 740 case PL080_WIDTH_8BIT: 741 break; 742 case PL080_WIDTH_16BIT: 743 bytes *= 2; 744 break; 745 case PL080_WIDTH_32BIT: 746 bytes *= 4; 747 break; 748 } 749 return bytes; 750 } 751 752 /* The channel should be paused when calling this */ 753 static u32 pl08x_getbytes_chan(struct pl08x_dma_chan *plchan) 754 { 755 struct pl08x_driver_data *pl08x = plchan->host; 756 const u32 *llis_va, *llis_va_limit; 757 struct pl08x_phy_chan *ch; 758 dma_addr_t llis_bus; 759 struct pl08x_txd *txd; 760 u32 llis_max_words; 761 size_t bytes; 762 u32 clli; 763 764 ch = plchan->phychan; 765 txd = plchan->at; 766 767 if (!ch || !txd) 768 return 0; 769 770 /* 771 * Follow the LLIs to get the number of remaining 772 * bytes in the currently active transaction. 773 */ 774 clli = readl(ch->reg_lli) & ~PL080_LLI_LM_AHB2; 775 776 /* First get the remaining bytes in the active transfer */ 777 bytes = get_bytes_in_phy_channel(ch); 778 779 if (!clli) 780 return bytes; 781 782 llis_va = txd->llis_va; 783 llis_bus = txd->llis_bus; 784 785 llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS; 786 BUG_ON(clli < llis_bus || clli >= llis_bus + 787 sizeof(u32) * llis_max_words); 788 789 /* 790 * Locate the next LLI - as this is an array, 791 * it's simple maths to find. 792 */ 793 llis_va += (clli - llis_bus) / sizeof(u32); 794 795 llis_va_limit = llis_va + llis_max_words; 796 797 for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) { 798 bytes += get_bytes_in_lli(ch, llis_va); 799 800 /* 801 * A LLI pointer going backward terminates the LLI list 802 */ 803 if (llis_va[PL080_LLI_LLI] <= clli) 804 break; 805 } 806 807 return bytes; 808 } 809 810 /* 811 * Allocate a physical channel for a virtual channel 812 * 813 * Try to locate a physical channel to be used for this transfer. If all 814 * are taken return NULL and the requester will have to cope by using 815 * some fallback PIO mode or retrying later. 816 */ 817 static struct pl08x_phy_chan * 818 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x, 819 struct pl08x_dma_chan *virt_chan) 820 { 821 struct pl08x_phy_chan *ch = NULL; 822 unsigned long flags; 823 int i; 824 825 for (i = 0; i < pl08x->vd->channels; i++) { 826 ch = &pl08x->phy_chans[i]; 827 828 spin_lock_irqsave(&ch->lock, flags); 829 830 if (!ch->locked && !ch->serving) { 831 ch->serving = virt_chan; 832 spin_unlock_irqrestore(&ch->lock, flags); 833 break; 834 } 835 836 spin_unlock_irqrestore(&ch->lock, flags); 837 } 838 839 if (i == pl08x->vd->channels) { 840 /* No physical channel available, cope with it */ 841 return NULL; 842 } 843 844 return ch; 845 } 846 847 /* Mark the physical channel as free. Note, this write is atomic. */ 848 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x, 849 struct pl08x_phy_chan *ch) 850 { 851 ch->serving = NULL; 852 } 853 854 /* 855 * Try to allocate a physical channel. When successful, assign it to 856 * this virtual channel, and initiate the next descriptor. The 857 * virtual channel lock must be held at this point. 858 */ 859 static void pl08x_phy_alloc_and_start(struct pl08x_dma_chan *plchan) 860 { 861 struct pl08x_driver_data *pl08x = plchan->host; 862 struct pl08x_phy_chan *ch; 863 864 ch = pl08x_get_phy_channel(pl08x, plchan); 865 if (!ch) { 866 dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name); 867 plchan->state = PL08X_CHAN_WAITING; 868 plchan->waiting_at = jiffies; 869 return; 870 } 871 872 dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n", 873 ch->id, plchan->name); 874 875 plchan->phychan = ch; 876 plchan->state = PL08X_CHAN_RUNNING; 877 pl08x_start_next_txd(plchan); 878 } 879 880 static void pl08x_phy_reassign_start(struct pl08x_phy_chan *ch, 881 struct pl08x_dma_chan *plchan) 882 { 883 struct pl08x_driver_data *pl08x = plchan->host; 884 885 dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n", 886 ch->id, plchan->name); 887 888 /* 889 * We do this without taking the lock; we're really only concerned 890 * about whether this pointer is NULL or not, and we're guaranteed 891 * that this will only be called when it _already_ is non-NULL. 892 */ 893 ch->serving = plchan; 894 plchan->phychan = ch; 895 plchan->state = PL08X_CHAN_RUNNING; 896 pl08x_start_next_txd(plchan); 897 } 898 899 /* 900 * Free a physical DMA channel, potentially reallocating it to another 901 * virtual channel if we have any pending. 902 */ 903 static void pl08x_phy_free(struct pl08x_dma_chan *plchan) 904 { 905 struct pl08x_driver_data *pl08x = plchan->host; 906 struct pl08x_dma_chan *p, *next; 907 unsigned long waiting_at; 908 retry: 909 next = NULL; 910 waiting_at = jiffies; 911 912 /* 913 * Find a waiting virtual channel for the next transfer. 914 * To be fair, time when each channel reached waiting state is compared 915 * to select channel that is waiting for the longest time. 916 */ 917 list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node) 918 if (p->state == PL08X_CHAN_WAITING && 919 p->waiting_at <= waiting_at) { 920 next = p; 921 waiting_at = p->waiting_at; 922 } 923 924 if (!next && pl08x->has_slave) { 925 list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node) 926 if (p->state == PL08X_CHAN_WAITING && 927 p->waiting_at <= waiting_at) { 928 next = p; 929 waiting_at = p->waiting_at; 930 } 931 } 932 933 /* Ensure that the physical channel is stopped */ 934 pl08x_terminate_phy_chan(pl08x, plchan->phychan); 935 936 if (next) { 937 bool success; 938 939 /* 940 * Eww. We know this isn't going to deadlock 941 * but lockdep probably doesn't. 942 */ 943 spin_lock(&next->vc.lock); 944 /* Re-check the state now that we have the lock */ 945 success = next->state == PL08X_CHAN_WAITING; 946 if (success) 947 pl08x_phy_reassign_start(plchan->phychan, next); 948 spin_unlock(&next->vc.lock); 949 950 /* If the state changed, try to find another channel */ 951 if (!success) 952 goto retry; 953 } else { 954 /* No more jobs, so free up the physical channel */ 955 pl08x_put_phy_channel(pl08x, plchan->phychan); 956 } 957 958 plchan->phychan = NULL; 959 plchan->state = PL08X_CHAN_IDLE; 960 } 961 962 /* 963 * LLI handling 964 */ 965 966 static inline unsigned int 967 pl08x_get_bytes_for_lli(struct pl08x_driver_data *pl08x, 968 u32 cctl, 969 bool source) 970 { 971 u32 val; 972 973 if (pl08x->vd->ftdmac020) { 974 if (source) 975 val = (cctl & FTDMAC020_LLI_SRC_WIDTH_MSK) >> 976 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 977 else 978 val = (cctl & FTDMAC020_LLI_DST_WIDTH_MSK) >> 979 FTDMAC020_LLI_DST_WIDTH_SHIFT; 980 } else { 981 if (source) 982 val = (cctl & PL080_CONTROL_SWIDTH_MASK) >> 983 PL080_CONTROL_SWIDTH_SHIFT; 984 else 985 val = (cctl & PL080_CONTROL_DWIDTH_MASK) >> 986 PL080_CONTROL_DWIDTH_SHIFT; 987 } 988 989 switch (val) { 990 case PL080_WIDTH_8BIT: 991 return 1; 992 case PL080_WIDTH_16BIT: 993 return 2; 994 case PL080_WIDTH_32BIT: 995 return 4; 996 default: 997 break; 998 } 999 BUG(); 1000 return 0; 1001 } 1002 1003 static inline u32 pl08x_lli_control_bits(struct pl08x_driver_data *pl08x, 1004 u32 cctl, 1005 u8 srcwidth, u8 dstwidth, 1006 size_t tsize) 1007 { 1008 u32 retbits = cctl; 1009 1010 /* 1011 * Remove all src, dst and transfer size bits, then set the 1012 * width and size according to the parameters. The bit offsets 1013 * are different in the FTDMAC020 so we need to accound for this. 1014 */ 1015 if (pl08x->vd->ftdmac020) { 1016 retbits &= ~FTDMAC020_LLI_DST_WIDTH_MSK; 1017 retbits &= ~FTDMAC020_LLI_SRC_WIDTH_MSK; 1018 retbits &= ~FTDMAC020_LLI_TRANSFER_SIZE_MASK; 1019 1020 switch (srcwidth) { 1021 case 1: 1022 retbits |= PL080_WIDTH_8BIT << 1023 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1024 break; 1025 case 2: 1026 retbits |= PL080_WIDTH_16BIT << 1027 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1028 break; 1029 case 4: 1030 retbits |= PL080_WIDTH_32BIT << 1031 FTDMAC020_LLI_SRC_WIDTH_SHIFT; 1032 break; 1033 default: 1034 BUG(); 1035 break; 1036 } 1037 1038 switch (dstwidth) { 1039 case 1: 1040 retbits |= PL080_WIDTH_8BIT << 1041 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1042 break; 1043 case 2: 1044 retbits |= PL080_WIDTH_16BIT << 1045 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1046 break; 1047 case 4: 1048 retbits |= PL080_WIDTH_32BIT << 1049 FTDMAC020_LLI_DST_WIDTH_SHIFT; 1050 break; 1051 default: 1052 BUG(); 1053 break; 1054 } 1055 1056 tsize &= FTDMAC020_LLI_TRANSFER_SIZE_MASK; 1057 retbits |= tsize << FTDMAC020_LLI_TRANSFER_SIZE_SHIFT; 1058 } else { 1059 retbits &= ~PL080_CONTROL_DWIDTH_MASK; 1060 retbits &= ~PL080_CONTROL_SWIDTH_MASK; 1061 retbits &= ~PL080_CONTROL_TRANSFER_SIZE_MASK; 1062 1063 switch (srcwidth) { 1064 case 1: 1065 retbits |= PL080_WIDTH_8BIT << 1066 PL080_CONTROL_SWIDTH_SHIFT; 1067 break; 1068 case 2: 1069 retbits |= PL080_WIDTH_16BIT << 1070 PL080_CONTROL_SWIDTH_SHIFT; 1071 break; 1072 case 4: 1073 retbits |= PL080_WIDTH_32BIT << 1074 PL080_CONTROL_SWIDTH_SHIFT; 1075 break; 1076 default: 1077 BUG(); 1078 break; 1079 } 1080 1081 switch (dstwidth) { 1082 case 1: 1083 retbits |= PL080_WIDTH_8BIT << 1084 PL080_CONTROL_DWIDTH_SHIFT; 1085 break; 1086 case 2: 1087 retbits |= PL080_WIDTH_16BIT << 1088 PL080_CONTROL_DWIDTH_SHIFT; 1089 break; 1090 case 4: 1091 retbits |= PL080_WIDTH_32BIT << 1092 PL080_CONTROL_DWIDTH_SHIFT; 1093 break; 1094 default: 1095 BUG(); 1096 break; 1097 } 1098 1099 tsize &= PL080_CONTROL_TRANSFER_SIZE_MASK; 1100 retbits |= tsize << PL080_CONTROL_TRANSFER_SIZE_SHIFT; 1101 } 1102 1103 return retbits; 1104 } 1105 1106 struct pl08x_lli_build_data { 1107 struct pl08x_txd *txd; 1108 struct pl08x_bus_data srcbus; 1109 struct pl08x_bus_data dstbus; 1110 size_t remainder; 1111 u32 lli_bus; 1112 }; 1113 1114 /* 1115 * Autoselect a master bus to use for the transfer. Slave will be the chosen as 1116 * victim in case src & dest are not similarly aligned. i.e. If after aligning 1117 * masters address with width requirements of transfer (by sending few byte by 1118 * byte data), slave is still not aligned, then its width will be reduced to 1119 * BYTE. 1120 * - prefers the destination bus if both available 1121 * - prefers bus with fixed address (i.e. peripheral) 1122 */ 1123 static void pl08x_choose_master_bus(struct pl08x_driver_data *pl08x, 1124 struct pl08x_lli_build_data *bd, 1125 struct pl08x_bus_data **mbus, 1126 struct pl08x_bus_data **sbus, 1127 u32 cctl) 1128 { 1129 bool dst_incr; 1130 bool src_incr; 1131 1132 /* 1133 * The FTDMAC020 only supports memory-to-memory transfer, so 1134 * source and destination always increase. 1135 */ 1136 if (pl08x->vd->ftdmac020) { 1137 dst_incr = true; 1138 src_incr = true; 1139 } else { 1140 dst_incr = !!(cctl & PL080_CONTROL_DST_INCR); 1141 src_incr = !!(cctl & PL080_CONTROL_SRC_INCR); 1142 } 1143 1144 /* 1145 * If either bus is not advancing, i.e. it is a peripheral, that 1146 * one becomes master 1147 */ 1148 if (!dst_incr) { 1149 *mbus = &bd->dstbus; 1150 *sbus = &bd->srcbus; 1151 } else if (!src_incr) { 1152 *mbus = &bd->srcbus; 1153 *sbus = &bd->dstbus; 1154 } else { 1155 if (bd->dstbus.buswidth >= bd->srcbus.buswidth) { 1156 *mbus = &bd->dstbus; 1157 *sbus = &bd->srcbus; 1158 } else { 1159 *mbus = &bd->srcbus; 1160 *sbus = &bd->dstbus; 1161 } 1162 } 1163 } 1164 1165 /* 1166 * Fills in one LLI for a certain transfer descriptor and advance the counter 1167 */ 1168 static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x, 1169 struct pl08x_lli_build_data *bd, 1170 int num_llis, int len, u32 cctl, u32 cctl2) 1171 { 1172 u32 offset = num_llis * pl08x->lli_words; 1173 u32 *llis_va = bd->txd->llis_va + offset; 1174 dma_addr_t llis_bus = bd->txd->llis_bus; 1175 1176 BUG_ON(num_llis >= MAX_NUM_TSFR_LLIS); 1177 1178 /* Advance the offset to next LLI. */ 1179 offset += pl08x->lli_words; 1180 1181 llis_va[PL080_LLI_SRC] = bd->srcbus.addr; 1182 llis_va[PL080_LLI_DST] = bd->dstbus.addr; 1183 llis_va[PL080_LLI_LLI] = (llis_bus + sizeof(u32) * offset); 1184 llis_va[PL080_LLI_LLI] |= bd->lli_bus; 1185 llis_va[PL080_LLI_CCTL] = cctl; 1186 if (pl08x->vd->pl080s) 1187 llis_va[PL080S_LLI_CCTL2] = cctl2; 1188 1189 if (pl08x->vd->ftdmac020) { 1190 /* FIXME: only memcpy so far so both increase */ 1191 bd->srcbus.addr += len; 1192 bd->dstbus.addr += len; 1193 } else { 1194 if (cctl & PL080_CONTROL_SRC_INCR) 1195 bd->srcbus.addr += len; 1196 if (cctl & PL080_CONTROL_DST_INCR) 1197 bd->dstbus.addr += len; 1198 } 1199 1200 BUG_ON(bd->remainder < len); 1201 1202 bd->remainder -= len; 1203 } 1204 1205 static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x, 1206 struct pl08x_lli_build_data *bd, u32 *cctl, u32 len, 1207 int num_llis, size_t *total_bytes) 1208 { 1209 *cctl = pl08x_lli_control_bits(pl08x, *cctl, 1, 1, len); 1210 pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len); 1211 (*total_bytes) += len; 1212 } 1213 1214 #if 1 1215 static void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 1216 const u32 *llis_va, int num_llis) 1217 { 1218 int i; 1219 1220 if (pl08x->vd->pl080s) { 1221 dev_vdbg(&pl08x->adev->dev, 1222 "%-3s %-9s %-10s %-10s %-10s %-10s %s\n", 1223 "lli", "", "csrc", "cdst", "clli", "cctl", "cctl2"); 1224 for (i = 0; i < num_llis; i++) { 1225 dev_vdbg(&pl08x->adev->dev, 1226 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n", 1227 i, llis_va, llis_va[PL080_LLI_SRC], 1228 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 1229 llis_va[PL080_LLI_CCTL], 1230 llis_va[PL080S_LLI_CCTL2]); 1231 llis_va += pl08x->lli_words; 1232 } 1233 } else { 1234 dev_vdbg(&pl08x->adev->dev, 1235 "%-3s %-9s %-10s %-10s %-10s %s\n", 1236 "lli", "", "csrc", "cdst", "clli", "cctl"); 1237 for (i = 0; i < num_llis; i++) { 1238 dev_vdbg(&pl08x->adev->dev, 1239 "%3d @%p: 0x%08x 0x%08x 0x%08x 0x%08x\n", 1240 i, llis_va, llis_va[PL080_LLI_SRC], 1241 llis_va[PL080_LLI_DST], llis_va[PL080_LLI_LLI], 1242 llis_va[PL080_LLI_CCTL]); 1243 llis_va += pl08x->lli_words; 1244 } 1245 } 1246 } 1247 #else 1248 static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x, 1249 const u32 *llis_va, int num_llis) {} 1250 #endif 1251 1252 /* 1253 * This fills in the table of LLIs for the transfer descriptor 1254 * Note that we assume we never have to change the burst sizes 1255 * Return 0 for error 1256 */ 1257 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x, 1258 struct pl08x_txd *txd) 1259 { 1260 struct pl08x_bus_data *mbus, *sbus; 1261 struct pl08x_lli_build_data bd; 1262 int num_llis = 0; 1263 u32 cctl, early_bytes = 0; 1264 size_t max_bytes_per_lli, total_bytes; 1265 u32 *llis_va, *last_lli; 1266 struct pl08x_sg *dsg; 1267 1268 txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus); 1269 if (!txd->llis_va) { 1270 dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__); 1271 return 0; 1272 } 1273 1274 bd.txd = txd; 1275 bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0; 1276 cctl = txd->cctl; 1277 1278 /* Find maximum width of the source bus */ 1279 bd.srcbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, true); 1280 1281 /* Find maximum width of the destination bus */ 1282 bd.dstbus.maxwidth = pl08x_get_bytes_for_lli(pl08x, cctl, false); 1283 1284 list_for_each_entry(dsg, &txd->dsg_list, node) { 1285 total_bytes = 0; 1286 cctl = txd->cctl; 1287 1288 bd.srcbus.addr = dsg->src_addr; 1289 bd.dstbus.addr = dsg->dst_addr; 1290 bd.remainder = dsg->len; 1291 bd.srcbus.buswidth = bd.srcbus.maxwidth; 1292 bd.dstbus.buswidth = bd.dstbus.maxwidth; 1293 1294 pl08x_choose_master_bus(pl08x, &bd, &mbus, &sbus, cctl); 1295 1296 dev_vdbg(&pl08x->adev->dev, 1297 "src=0x%08llx%s/%u dst=0x%08llx%s/%u len=%zu\n", 1298 (u64)bd.srcbus.addr, 1299 cctl & PL080_CONTROL_SRC_INCR ? "+" : "", 1300 bd.srcbus.buswidth, 1301 (u64)bd.dstbus.addr, 1302 cctl & PL080_CONTROL_DST_INCR ? "+" : "", 1303 bd.dstbus.buswidth, 1304 bd.remainder); 1305 dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n", 1306 mbus == &bd.srcbus ? "src" : "dst", 1307 sbus == &bd.srcbus ? "src" : "dst"); 1308 1309 /* 1310 * Zero length is only allowed if all these requirements are 1311 * met: 1312 * - flow controller is peripheral. 1313 * - src.addr is aligned to src.width 1314 * - dst.addr is aligned to dst.width 1315 * 1316 * sg_len == 1 should be true, as there can be two cases here: 1317 * 1318 * - Memory addresses are contiguous and are not scattered. 1319 * Here, Only one sg will be passed by user driver, with 1320 * memory address and zero length. We pass this to controller 1321 * and after the transfer it will receive the last burst 1322 * request from peripheral and so transfer finishes. 1323 * 1324 * - Memory addresses are scattered and are not contiguous. 1325 * Here, Obviously as DMA controller doesn't know when a lli's 1326 * transfer gets over, it can't load next lli. So in this 1327 * case, there has to be an assumption that only one lli is 1328 * supported. Thus, we can't have scattered addresses. 1329 */ 1330 if (!bd.remainder) { 1331 u32 fc; 1332 1333 /* FTDMAC020 only does memory-to-memory */ 1334 if (pl08x->vd->ftdmac020) 1335 fc = PL080_FLOW_MEM2MEM; 1336 else 1337 fc = (txd->ccfg & PL080_CONFIG_FLOW_CONTROL_MASK) >> 1338 PL080_CONFIG_FLOW_CONTROL_SHIFT; 1339 if (!((fc >= PL080_FLOW_SRC2DST_DST) && 1340 (fc <= PL080_FLOW_SRC2DST_SRC))) { 1341 dev_err(&pl08x->adev->dev, "%s sg len can't be zero", 1342 __func__); 1343 return 0; 1344 } 1345 1346 if (!IS_BUS_ALIGNED(&bd.srcbus) || 1347 !IS_BUS_ALIGNED(&bd.dstbus)) { 1348 dev_err(&pl08x->adev->dev, 1349 "%s src & dst address must be aligned to src" 1350 " & dst width if peripheral is flow controller", 1351 __func__); 1352 return 0; 1353 } 1354 1355 cctl = pl08x_lli_control_bits(pl08x, cctl, 1356 bd.srcbus.buswidth, bd.dstbus.buswidth, 1357 0); 1358 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1359 0, cctl, 0); 1360 break; 1361 } 1362 1363 /* 1364 * Send byte by byte for following cases 1365 * - Less than a bus width available 1366 * - until master bus is aligned 1367 */ 1368 if (bd.remainder < mbus->buswidth) 1369 early_bytes = bd.remainder; 1370 else if (!IS_BUS_ALIGNED(mbus)) { 1371 early_bytes = mbus->buswidth - 1372 (mbus->addr & (mbus->buswidth - 1)); 1373 if ((bd.remainder - early_bytes) < mbus->buswidth) 1374 early_bytes = bd.remainder; 1375 } 1376 1377 if (early_bytes) { 1378 dev_vdbg(&pl08x->adev->dev, 1379 "%s byte width LLIs (remain 0x%08zx)\n", 1380 __func__, bd.remainder); 1381 prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes, 1382 num_llis++, &total_bytes); 1383 } 1384 1385 if (bd.remainder) { 1386 /* 1387 * Master now aligned 1388 * - if slave is not then we must set its width down 1389 */ 1390 if (!IS_BUS_ALIGNED(sbus)) { 1391 dev_dbg(&pl08x->adev->dev, 1392 "%s set down bus width to one byte\n", 1393 __func__); 1394 1395 sbus->buswidth = 1; 1396 } 1397 1398 /* 1399 * Bytes transferred = tsize * src width, not 1400 * MIN(buswidths) 1401 */ 1402 max_bytes_per_lli = bd.srcbus.buswidth * 1403 pl08x->vd->max_transfer_size; 1404 dev_vdbg(&pl08x->adev->dev, 1405 "%s max bytes per lli = %zu\n", 1406 __func__, max_bytes_per_lli); 1407 1408 /* 1409 * Make largest possible LLIs until less than one bus 1410 * width left 1411 */ 1412 while (bd.remainder > (mbus->buswidth - 1)) { 1413 size_t lli_len, tsize, width; 1414 1415 /* 1416 * If enough left try to send max possible, 1417 * otherwise try to send the remainder 1418 */ 1419 lli_len = min(bd.remainder, max_bytes_per_lli); 1420 1421 /* 1422 * Check against maximum bus alignment: 1423 * Calculate actual transfer size in relation to 1424 * bus width an get a maximum remainder of the 1425 * highest bus width - 1 1426 */ 1427 width = max(mbus->buswidth, sbus->buswidth); 1428 lli_len = (lli_len / width) * width; 1429 tsize = lli_len / bd.srcbus.buswidth; 1430 1431 dev_vdbg(&pl08x->adev->dev, 1432 "%s fill lli with single lli chunk of " 1433 "size 0x%08zx (remainder 0x%08zx)\n", 1434 __func__, lli_len, bd.remainder); 1435 1436 cctl = pl08x_lli_control_bits(pl08x, cctl, 1437 bd.srcbus.buswidth, bd.dstbus.buswidth, 1438 tsize); 1439 pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++, 1440 lli_len, cctl, tsize); 1441 total_bytes += lli_len; 1442 } 1443 1444 /* 1445 * Send any odd bytes 1446 */ 1447 if (bd.remainder) { 1448 dev_vdbg(&pl08x->adev->dev, 1449 "%s align with boundary, send odd bytes (remain %zu)\n", 1450 __func__, bd.remainder); 1451 prep_byte_width_lli(pl08x, &bd, &cctl, 1452 bd.remainder, num_llis++, &total_bytes); 1453 } 1454 } 1455 1456 if (total_bytes != dsg->len) { 1457 dev_err(&pl08x->adev->dev, 1458 "%s size of encoded lli:s don't match total txd, transferred 0x%08zx from size 0x%08zx\n", 1459 __func__, total_bytes, dsg->len); 1460 return 0; 1461 } 1462 1463 if (num_llis >= MAX_NUM_TSFR_LLIS) { 1464 dev_err(&pl08x->adev->dev, 1465 "%s need to increase MAX_NUM_TSFR_LLIS from 0x%08x\n", 1466 __func__, MAX_NUM_TSFR_LLIS); 1467 return 0; 1468 } 1469 } 1470 1471 llis_va = txd->llis_va; 1472 last_lli = llis_va + (num_llis - 1) * pl08x->lli_words; 1473 1474 if (txd->cyclic) { 1475 /* Link back to the first LLI. */ 1476 last_lli[PL080_LLI_LLI] = txd->llis_bus | bd.lli_bus; 1477 } else { 1478 /* The final LLI terminates the LLI. */ 1479 last_lli[PL080_LLI_LLI] = 0; 1480 /* The final LLI element shall also fire an interrupt. */ 1481 if (pl08x->vd->ftdmac020) 1482 last_lli[PL080_LLI_CCTL] &= ~FTDMAC020_LLI_TC_MSK; 1483 else 1484 last_lli[PL080_LLI_CCTL] |= PL080_CONTROL_TC_IRQ_EN; 1485 } 1486 1487 pl08x_dump_lli(pl08x, llis_va, num_llis); 1488 1489 return num_llis; 1490 } 1491 1492 static void pl08x_free_txd(struct pl08x_driver_data *pl08x, 1493 struct pl08x_txd *txd) 1494 { 1495 struct pl08x_sg *dsg, *_dsg; 1496 1497 if (txd->llis_va) 1498 dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus); 1499 1500 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { 1501 list_del(&dsg->node); 1502 kfree(dsg); 1503 } 1504 1505 kfree(txd); 1506 } 1507 1508 static void pl08x_desc_free(struct virt_dma_desc *vd) 1509 { 1510 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1511 struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); 1512 1513 dma_descriptor_unmap(&vd->tx); 1514 if (!txd->done) 1515 pl08x_release_mux(plchan); 1516 1517 pl08x_free_txd(plchan->host, txd); 1518 } 1519 1520 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x, 1521 struct pl08x_dma_chan *plchan) 1522 { 1523 LIST_HEAD(head); 1524 1525 vchan_get_all_descriptors(&plchan->vc, &head); 1526 vchan_dma_desc_free_list(&plchan->vc, &head); 1527 } 1528 1529 /* 1530 * The DMA ENGINE API 1531 */ 1532 static void pl08x_free_chan_resources(struct dma_chan *chan) 1533 { 1534 /* Ensure all queued descriptors are freed */ 1535 vchan_free_chan_resources(to_virt_chan(chan)); 1536 } 1537 1538 /* 1539 * Code accessing dma_async_is_complete() in a tight loop may give problems. 1540 * If slaves are relying on interrupts to signal completion this function 1541 * must not be called with interrupts disabled. 1542 */ 1543 static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, 1544 dma_cookie_t cookie, struct dma_tx_state *txstate) 1545 { 1546 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1547 struct virt_dma_desc *vd; 1548 unsigned long flags; 1549 enum dma_status ret; 1550 size_t bytes = 0; 1551 1552 ret = dma_cookie_status(chan, cookie, txstate); 1553 if (ret == DMA_COMPLETE) 1554 return ret; 1555 1556 /* 1557 * There's no point calculating the residue if there's 1558 * no txstate to store the value. 1559 */ 1560 if (!txstate) { 1561 if (plchan->state == PL08X_CHAN_PAUSED) 1562 ret = DMA_PAUSED; 1563 return ret; 1564 } 1565 1566 spin_lock_irqsave(&plchan->vc.lock, flags); 1567 ret = dma_cookie_status(chan, cookie, txstate); 1568 if (ret != DMA_COMPLETE) { 1569 vd = vchan_find_desc(&plchan->vc, cookie); 1570 if (vd) { 1571 /* On the issued list, so hasn't been processed yet */ 1572 struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); 1573 struct pl08x_sg *dsg; 1574 1575 list_for_each_entry(dsg, &txd->dsg_list, node) 1576 bytes += dsg->len; 1577 } else { 1578 bytes = pl08x_getbytes_chan(plchan); 1579 } 1580 } 1581 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1582 1583 /* 1584 * This cookie not complete yet 1585 * Get number of bytes left in the active transactions and queue 1586 */ 1587 dma_set_residue(txstate, bytes); 1588 1589 if (plchan->state == PL08X_CHAN_PAUSED && ret == DMA_IN_PROGRESS) 1590 ret = DMA_PAUSED; 1591 1592 /* Whether waiting or running, we're in progress */ 1593 return ret; 1594 } 1595 1596 /* PrimeCell DMA extension */ 1597 struct burst_table { 1598 u32 burstwords; 1599 u32 reg; 1600 }; 1601 1602 static const struct burst_table burst_sizes[] = { 1603 { 1604 .burstwords = 256, 1605 .reg = PL080_BSIZE_256, 1606 }, 1607 { 1608 .burstwords = 128, 1609 .reg = PL080_BSIZE_128, 1610 }, 1611 { 1612 .burstwords = 64, 1613 .reg = PL080_BSIZE_64, 1614 }, 1615 { 1616 .burstwords = 32, 1617 .reg = PL080_BSIZE_32, 1618 }, 1619 { 1620 .burstwords = 16, 1621 .reg = PL080_BSIZE_16, 1622 }, 1623 { 1624 .burstwords = 8, 1625 .reg = PL080_BSIZE_8, 1626 }, 1627 { 1628 .burstwords = 4, 1629 .reg = PL080_BSIZE_4, 1630 }, 1631 { 1632 .burstwords = 0, 1633 .reg = PL080_BSIZE_1, 1634 }, 1635 }; 1636 1637 /* 1638 * Given the source and destination available bus masks, select which 1639 * will be routed to each port. We try to have source and destination 1640 * on separate ports, but always respect the allowable settings. 1641 */ 1642 static u32 pl08x_select_bus(bool ftdmac020, u8 src, u8 dst) 1643 { 1644 u32 cctl = 0; 1645 u32 dst_ahb2; 1646 u32 src_ahb2; 1647 1648 /* The FTDMAC020 use different bits to indicate src/dst bus */ 1649 if (ftdmac020) { 1650 dst_ahb2 = FTDMAC020_LLI_DST_SEL; 1651 src_ahb2 = FTDMAC020_LLI_SRC_SEL; 1652 } else { 1653 dst_ahb2 = PL080_CONTROL_DST_AHB2; 1654 src_ahb2 = PL080_CONTROL_SRC_AHB2; 1655 } 1656 1657 if (!(dst & PL08X_AHB1) || ((dst & PL08X_AHB2) && (src & PL08X_AHB1))) 1658 cctl |= dst_ahb2; 1659 if (!(src & PL08X_AHB1) || ((src & PL08X_AHB2) && !(dst & PL08X_AHB2))) 1660 cctl |= src_ahb2; 1661 1662 return cctl; 1663 } 1664 1665 static u32 pl08x_cctl(u32 cctl) 1666 { 1667 cctl &= ~(PL080_CONTROL_SRC_AHB2 | PL080_CONTROL_DST_AHB2 | 1668 PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR | 1669 PL080_CONTROL_PROT_MASK); 1670 1671 /* Access the cell in privileged mode, non-bufferable, non-cacheable */ 1672 return cctl | PL080_CONTROL_PROT_SYS; 1673 } 1674 1675 static u32 pl08x_width(enum dma_slave_buswidth width) 1676 { 1677 switch (width) { 1678 case DMA_SLAVE_BUSWIDTH_1_BYTE: 1679 return PL080_WIDTH_8BIT; 1680 case DMA_SLAVE_BUSWIDTH_2_BYTES: 1681 return PL080_WIDTH_16BIT; 1682 case DMA_SLAVE_BUSWIDTH_4_BYTES: 1683 return PL080_WIDTH_32BIT; 1684 default: 1685 return ~0; 1686 } 1687 } 1688 1689 static u32 pl08x_burst(u32 maxburst) 1690 { 1691 int i; 1692 1693 for (i = 0; i < ARRAY_SIZE(burst_sizes); i++) 1694 if (burst_sizes[i].burstwords <= maxburst) 1695 break; 1696 1697 return burst_sizes[i].reg; 1698 } 1699 1700 static u32 pl08x_get_cctl(struct pl08x_dma_chan *plchan, 1701 enum dma_slave_buswidth addr_width, u32 maxburst) 1702 { 1703 u32 width, burst, cctl = 0; 1704 1705 width = pl08x_width(addr_width); 1706 if (width == ~0) 1707 return ~0; 1708 1709 cctl |= width << PL080_CONTROL_SWIDTH_SHIFT; 1710 cctl |= width << PL080_CONTROL_DWIDTH_SHIFT; 1711 1712 /* 1713 * If this channel will only request single transfers, set this 1714 * down to ONE element. Also select one element if no maxburst 1715 * is specified. 1716 */ 1717 if (plchan->cd->single) 1718 maxburst = 1; 1719 1720 burst = pl08x_burst(maxburst); 1721 cctl |= burst << PL080_CONTROL_SB_SIZE_SHIFT; 1722 cctl |= burst << PL080_CONTROL_DB_SIZE_SHIFT; 1723 1724 return pl08x_cctl(cctl); 1725 } 1726 1727 /* 1728 * Slave transactions callback to the slave device to allow 1729 * synchronization of slave DMA signals with the DMAC enable 1730 */ 1731 static void pl08x_issue_pending(struct dma_chan *chan) 1732 { 1733 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1734 unsigned long flags; 1735 1736 spin_lock_irqsave(&plchan->vc.lock, flags); 1737 if (vchan_issue_pending(&plchan->vc)) { 1738 if (!plchan->phychan && plchan->state != PL08X_CHAN_WAITING) 1739 pl08x_phy_alloc_and_start(plchan); 1740 } 1741 spin_unlock_irqrestore(&plchan->vc.lock, flags); 1742 } 1743 1744 static struct pl08x_txd *pl08x_get_txd(struct pl08x_dma_chan *plchan) 1745 { 1746 struct pl08x_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); 1747 1748 if (txd) 1749 INIT_LIST_HEAD(&txd->dsg_list); 1750 return txd; 1751 } 1752 1753 static u32 pl08x_memcpy_cctl(struct pl08x_driver_data *pl08x) 1754 { 1755 u32 cctl = 0; 1756 1757 /* Conjure cctl */ 1758 switch (pl08x->pd->memcpy_burst_size) { 1759 default: 1760 dev_err(&pl08x->adev->dev, 1761 "illegal burst size for memcpy, set to 1\n"); 1762 fallthrough; 1763 case PL08X_BURST_SZ_1: 1764 cctl |= PL080_BSIZE_1 << PL080_CONTROL_SB_SIZE_SHIFT | 1765 PL080_BSIZE_1 << PL080_CONTROL_DB_SIZE_SHIFT; 1766 break; 1767 case PL08X_BURST_SZ_4: 1768 cctl |= PL080_BSIZE_4 << PL080_CONTROL_SB_SIZE_SHIFT | 1769 PL080_BSIZE_4 << PL080_CONTROL_DB_SIZE_SHIFT; 1770 break; 1771 case PL08X_BURST_SZ_8: 1772 cctl |= PL080_BSIZE_8 << PL080_CONTROL_SB_SIZE_SHIFT | 1773 PL080_BSIZE_8 << PL080_CONTROL_DB_SIZE_SHIFT; 1774 break; 1775 case PL08X_BURST_SZ_16: 1776 cctl |= PL080_BSIZE_16 << PL080_CONTROL_SB_SIZE_SHIFT | 1777 PL080_BSIZE_16 << PL080_CONTROL_DB_SIZE_SHIFT; 1778 break; 1779 case PL08X_BURST_SZ_32: 1780 cctl |= PL080_BSIZE_32 << PL080_CONTROL_SB_SIZE_SHIFT | 1781 PL080_BSIZE_32 << PL080_CONTROL_DB_SIZE_SHIFT; 1782 break; 1783 case PL08X_BURST_SZ_64: 1784 cctl |= PL080_BSIZE_64 << PL080_CONTROL_SB_SIZE_SHIFT | 1785 PL080_BSIZE_64 << PL080_CONTROL_DB_SIZE_SHIFT; 1786 break; 1787 case PL08X_BURST_SZ_128: 1788 cctl |= PL080_BSIZE_128 << PL080_CONTROL_SB_SIZE_SHIFT | 1789 PL080_BSIZE_128 << PL080_CONTROL_DB_SIZE_SHIFT; 1790 break; 1791 case PL08X_BURST_SZ_256: 1792 cctl |= PL080_BSIZE_256 << PL080_CONTROL_SB_SIZE_SHIFT | 1793 PL080_BSIZE_256 << PL080_CONTROL_DB_SIZE_SHIFT; 1794 break; 1795 } 1796 1797 switch (pl08x->pd->memcpy_bus_width) { 1798 default: 1799 dev_err(&pl08x->adev->dev, 1800 "illegal bus width for memcpy, set to 8 bits\n"); 1801 fallthrough; 1802 case PL08X_BUS_WIDTH_8_BITS: 1803 cctl |= PL080_WIDTH_8BIT << PL080_CONTROL_SWIDTH_SHIFT | 1804 PL080_WIDTH_8BIT << PL080_CONTROL_DWIDTH_SHIFT; 1805 break; 1806 case PL08X_BUS_WIDTH_16_BITS: 1807 cctl |= PL080_WIDTH_16BIT << PL080_CONTROL_SWIDTH_SHIFT | 1808 PL080_WIDTH_16BIT << PL080_CONTROL_DWIDTH_SHIFT; 1809 break; 1810 case PL08X_BUS_WIDTH_32_BITS: 1811 cctl |= PL080_WIDTH_32BIT << PL080_CONTROL_SWIDTH_SHIFT | 1812 PL080_WIDTH_32BIT << PL080_CONTROL_DWIDTH_SHIFT; 1813 break; 1814 } 1815 1816 /* Protection flags */ 1817 if (pl08x->pd->memcpy_prot_buff) 1818 cctl |= PL080_CONTROL_PROT_BUFF; 1819 if (pl08x->pd->memcpy_prot_cache) 1820 cctl |= PL080_CONTROL_PROT_CACHE; 1821 1822 /* We are the kernel, so we are in privileged mode */ 1823 cctl |= PL080_CONTROL_PROT_SYS; 1824 1825 /* Both to be incremented or the code will break */ 1826 cctl |= PL080_CONTROL_SRC_INCR | PL080_CONTROL_DST_INCR; 1827 1828 if (pl08x->vd->dualmaster) 1829 cctl |= pl08x_select_bus(false, 1830 pl08x->mem_buses, 1831 pl08x->mem_buses); 1832 1833 return cctl; 1834 } 1835 1836 static u32 pl08x_ftdmac020_memcpy_cctl(struct pl08x_driver_data *pl08x) 1837 { 1838 u32 cctl = 0; 1839 1840 /* Conjure cctl */ 1841 switch (pl08x->pd->memcpy_bus_width) { 1842 default: 1843 dev_err(&pl08x->adev->dev, 1844 "illegal bus width for memcpy, set to 8 bits\n"); 1845 fallthrough; 1846 case PL08X_BUS_WIDTH_8_BITS: 1847 cctl |= PL080_WIDTH_8BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1848 PL080_WIDTH_8BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1849 break; 1850 case PL08X_BUS_WIDTH_16_BITS: 1851 cctl |= PL080_WIDTH_16BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1852 PL080_WIDTH_16BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1853 break; 1854 case PL08X_BUS_WIDTH_32_BITS: 1855 cctl |= PL080_WIDTH_32BIT << FTDMAC020_LLI_SRC_WIDTH_SHIFT | 1856 PL080_WIDTH_32BIT << FTDMAC020_LLI_DST_WIDTH_SHIFT; 1857 break; 1858 } 1859 1860 /* 1861 * By default mask the TC IRQ on all LLIs, it will be unmasked on 1862 * the last LLI item by other code. 1863 */ 1864 cctl |= FTDMAC020_LLI_TC_MSK; 1865 1866 /* 1867 * Both to be incremented so leave bits FTDMAC020_LLI_SRCAD_CTL 1868 * and FTDMAC020_LLI_DSTAD_CTL as zero 1869 */ 1870 if (pl08x->vd->dualmaster) 1871 cctl |= pl08x_select_bus(true, 1872 pl08x->mem_buses, 1873 pl08x->mem_buses); 1874 1875 return cctl; 1876 } 1877 1878 /* 1879 * Initialize a descriptor to be used by memcpy submit 1880 */ 1881 static struct dma_async_tx_descriptor *pl08x_prep_dma_memcpy( 1882 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 1883 size_t len, unsigned long flags) 1884 { 1885 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1886 struct pl08x_driver_data *pl08x = plchan->host; 1887 struct pl08x_txd *txd; 1888 struct pl08x_sg *dsg; 1889 int ret; 1890 1891 txd = pl08x_get_txd(plchan); 1892 if (!txd) { 1893 dev_err(&pl08x->adev->dev, 1894 "%s no memory for descriptor\n", __func__); 1895 return NULL; 1896 } 1897 1898 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 1899 if (!dsg) { 1900 pl08x_free_txd(pl08x, txd); 1901 return NULL; 1902 } 1903 list_add_tail(&dsg->node, &txd->dsg_list); 1904 1905 dsg->src_addr = src; 1906 dsg->dst_addr = dest; 1907 dsg->len = len; 1908 if (pl08x->vd->ftdmac020) { 1909 /* Writing CCFG zero ENABLES all interrupts */ 1910 txd->ccfg = 0; 1911 txd->cctl = pl08x_ftdmac020_memcpy_cctl(pl08x); 1912 } else { 1913 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 1914 PL080_CONFIG_TC_IRQ_MASK | 1915 PL080_FLOW_MEM2MEM << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1916 txd->cctl = pl08x_memcpy_cctl(pl08x); 1917 } 1918 1919 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 1920 if (!ret) { 1921 pl08x_free_txd(pl08x, txd); 1922 return NULL; 1923 } 1924 1925 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 1926 } 1927 1928 static struct pl08x_txd *pl08x_init_txd( 1929 struct dma_chan *chan, 1930 enum dma_transfer_direction direction, 1931 dma_addr_t *slave_addr) 1932 { 1933 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 1934 struct pl08x_driver_data *pl08x = plchan->host; 1935 struct pl08x_txd *txd; 1936 enum dma_slave_buswidth addr_width; 1937 int ret, tmp; 1938 u8 src_buses, dst_buses; 1939 u32 maxburst, cctl; 1940 1941 txd = pl08x_get_txd(plchan); 1942 if (!txd) { 1943 dev_err(&pl08x->adev->dev, "%s no txd\n", __func__); 1944 return NULL; 1945 } 1946 1947 /* 1948 * Set up addresses, the PrimeCell configured address 1949 * will take precedence since this may configure the 1950 * channel target address dynamically at runtime. 1951 */ 1952 if (direction == DMA_MEM_TO_DEV) { 1953 cctl = PL080_CONTROL_SRC_INCR; 1954 *slave_addr = plchan->cfg.dst_addr; 1955 addr_width = plchan->cfg.dst_addr_width; 1956 maxburst = plchan->cfg.dst_maxburst; 1957 src_buses = pl08x->mem_buses; 1958 dst_buses = plchan->cd->periph_buses; 1959 } else if (direction == DMA_DEV_TO_MEM) { 1960 cctl = PL080_CONTROL_DST_INCR; 1961 *slave_addr = plchan->cfg.src_addr; 1962 addr_width = plchan->cfg.src_addr_width; 1963 maxburst = plchan->cfg.src_maxburst; 1964 src_buses = plchan->cd->periph_buses; 1965 dst_buses = pl08x->mem_buses; 1966 } else { 1967 pl08x_free_txd(pl08x, txd); 1968 dev_err(&pl08x->adev->dev, 1969 "%s direction unsupported\n", __func__); 1970 return NULL; 1971 } 1972 1973 cctl |= pl08x_get_cctl(plchan, addr_width, maxburst); 1974 if (cctl == ~0) { 1975 pl08x_free_txd(pl08x, txd); 1976 dev_err(&pl08x->adev->dev, 1977 "DMA slave configuration botched?\n"); 1978 return NULL; 1979 } 1980 1981 txd->cctl = cctl | pl08x_select_bus(false, src_buses, dst_buses); 1982 1983 if (plchan->cfg.device_fc) 1984 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER_PER : 1985 PL080_FLOW_PER2MEM_PER; 1986 else 1987 tmp = (direction == DMA_MEM_TO_DEV) ? PL080_FLOW_MEM2PER : 1988 PL080_FLOW_PER2MEM; 1989 1990 txd->ccfg = PL080_CONFIG_ERR_IRQ_MASK | 1991 PL080_CONFIG_TC_IRQ_MASK | 1992 tmp << PL080_CONFIG_FLOW_CONTROL_SHIFT; 1993 1994 ret = pl08x_request_mux(plchan); 1995 if (ret < 0) { 1996 pl08x_free_txd(pl08x, txd); 1997 dev_dbg(&pl08x->adev->dev, 1998 "unable to mux for transfer on %s due to platform restrictions\n", 1999 plchan->name); 2000 return NULL; 2001 } 2002 2003 dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n", 2004 plchan->signal, plchan->name); 2005 2006 /* Assign the flow control signal to this channel */ 2007 if (direction == DMA_MEM_TO_DEV) 2008 txd->ccfg |= plchan->signal << PL080_CONFIG_DST_SEL_SHIFT; 2009 else 2010 txd->ccfg |= plchan->signal << PL080_CONFIG_SRC_SEL_SHIFT; 2011 2012 return txd; 2013 } 2014 2015 static int pl08x_tx_add_sg(struct pl08x_txd *txd, 2016 enum dma_transfer_direction direction, 2017 dma_addr_t slave_addr, 2018 dma_addr_t buf_addr, 2019 unsigned int len) 2020 { 2021 struct pl08x_sg *dsg; 2022 2023 dsg = kzalloc(sizeof(struct pl08x_sg), GFP_NOWAIT); 2024 if (!dsg) 2025 return -ENOMEM; 2026 2027 list_add_tail(&dsg->node, &txd->dsg_list); 2028 2029 dsg->len = len; 2030 if (direction == DMA_MEM_TO_DEV) { 2031 dsg->src_addr = buf_addr; 2032 dsg->dst_addr = slave_addr; 2033 } else { 2034 dsg->src_addr = slave_addr; 2035 dsg->dst_addr = buf_addr; 2036 } 2037 2038 return 0; 2039 } 2040 2041 static struct dma_async_tx_descriptor *pl08x_prep_slave_sg( 2042 struct dma_chan *chan, struct scatterlist *sgl, 2043 unsigned int sg_len, enum dma_transfer_direction direction, 2044 unsigned long flags, void *context) 2045 { 2046 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2047 struct pl08x_driver_data *pl08x = plchan->host; 2048 struct pl08x_txd *txd; 2049 struct scatterlist *sg; 2050 int ret, tmp; 2051 dma_addr_t slave_addr; 2052 2053 dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n", 2054 __func__, sg_dma_len(sgl), plchan->name); 2055 2056 txd = pl08x_init_txd(chan, direction, &slave_addr); 2057 if (!txd) 2058 return NULL; 2059 2060 for_each_sg(sgl, sg, sg_len, tmp) { 2061 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 2062 sg_dma_address(sg), 2063 sg_dma_len(sg)); 2064 if (ret) { 2065 pl08x_release_mux(plchan); 2066 pl08x_free_txd(pl08x, txd); 2067 dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n", 2068 __func__); 2069 return NULL; 2070 } 2071 } 2072 2073 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 2074 if (!ret) { 2075 pl08x_release_mux(plchan); 2076 pl08x_free_txd(pl08x, txd); 2077 return NULL; 2078 } 2079 2080 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 2081 } 2082 2083 static struct dma_async_tx_descriptor *pl08x_prep_dma_cyclic( 2084 struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, 2085 size_t period_len, enum dma_transfer_direction direction, 2086 unsigned long flags) 2087 { 2088 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2089 struct pl08x_driver_data *pl08x = plchan->host; 2090 struct pl08x_txd *txd; 2091 int ret, tmp; 2092 dma_addr_t slave_addr; 2093 2094 dev_dbg(&pl08x->adev->dev, 2095 "%s prepare cyclic transaction of %zd/%zd bytes %s %s\n", 2096 __func__, period_len, buf_len, 2097 direction == DMA_MEM_TO_DEV ? "to" : "from", 2098 plchan->name); 2099 2100 txd = pl08x_init_txd(chan, direction, &slave_addr); 2101 if (!txd) 2102 return NULL; 2103 2104 txd->cyclic = true; 2105 txd->cctl |= PL080_CONTROL_TC_IRQ_EN; 2106 for (tmp = 0; tmp < buf_len; tmp += period_len) { 2107 ret = pl08x_tx_add_sg(txd, direction, slave_addr, 2108 buf_addr + tmp, period_len); 2109 if (ret) { 2110 pl08x_release_mux(plchan); 2111 pl08x_free_txd(pl08x, txd); 2112 return NULL; 2113 } 2114 } 2115 2116 ret = pl08x_fill_llis_for_desc(plchan->host, txd); 2117 if (!ret) { 2118 pl08x_release_mux(plchan); 2119 pl08x_free_txd(pl08x, txd); 2120 return NULL; 2121 } 2122 2123 return vchan_tx_prep(&plchan->vc, &txd->vd, flags); 2124 } 2125 2126 static int pl08x_config(struct dma_chan *chan, 2127 struct dma_slave_config *config) 2128 { 2129 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2130 struct pl08x_driver_data *pl08x = plchan->host; 2131 2132 if (!plchan->slave) 2133 return -EINVAL; 2134 2135 /* Reject definitely invalid configurations */ 2136 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || 2137 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) 2138 return -EINVAL; 2139 2140 if (config->device_fc && pl08x->vd->pl080s) { 2141 dev_err(&pl08x->adev->dev, 2142 "%s: PL080S does not support peripheral flow control\n", 2143 __func__); 2144 return -EINVAL; 2145 } 2146 2147 plchan->cfg = *config; 2148 2149 return 0; 2150 } 2151 2152 static int pl08x_terminate_all(struct dma_chan *chan) 2153 { 2154 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2155 struct pl08x_driver_data *pl08x = plchan->host; 2156 unsigned long flags; 2157 2158 spin_lock_irqsave(&plchan->vc.lock, flags); 2159 if (!plchan->phychan && !plchan->at) { 2160 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2161 return 0; 2162 } 2163 2164 plchan->state = PL08X_CHAN_IDLE; 2165 2166 if (plchan->phychan) { 2167 /* 2168 * Mark physical channel as free and free any slave 2169 * signal 2170 */ 2171 pl08x_phy_free(plchan); 2172 } 2173 /* Dequeue jobs and free LLIs */ 2174 if (plchan->at) { 2175 vchan_terminate_vdesc(&plchan->at->vd); 2176 plchan->at = NULL; 2177 } 2178 /* Dequeue jobs not yet fired as well */ 2179 pl08x_free_txd_list(pl08x, plchan); 2180 2181 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2182 2183 return 0; 2184 } 2185 2186 static void pl08x_synchronize(struct dma_chan *chan) 2187 { 2188 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2189 2190 vchan_synchronize(&plchan->vc); 2191 } 2192 2193 static int pl08x_pause(struct dma_chan *chan) 2194 { 2195 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2196 unsigned long flags; 2197 2198 /* 2199 * Anything succeeds on channels with no physical allocation and 2200 * no queued transfers. 2201 */ 2202 spin_lock_irqsave(&plchan->vc.lock, flags); 2203 if (!plchan->phychan && !plchan->at) { 2204 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2205 return 0; 2206 } 2207 2208 pl08x_pause_phy_chan(plchan->phychan); 2209 plchan->state = PL08X_CHAN_PAUSED; 2210 2211 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2212 2213 return 0; 2214 } 2215 2216 static int pl08x_resume(struct dma_chan *chan) 2217 { 2218 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2219 unsigned long flags; 2220 2221 /* 2222 * Anything succeeds on channels with no physical allocation and 2223 * no queued transfers. 2224 */ 2225 spin_lock_irqsave(&plchan->vc.lock, flags); 2226 if (!plchan->phychan && !plchan->at) { 2227 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2228 return 0; 2229 } 2230 2231 pl08x_resume_phy_chan(plchan->phychan); 2232 plchan->state = PL08X_CHAN_RUNNING; 2233 2234 spin_unlock_irqrestore(&plchan->vc.lock, flags); 2235 2236 return 0; 2237 } 2238 2239 bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) 2240 { 2241 struct pl08x_dma_chan *plchan; 2242 const char *name = chan_id; 2243 2244 /* Reject channels for devices not bound to this driver */ 2245 if (chan->device->dev->driver != &pl08x_amba_driver.drv) 2246 return false; 2247 2248 plchan = to_pl08x_chan(chan); 2249 2250 /* Check that the channel is not taken! */ 2251 if (!strcmp(plchan->name, name)) 2252 return true; 2253 2254 return false; 2255 } 2256 EXPORT_SYMBOL_GPL(pl08x_filter_id); 2257 2258 static bool pl08x_filter_fn(struct dma_chan *chan, void *chan_id) 2259 { 2260 struct pl08x_dma_chan *plchan = to_pl08x_chan(chan); 2261 2262 return plchan->cd == chan_id; 2263 } 2264 2265 /* 2266 * Just check that the device is there and active 2267 * TODO: turn this bit on/off depending on the number of physical channels 2268 * actually used, if it is zero... well shut it off. That will save some 2269 * power. Cut the clock at the same time. 2270 */ 2271 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x) 2272 { 2273 /* The Nomadik variant does not have the config register */ 2274 if (pl08x->vd->nomadik) 2275 return; 2276 /* The FTDMAC020 variant does this in another register */ 2277 if (pl08x->vd->ftdmac020) { 2278 writel(PL080_CONFIG_ENABLE, pl08x->base + FTDMAC020_CSR); 2279 return; 2280 } 2281 writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG); 2282 } 2283 2284 static irqreturn_t pl08x_irq(int irq, void *dev) 2285 { 2286 struct pl08x_driver_data *pl08x = dev; 2287 u32 mask = 0, err, tc, i; 2288 2289 /* check & clear - ERR & TC interrupts */ 2290 err = readl(pl08x->base + PL080_ERR_STATUS); 2291 if (err) { 2292 dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n", 2293 __func__, err); 2294 writel(err, pl08x->base + PL080_ERR_CLEAR); 2295 } 2296 tc = readl(pl08x->base + PL080_TC_STATUS); 2297 if (tc) 2298 writel(tc, pl08x->base + PL080_TC_CLEAR); 2299 2300 if (!err && !tc) 2301 return IRQ_NONE; 2302 2303 for (i = 0; i < pl08x->vd->channels; i++) { 2304 if ((BIT(i) & err) || (BIT(i) & tc)) { 2305 /* Locate physical channel */ 2306 struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i]; 2307 struct pl08x_dma_chan *plchan = phychan->serving; 2308 struct pl08x_txd *tx; 2309 2310 if (!plchan) { 2311 dev_err(&pl08x->adev->dev, 2312 "%s Error TC interrupt on unused channel: 0x%08x\n", 2313 __func__, i); 2314 continue; 2315 } 2316 2317 spin_lock(&plchan->vc.lock); 2318 tx = plchan->at; 2319 if (tx && tx->cyclic) { 2320 vchan_cyclic_callback(&tx->vd); 2321 } else if (tx) { 2322 plchan->at = NULL; 2323 /* 2324 * This descriptor is done, release its mux 2325 * reservation. 2326 */ 2327 pl08x_release_mux(plchan); 2328 tx->done = true; 2329 vchan_cookie_complete(&tx->vd); 2330 2331 /* 2332 * And start the next descriptor (if any), 2333 * otherwise free this channel. 2334 */ 2335 if (vchan_next_desc(&plchan->vc)) 2336 pl08x_start_next_txd(plchan); 2337 else 2338 pl08x_phy_free(plchan); 2339 } 2340 spin_unlock(&plchan->vc.lock); 2341 2342 mask |= BIT(i); 2343 } 2344 } 2345 2346 return mask ? IRQ_HANDLED : IRQ_NONE; 2347 } 2348 2349 static void pl08x_dma_slave_init(struct pl08x_dma_chan *chan) 2350 { 2351 chan->slave = true; 2352 chan->name = chan->cd->bus_id; 2353 chan->cfg.src_addr = chan->cd->addr; 2354 chan->cfg.dst_addr = chan->cd->addr; 2355 } 2356 2357 /* 2358 * Initialise the DMAC memcpy/slave channels. 2359 * Make a local wrapper to hold required data 2360 */ 2361 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x, 2362 struct dma_device *dmadev, unsigned int channels, bool slave) 2363 { 2364 struct pl08x_dma_chan *chan; 2365 int i; 2366 2367 INIT_LIST_HEAD(&dmadev->channels); 2368 2369 /* 2370 * Register as many memcpy as we have physical channels, 2371 * we won't always be able to use all but the code will have 2372 * to cope with that situation. 2373 */ 2374 for (i = 0; i < channels; i++) { 2375 chan = kzalloc(sizeof(*chan), GFP_KERNEL); 2376 if (!chan) 2377 return -ENOMEM; 2378 2379 chan->host = pl08x; 2380 chan->state = PL08X_CHAN_IDLE; 2381 chan->signal = -1; 2382 2383 if (slave) { 2384 chan->cd = &pl08x->pd->slave_channels[i]; 2385 /* 2386 * Some implementations have muxed signals, whereas some 2387 * use a mux in front of the signals and need dynamic 2388 * assignment of signals. 2389 */ 2390 chan->signal = i; 2391 pl08x_dma_slave_init(chan); 2392 } else { 2393 chan->cd = kzalloc(sizeof(*chan->cd), GFP_KERNEL); 2394 if (!chan->cd) { 2395 kfree(chan); 2396 return -ENOMEM; 2397 } 2398 chan->cd->bus_id = "memcpy"; 2399 chan->cd->periph_buses = pl08x->pd->mem_buses; 2400 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); 2401 if (!chan->name) { 2402 kfree(chan->cd); 2403 kfree(chan); 2404 return -ENOMEM; 2405 } 2406 } 2407 dev_dbg(&pl08x->adev->dev, 2408 "initialize virtual channel \"%s\"\n", 2409 chan->name); 2410 2411 chan->vc.desc_free = pl08x_desc_free; 2412 vchan_init(&chan->vc, dmadev); 2413 } 2414 dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n", 2415 i, slave ? "slave" : "memcpy"); 2416 return i; 2417 } 2418 2419 static void pl08x_free_virtual_channels(struct dma_device *dmadev) 2420 { 2421 struct pl08x_dma_chan *chan = NULL; 2422 struct pl08x_dma_chan *next; 2423 2424 list_for_each_entry_safe(chan, 2425 next, &dmadev->channels, vc.chan.device_node) { 2426 list_del(&chan->vc.chan.device_node); 2427 kfree(chan); 2428 } 2429 } 2430 2431 #ifdef CONFIG_DEBUG_FS 2432 static const char *pl08x_state_str(enum pl08x_dma_chan_state state) 2433 { 2434 switch (state) { 2435 case PL08X_CHAN_IDLE: 2436 return "idle"; 2437 case PL08X_CHAN_RUNNING: 2438 return "running"; 2439 case PL08X_CHAN_PAUSED: 2440 return "paused"; 2441 case PL08X_CHAN_WAITING: 2442 return "waiting"; 2443 default: 2444 break; 2445 } 2446 return "UNKNOWN STATE"; 2447 } 2448 2449 static int pl08x_debugfs_show(struct seq_file *s, void *data) 2450 { 2451 struct pl08x_driver_data *pl08x = s->private; 2452 struct pl08x_dma_chan *chan; 2453 struct pl08x_phy_chan *ch; 2454 unsigned long flags; 2455 int i; 2456 2457 seq_printf(s, "PL08x physical channels:\n"); 2458 seq_printf(s, "CHANNEL:\tUSER:\n"); 2459 seq_printf(s, "--------\t-----\n"); 2460 for (i = 0; i < pl08x->vd->channels; i++) { 2461 struct pl08x_dma_chan *virt_chan; 2462 2463 ch = &pl08x->phy_chans[i]; 2464 2465 spin_lock_irqsave(&ch->lock, flags); 2466 virt_chan = ch->serving; 2467 2468 seq_printf(s, "%d\t\t%s%s\n", 2469 ch->id, 2470 virt_chan ? virt_chan->name : "(none)", 2471 ch->locked ? " LOCKED" : ""); 2472 2473 spin_unlock_irqrestore(&ch->lock, flags); 2474 } 2475 2476 seq_printf(s, "\nPL08x virtual memcpy channels:\n"); 2477 seq_printf(s, "CHANNEL:\tSTATE:\n"); 2478 seq_printf(s, "--------\t------\n"); 2479 list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) { 2480 seq_printf(s, "%s\t\t%s\n", chan->name, 2481 pl08x_state_str(chan->state)); 2482 } 2483 2484 if (pl08x->has_slave) { 2485 seq_printf(s, "\nPL08x virtual slave channels:\n"); 2486 seq_printf(s, "CHANNEL:\tSTATE:\n"); 2487 seq_printf(s, "--------\t------\n"); 2488 list_for_each_entry(chan, &pl08x->slave.channels, 2489 vc.chan.device_node) { 2490 seq_printf(s, "%s\t\t%s\n", chan->name, 2491 pl08x_state_str(chan->state)); 2492 } 2493 } 2494 2495 return 0; 2496 } 2497 2498 DEFINE_SHOW_ATTRIBUTE(pl08x_debugfs); 2499 2500 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2501 { 2502 /* Expose a simple debugfs interface to view all clocks */ 2503 debugfs_create_file(dev_name(&pl08x->adev->dev), S_IFREG | S_IRUGO, 2504 NULL, pl08x, &pl08x_debugfs_fops); 2505 } 2506 2507 #else 2508 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x) 2509 { 2510 } 2511 #endif 2512 2513 #ifdef CONFIG_OF 2514 static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x, 2515 u32 id) 2516 { 2517 struct pl08x_dma_chan *chan; 2518 2519 /* Trying to get a slave channel from something with no slave support */ 2520 if (!pl08x->has_slave) 2521 return NULL; 2522 2523 list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) { 2524 if (chan->signal == id) 2525 return &chan->vc.chan; 2526 } 2527 2528 return NULL; 2529 } 2530 2531 static struct dma_chan *pl08x_of_xlate(struct of_phandle_args *dma_spec, 2532 struct of_dma *ofdma) 2533 { 2534 struct pl08x_driver_data *pl08x = ofdma->of_dma_data; 2535 struct dma_chan *dma_chan; 2536 struct pl08x_dma_chan *plchan; 2537 2538 if (!pl08x) 2539 return NULL; 2540 2541 if (dma_spec->args_count != 2) { 2542 dev_err(&pl08x->adev->dev, 2543 "DMA channel translation requires two cells\n"); 2544 return NULL; 2545 } 2546 2547 dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]); 2548 if (!dma_chan) { 2549 dev_err(&pl08x->adev->dev, 2550 "DMA slave channel not found\n"); 2551 return NULL; 2552 } 2553 2554 plchan = to_pl08x_chan(dma_chan); 2555 dev_dbg(&pl08x->adev->dev, 2556 "translated channel for signal %d\n", 2557 dma_spec->args[0]); 2558 2559 /* Augment channel data for applicable AHB buses */ 2560 plchan->cd->periph_buses = dma_spec->args[1]; 2561 return dma_get_slave_channel(dma_chan); 2562 } 2563 2564 static int pl08x_of_probe(struct amba_device *adev, 2565 struct pl08x_driver_data *pl08x, 2566 struct device_node *np) 2567 { 2568 struct pl08x_platform_data *pd; 2569 struct pl08x_channel_data *chanp = NULL; 2570 u32 val; 2571 int ret; 2572 int i; 2573 2574 pd = devm_kzalloc(&adev->dev, sizeof(*pd), GFP_KERNEL); 2575 if (!pd) 2576 return -ENOMEM; 2577 2578 /* Eligible bus masters for fetching LLIs */ 2579 if (of_property_read_bool(np, "lli-bus-interface-ahb1")) 2580 pd->lli_buses |= PL08X_AHB1; 2581 if (of_property_read_bool(np, "lli-bus-interface-ahb2")) 2582 pd->lli_buses |= PL08X_AHB2; 2583 if (!pd->lli_buses) { 2584 dev_info(&adev->dev, "no bus masters for LLIs stated, assume all\n"); 2585 pd->lli_buses |= PL08X_AHB1 | PL08X_AHB2; 2586 } 2587 2588 /* Eligible bus masters for memory access */ 2589 if (of_property_read_bool(np, "mem-bus-interface-ahb1")) 2590 pd->mem_buses |= PL08X_AHB1; 2591 if (of_property_read_bool(np, "mem-bus-interface-ahb2")) 2592 pd->mem_buses |= PL08X_AHB2; 2593 if (!pd->mem_buses) { 2594 dev_info(&adev->dev, "no bus masters for memory stated, assume all\n"); 2595 pd->mem_buses |= PL08X_AHB1 | PL08X_AHB2; 2596 } 2597 2598 /* Parse the memcpy channel properties */ 2599 ret = of_property_read_u32(np, "memcpy-burst-size", &val); 2600 if (ret) { 2601 dev_info(&adev->dev, "no memcpy burst size specified, using 1 byte\n"); 2602 val = 1; 2603 } 2604 switch (val) { 2605 default: 2606 dev_err(&adev->dev, "illegal burst size for memcpy, set to 1\n"); 2607 fallthrough; 2608 case 1: 2609 pd->memcpy_burst_size = PL08X_BURST_SZ_1; 2610 break; 2611 case 4: 2612 pd->memcpy_burst_size = PL08X_BURST_SZ_4; 2613 break; 2614 case 8: 2615 pd->memcpy_burst_size = PL08X_BURST_SZ_8; 2616 break; 2617 case 16: 2618 pd->memcpy_burst_size = PL08X_BURST_SZ_16; 2619 break; 2620 case 32: 2621 pd->memcpy_burst_size = PL08X_BURST_SZ_32; 2622 break; 2623 case 64: 2624 pd->memcpy_burst_size = PL08X_BURST_SZ_64; 2625 break; 2626 case 128: 2627 pd->memcpy_burst_size = PL08X_BURST_SZ_128; 2628 break; 2629 case 256: 2630 pd->memcpy_burst_size = PL08X_BURST_SZ_256; 2631 break; 2632 } 2633 2634 ret = of_property_read_u32(np, "memcpy-bus-width", &val); 2635 if (ret) { 2636 dev_info(&adev->dev, "no memcpy bus width specified, using 8 bits\n"); 2637 val = 8; 2638 } 2639 switch (val) { 2640 default: 2641 dev_err(&adev->dev, "illegal bus width for memcpy, set to 8 bits\n"); 2642 fallthrough; 2643 case 8: 2644 pd->memcpy_bus_width = PL08X_BUS_WIDTH_8_BITS; 2645 break; 2646 case 16: 2647 pd->memcpy_bus_width = PL08X_BUS_WIDTH_16_BITS; 2648 break; 2649 case 32: 2650 pd->memcpy_bus_width = PL08X_BUS_WIDTH_32_BITS; 2651 break; 2652 } 2653 2654 /* 2655 * Allocate channel data for all possible slave channels (one 2656 * for each possible signal), channels will then be allocated 2657 * for a device and have it's AHB interfaces set up at 2658 * translation time. 2659 */ 2660 if (pl08x->vd->signals) { 2661 chanp = devm_kcalloc(&adev->dev, 2662 pl08x->vd->signals, 2663 sizeof(struct pl08x_channel_data), 2664 GFP_KERNEL); 2665 if (!chanp) 2666 return -ENOMEM; 2667 2668 pd->slave_channels = chanp; 2669 for (i = 0; i < pl08x->vd->signals; i++) { 2670 /* 2671 * chanp->periph_buses will be assigned at translation 2672 */ 2673 chanp->bus_id = kasprintf(GFP_KERNEL, "slave%d", i); 2674 chanp++; 2675 } 2676 pd->num_slave_channels = pl08x->vd->signals; 2677 } 2678 2679 pl08x->pd = pd; 2680 2681 return of_dma_controller_register(adev->dev.of_node, pl08x_of_xlate, 2682 pl08x); 2683 } 2684 #else 2685 static inline int pl08x_of_probe(struct amba_device *adev, 2686 struct pl08x_driver_data *pl08x, 2687 struct device_node *np) 2688 { 2689 return -EINVAL; 2690 } 2691 #endif 2692 2693 static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) 2694 { 2695 struct pl08x_driver_data *pl08x; 2696 struct vendor_data *vd = id->data; 2697 struct device_node *np = adev->dev.of_node; 2698 u32 tsfr_size; 2699 int ret = 0; 2700 int i; 2701 2702 ret = amba_request_regions(adev, NULL); 2703 if (ret) 2704 return ret; 2705 2706 /* Ensure that we can do DMA */ 2707 ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); 2708 if (ret) 2709 goto out_no_pl08x; 2710 2711 /* Create the driver state holder */ 2712 pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); 2713 if (!pl08x) { 2714 ret = -ENOMEM; 2715 goto out_no_pl08x; 2716 } 2717 2718 /* Assign useful pointers to the driver state */ 2719 pl08x->adev = adev; 2720 pl08x->vd = vd; 2721 2722 pl08x->base = ioremap(adev->res.start, resource_size(&adev->res)); 2723 if (!pl08x->base) { 2724 ret = -ENOMEM; 2725 goto out_no_ioremap; 2726 } 2727 2728 if (vd->ftdmac020) { 2729 u32 val; 2730 2731 val = readl(pl08x->base + FTDMAC020_REVISION); 2732 dev_info(&pl08x->adev->dev, "FTDMAC020 %d.%d rel %d\n", 2733 (val >> 16) & 0xff, (val >> 8) & 0xff, val & 0xff); 2734 val = readl(pl08x->base + FTDMAC020_FEATURE); 2735 dev_info(&pl08x->adev->dev, "FTDMAC020 %d channels, " 2736 "%s built-in bridge, %s, %s linked lists\n", 2737 (val >> 12) & 0x0f, 2738 (val & BIT(10)) ? "no" : "has", 2739 (val & BIT(9)) ? "AHB0 and AHB1" : "AHB0", 2740 (val & BIT(8)) ? "supports" : "does not support"); 2741 2742 /* Vendor data from feature register */ 2743 if (!(val & BIT(8))) 2744 dev_warn(&pl08x->adev->dev, 2745 "linked lists not supported, required\n"); 2746 vd->channels = (val >> 12) & 0x0f; 2747 vd->dualmaster = !!(val & BIT(9)); 2748 } 2749 2750 /* Initialize memcpy engine */ 2751 dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask); 2752 pl08x->memcpy.dev = &adev->dev; 2753 pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources; 2754 pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy; 2755 pl08x->memcpy.device_tx_status = pl08x_dma_tx_status; 2756 pl08x->memcpy.device_issue_pending = pl08x_issue_pending; 2757 pl08x->memcpy.device_config = pl08x_config; 2758 pl08x->memcpy.device_pause = pl08x_pause; 2759 pl08x->memcpy.device_resume = pl08x_resume; 2760 pl08x->memcpy.device_terminate_all = pl08x_terminate_all; 2761 pl08x->memcpy.device_synchronize = pl08x_synchronize; 2762 pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2763 pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2764 pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM); 2765 pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT; 2766 if (vd->ftdmac020) 2767 pl08x->memcpy.copy_align = DMAENGINE_ALIGN_4_BYTES; 2768 2769 2770 /* 2771 * Initialize slave engine, if the block has no signals, that means 2772 * we have no slave support. 2773 */ 2774 if (vd->signals) { 2775 pl08x->has_slave = true; 2776 dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask); 2777 dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask); 2778 pl08x->slave.dev = &adev->dev; 2779 pl08x->slave.device_free_chan_resources = 2780 pl08x_free_chan_resources; 2781 pl08x->slave.device_tx_status = pl08x_dma_tx_status; 2782 pl08x->slave.device_issue_pending = pl08x_issue_pending; 2783 pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg; 2784 pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic; 2785 pl08x->slave.device_config = pl08x_config; 2786 pl08x->slave.device_pause = pl08x_pause; 2787 pl08x->slave.device_resume = pl08x_resume; 2788 pl08x->slave.device_terminate_all = pl08x_terminate_all; 2789 pl08x->slave.device_synchronize = pl08x_synchronize; 2790 pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS; 2791 pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS; 2792 pl08x->slave.directions = 2793 BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); 2794 pl08x->slave.residue_granularity = 2795 DMA_RESIDUE_GRANULARITY_SEGMENT; 2796 } 2797 2798 /* Get the platform data */ 2799 pl08x->pd = dev_get_platdata(&adev->dev); 2800 if (!pl08x->pd) { 2801 if (np) { 2802 ret = pl08x_of_probe(adev, pl08x, np); 2803 if (ret) 2804 goto out_no_platdata; 2805 } else { 2806 dev_err(&adev->dev, "no platform data supplied\n"); 2807 ret = -EINVAL; 2808 goto out_no_platdata; 2809 } 2810 } else { 2811 pl08x->slave.filter.map = pl08x->pd->slave_map; 2812 pl08x->slave.filter.mapcnt = pl08x->pd->slave_map_len; 2813 pl08x->slave.filter.fn = pl08x_filter_fn; 2814 } 2815 2816 /* By default, AHB1 only. If dualmaster, from platform */ 2817 pl08x->lli_buses = PL08X_AHB1; 2818 pl08x->mem_buses = PL08X_AHB1; 2819 if (pl08x->vd->dualmaster) { 2820 pl08x->lli_buses = pl08x->pd->lli_buses; 2821 pl08x->mem_buses = pl08x->pd->mem_buses; 2822 } 2823 2824 if (vd->pl080s) 2825 pl08x->lli_words = PL080S_LLI_WORDS; 2826 else 2827 pl08x->lli_words = PL080_LLI_WORDS; 2828 tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32); 2829 2830 /* A DMA memory pool for LLIs, align on 1-byte boundary */ 2831 pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev, 2832 tsfr_size, PL08X_ALIGN, 0); 2833 if (!pl08x->pool) { 2834 ret = -ENOMEM; 2835 goto out_no_lli_pool; 2836 } 2837 2838 /* Turn on the PL08x */ 2839 pl08x_ensure_on(pl08x); 2840 2841 /* Clear any pending interrupts */ 2842 if (vd->ftdmac020) 2843 /* This variant has error IRQs in bits 16-19 */ 2844 writel(0x0000FFFF, pl08x->base + PL080_ERR_CLEAR); 2845 else 2846 writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); 2847 writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); 2848 2849 /* Attach the interrupt handler */ 2850 ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); 2851 if (ret) { 2852 dev_err(&adev->dev, "%s failed to request interrupt %d\n", 2853 __func__, adev->irq[0]); 2854 goto out_no_irq; 2855 } 2856 2857 /* Initialize physical channels */ 2858 pl08x->phy_chans = kcalloc(vd->channels, sizeof(*pl08x->phy_chans), 2859 GFP_KERNEL); 2860 if (!pl08x->phy_chans) { 2861 ret = -ENOMEM; 2862 goto out_no_phychans; 2863 } 2864 2865 for (i = 0; i < vd->channels; i++) { 2866 struct pl08x_phy_chan *ch = &pl08x->phy_chans[i]; 2867 2868 ch->id = i; 2869 ch->base = pl08x->base + PL080_Cx_BASE(i); 2870 if (vd->ftdmac020) { 2871 /* FTDMA020 has a special channel busy register */ 2872 ch->reg_busy = ch->base + FTDMAC020_CH_BUSY; 2873 ch->reg_config = ch->base + FTDMAC020_CH_CFG; 2874 ch->reg_control = ch->base + FTDMAC020_CH_CSR; 2875 ch->reg_src = ch->base + FTDMAC020_CH_SRC_ADDR; 2876 ch->reg_dst = ch->base + FTDMAC020_CH_DST_ADDR; 2877 ch->reg_lli = ch->base + FTDMAC020_CH_LLP; 2878 ch->ftdmac020 = true; 2879 } else { 2880 ch->reg_config = ch->base + vd->config_offset; 2881 ch->reg_control = ch->base + PL080_CH_CONTROL; 2882 ch->reg_src = ch->base + PL080_CH_SRC_ADDR; 2883 ch->reg_dst = ch->base + PL080_CH_DST_ADDR; 2884 ch->reg_lli = ch->base + PL080_CH_LLI; 2885 } 2886 if (vd->pl080s) 2887 ch->pl080s = true; 2888 2889 spin_lock_init(&ch->lock); 2890 2891 /* 2892 * Nomadik variants can have channels that are locked 2893 * down for the secure world only. Lock up these channels 2894 * by perpetually serving a dummy virtual channel. 2895 */ 2896 if (vd->nomadik) { 2897 u32 val; 2898 2899 val = readl(ch->reg_config); 2900 if (val & (PL080N_CONFIG_ITPROT | PL080N_CONFIG_SECPROT)) { 2901 dev_info(&adev->dev, "physical channel %d reserved for secure access only\n", i); 2902 ch->locked = true; 2903 } 2904 } 2905 2906 dev_dbg(&adev->dev, "physical channel %d is %s\n", 2907 i, pl08x_phy_channel_busy(ch) ? "BUSY" : "FREE"); 2908 } 2909 2910 /* Register as many memcpy channels as there are physical channels */ 2911 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy, 2912 pl08x->vd->channels, false); 2913 if (ret <= 0) { 2914 dev_warn(&pl08x->adev->dev, 2915 "%s failed to enumerate memcpy channels - %d\n", 2916 __func__, ret); 2917 goto out_no_memcpy; 2918 } 2919 2920 /* Register slave channels */ 2921 if (pl08x->has_slave) { 2922 ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, 2923 pl08x->pd->num_slave_channels, true); 2924 if (ret < 0) { 2925 dev_warn(&pl08x->adev->dev, 2926 "%s failed to enumerate slave channels - %d\n", 2927 __func__, ret); 2928 goto out_no_slave; 2929 } 2930 } 2931 2932 ret = dma_async_device_register(&pl08x->memcpy); 2933 if (ret) { 2934 dev_warn(&pl08x->adev->dev, 2935 "%s failed to register memcpy as an async device - %d\n", 2936 __func__, ret); 2937 goto out_no_memcpy_reg; 2938 } 2939 2940 if (pl08x->has_slave) { 2941 ret = dma_async_device_register(&pl08x->slave); 2942 if (ret) { 2943 dev_warn(&pl08x->adev->dev, 2944 "%s failed to register slave as an async device - %d\n", 2945 __func__, ret); 2946 goto out_no_slave_reg; 2947 } 2948 } 2949 2950 amba_set_drvdata(adev, pl08x); 2951 init_pl08x_debugfs(pl08x); 2952 dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n", 2953 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev), 2954 (unsigned long long)adev->res.start, adev->irq[0]); 2955 2956 return 0; 2957 2958 out_no_slave_reg: 2959 dma_async_device_unregister(&pl08x->memcpy); 2960 out_no_memcpy_reg: 2961 if (pl08x->has_slave) 2962 pl08x_free_virtual_channels(&pl08x->slave); 2963 out_no_slave: 2964 pl08x_free_virtual_channels(&pl08x->memcpy); 2965 out_no_memcpy: 2966 kfree(pl08x->phy_chans); 2967 out_no_phychans: 2968 free_irq(adev->irq[0], pl08x); 2969 out_no_irq: 2970 dma_pool_destroy(pl08x->pool); 2971 out_no_lli_pool: 2972 out_no_platdata: 2973 iounmap(pl08x->base); 2974 out_no_ioremap: 2975 kfree(pl08x); 2976 out_no_pl08x: 2977 amba_release_regions(adev); 2978 return ret; 2979 } 2980 2981 /* PL080 has 8 channels and the PL080 have just 2 */ 2982 static struct vendor_data vendor_pl080 = { 2983 .config_offset = PL080_CH_CONFIG, 2984 .channels = 8, 2985 .signals = 16, 2986 .dualmaster = true, 2987 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2988 }; 2989 2990 static struct vendor_data vendor_nomadik = { 2991 .config_offset = PL080_CH_CONFIG, 2992 .channels = 8, 2993 .signals = 32, 2994 .dualmaster = true, 2995 .nomadik = true, 2996 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 2997 }; 2998 2999 static struct vendor_data vendor_pl080s = { 3000 .config_offset = PL080S_CH_CONFIG, 3001 .channels = 8, 3002 .signals = 32, 3003 .pl080s = true, 3004 .max_transfer_size = PL080S_CONTROL_TRANSFER_SIZE_MASK, 3005 }; 3006 3007 static struct vendor_data vendor_pl081 = { 3008 .config_offset = PL080_CH_CONFIG, 3009 .channels = 2, 3010 .signals = 16, 3011 .dualmaster = false, 3012 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3013 }; 3014 3015 static struct vendor_data vendor_ftdmac020 = { 3016 .config_offset = PL080_CH_CONFIG, 3017 .ftdmac020 = true, 3018 .max_transfer_size = PL080_CONTROL_TRANSFER_SIZE_MASK, 3019 }; 3020 3021 static const struct amba_id pl08x_ids[] = { 3022 /* Samsung PL080S variant */ 3023 { 3024 .id = 0x0a141080, 3025 .mask = 0xffffffff, 3026 .data = &vendor_pl080s, 3027 }, 3028 /* PL080 */ 3029 { 3030 .id = 0x00041080, 3031 .mask = 0x000fffff, 3032 .data = &vendor_pl080, 3033 }, 3034 /* PL081 */ 3035 { 3036 .id = 0x00041081, 3037 .mask = 0x000fffff, 3038 .data = &vendor_pl081, 3039 }, 3040 /* Nomadik 8815 PL080 variant */ 3041 { 3042 .id = 0x00280080, 3043 .mask = 0x00ffffff, 3044 .data = &vendor_nomadik, 3045 }, 3046 /* Faraday Technology FTDMAC020 */ 3047 { 3048 .id = 0x0003b080, 3049 .mask = 0x000fffff, 3050 .data = &vendor_ftdmac020, 3051 }, 3052 { 0, 0 }, 3053 }; 3054 3055 MODULE_DEVICE_TABLE(amba, pl08x_ids); 3056 3057 static struct amba_driver pl08x_amba_driver = { 3058 .drv.name = DRIVER_NAME, 3059 .id_table = pl08x_ids, 3060 .probe = pl08x_probe, 3061 }; 3062 3063 static int __init pl08x_init(void) 3064 { 3065 int retval; 3066 retval = amba_driver_register(&pl08x_amba_driver); 3067 if (retval) 3068 printk(KERN_WARNING DRIVER_NAME 3069 "failed to register as an AMBA device (%d)\n", 3070 retval); 3071 return retval; 3072 } 3073 subsys_initcall(pl08x_init); 3074