1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2012 Marvell International Ltd.
4 */
5
6 #include <linux/err.h>
7 #include <linux/module.h>
8 #include <linux/init.h>
9 #include <linux/types.h>
10 #include <linux/interrupt.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/slab.h>
13 #include <linux/dmaengine.h>
14 #include <linux/platform_device.h>
15 #include <linux/device.h>
16 #include <linux/platform_data/mmp_dma.h>
17 #include <linux/dmapool.h>
18 #include <linux/clk.h>
19 #include <linux/reset.h>
20 #include <linux/of_dma.h>
21 #include <linux/of.h>
22
23 #include "dmaengine.h"
24
25 #define DCSR 0x0000
26 #define DALGN 0x00a0
27 #define DINT 0x00f0
28 #define DDADR(n) (0x0200 + ((n) << 4))
29 #define DSADR(n) (0x0204 + ((n) << 4))
30 #define DTADR(n) (0x0208 + ((n) << 4))
31 #define DDADRH(n) (0x0300 + ((n) << 4))
32 #define DSADRH(n) (0x0304 + ((n) << 4))
33 #define DTADRH(n) (0x0308 + ((n) << 4))
34 #define DCMD 0x020c
35
36 #define DCSR_RUN BIT(31) /* Run Bit (read / write) */
37 #define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */
38 #define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */
39 #define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */
40 #define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */
41 #define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */
42 #define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */
43 #define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */
44
45 #define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */
46 #define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */
47 #define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */
48 #define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */
49 #define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */
50 #define DCSR_LPAEEN BIT(21) /* Long Physical Address Extension Enable */
51 #define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */
52 #define DCSR_EORINTR BIT(9) /* The end of Receive */
53
54 #define DRCMR_BASE 0x0100
55 #define DRCMR_EXT_BASE_DEFAULT 0x1100
56 #define DRCMR_REQ_LIMIT 64
57 #define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */
58 #define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */
59
60 #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */
61 #define DDADR_STOP BIT(0) /* Stop (read / write) */
62
63 #define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */
64 #define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */
65 #define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */
66 #define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */
67 #define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */
68 #define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */
69 #define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */
70 #define DCMD_BURST8 (1 << 16) /* 8 byte burst */
71 #define DCMD_BURST16 (2 << 16) /* 16 byte burst */
72 #define DCMD_BURST32 (3 << 16) /* 32 byte burst */
73 #define DCMD_WIDTH1 (1 << 14) /* 1 byte width */
74 #define DCMD_WIDTH2 (2 << 14) /* 2 byte width (HalfWord) */
75 #define DCMD_WIDTH4 (3 << 14) /* 4 byte width (Word) */
76 #define DCMD_LENGTH 0x01fff /* length mask (max = 8K - 1) */
77
78 #define PDMA_MAX_DESC_BYTES DCMD_LENGTH
79
80 struct mmp_pdma_desc_hw {
81 u32 ddadr; /* Points to the next descriptor + flags */
82 u32 dsadr; /* DSADR value for the current transfer */
83 u32 dtadr; /* DTADR value for the current transfer */
84 u32 dcmd; /* DCMD value for the current transfer */
85 /*
86 * The following 32-bit words are only used in the 64-bit, ie.
87 * LPAE (Long Physical Address Extension) mode.
88 * They are used to specify the high 32 bits of the descriptor's
89 * addresses.
90 */
91 u32 ddadrh; /* High 32-bit of DDADR */
92 u32 dsadrh; /* High 32-bit of DSADR */
93 u32 dtadrh; /* High 32-bit of DTADR */
94 u32 rsvd; /* reserved */
95 } __aligned(32);
96
97 struct mmp_pdma_desc_sw {
98 struct mmp_pdma_desc_hw desc;
99 struct list_head node;
100 struct list_head tx_list;
101 struct dma_async_tx_descriptor async_tx;
102 };
103
104 struct mmp_pdma_phy;
105
106 struct mmp_pdma_chan {
107 struct device *dev;
108 struct dma_chan chan;
109 struct dma_async_tx_descriptor desc;
110 struct mmp_pdma_phy *phy;
111 enum dma_transfer_direction dir;
112 struct dma_slave_config slave_config;
113
114 struct mmp_pdma_desc_sw *cyclic_first; /* first desc_sw if channel
115 * is in cyclic mode */
116
117 /* channel's basic info */
118 struct tasklet_struct tasklet;
119 u32 dcmd;
120 u32 drcmr;
121 u32 dev_addr;
122
123 /* list for desc */
124 spinlock_t desc_lock; /* Descriptor list lock */
125 struct list_head chain_pending; /* Link descriptors queue for pending */
126 struct list_head chain_running; /* Link descriptors queue for running */
127 bool idle; /* channel statue machine */
128 bool byte_align;
129
130 struct dma_pool *desc_pool; /* Descriptors pool */
131 };
132
133 struct mmp_pdma_phy {
134 int idx;
135 void __iomem *base;
136 struct mmp_pdma_chan *vchan;
137 };
138
139 /**
140 * struct mmp_pdma_ops - Operations for the MMP PDMA controller
141 *
142 * Hardware Register Operations (read/write hardware registers):
143 * @write_next_addr: Function to program address of next descriptor into
144 * DDADR/DDADRH
145 * @read_src_addr: Function to read the source address from DSADR/DSADRH
146 * @read_dst_addr: Function to read the destination address from DTADR/DTADRH
147 *
148 * Descriptor Memory Operations (manipulate descriptor structs in memory):
149 * @set_desc_next_addr: Function to set next descriptor address in descriptor
150 * @set_desc_src_addr: Function to set the source address in descriptor
151 * @set_desc_dst_addr: Function to set the destination address in descriptor
152 * @get_desc_src_addr: Function to get the source address from descriptor
153 * @get_desc_dst_addr: Function to get the destination address from descriptor
154 *
155 * Controller Configuration:
156 * @run_bits: Control bits in DCSR register for channel start/stop
157 * @dma_width: DMA addressing width in bits (32 or 64). Determines the
158 * DMA mask capability of the controller hardware.
159 * @drcmr_ext_base: Base DRCMR address for extended requests
160 */
161 struct mmp_pdma_ops {
162 /* Hardware Register Operations */
163 void (*write_next_addr)(struct mmp_pdma_phy *phy, dma_addr_t addr);
164 u64 (*read_src_addr)(struct mmp_pdma_phy *phy);
165 u64 (*read_dst_addr)(struct mmp_pdma_phy *phy);
166
167 /* Descriptor Memory Operations */
168 void (*set_desc_next_addr)(struct mmp_pdma_desc_hw *desc,
169 dma_addr_t addr);
170 void (*set_desc_src_addr)(struct mmp_pdma_desc_hw *desc,
171 dma_addr_t addr);
172 void (*set_desc_dst_addr)(struct mmp_pdma_desc_hw *desc,
173 dma_addr_t addr);
174 u64 (*get_desc_src_addr)(const struct mmp_pdma_desc_hw *desc);
175 u64 (*get_desc_dst_addr)(const struct mmp_pdma_desc_hw *desc);
176
177 /* Controller Configuration */
178 u32 run_bits;
179 u32 dma_width;
180 u32 drcmr_ext_base;
181 };
182
183 struct mmp_pdma_device {
184 int dma_channels;
185 void __iomem *base;
186 struct device *dev;
187 struct dma_device device;
188 struct mmp_pdma_phy *phy;
189 const struct mmp_pdma_ops *ops;
190 spinlock_t phy_lock; /* protect alloc/free phy channels */
191 };
192
193 #define tx_to_mmp_pdma_desc(tx) \
194 container_of(tx, struct mmp_pdma_desc_sw, async_tx)
195 #define to_mmp_pdma_desc(lh) \
196 container_of(lh, struct mmp_pdma_desc_sw, node)
197 #define to_mmp_pdma_chan(dchan) \
198 container_of(dchan, struct mmp_pdma_chan, chan)
199 #define to_mmp_pdma_dev(dmadev) \
200 container_of(dmadev, struct mmp_pdma_device, device)
201
mmp_pdma_get_drcmr(struct mmp_pdma_device * pdev,u32 drcmr)202 static u32 mmp_pdma_get_drcmr(struct mmp_pdma_device *pdev, u32 drcmr)
203 {
204 if (drcmr < DRCMR_REQ_LIMIT)
205 return DRCMR_BASE + (drcmr << 2);
206 return pdev->ops->drcmr_ext_base + ((drcmr - DRCMR_REQ_LIMIT) << 2);
207 }
208
209 /* For 32-bit PDMA */
write_next_addr_32(struct mmp_pdma_phy * phy,dma_addr_t addr)210 static void write_next_addr_32(struct mmp_pdma_phy *phy, dma_addr_t addr)
211 {
212 writel(addr, phy->base + DDADR(phy->idx));
213 }
214
read_src_addr_32(struct mmp_pdma_phy * phy)215 static u64 read_src_addr_32(struct mmp_pdma_phy *phy)
216 {
217 return readl(phy->base + DSADR(phy->idx));
218 }
219
read_dst_addr_32(struct mmp_pdma_phy * phy)220 static u64 read_dst_addr_32(struct mmp_pdma_phy *phy)
221 {
222 return readl(phy->base + DTADR(phy->idx));
223 }
224
set_desc_next_addr_32(struct mmp_pdma_desc_hw * desc,dma_addr_t addr)225 static void set_desc_next_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr)
226 {
227 desc->ddadr = addr;
228 }
229
set_desc_src_addr_32(struct mmp_pdma_desc_hw * desc,dma_addr_t addr)230 static void set_desc_src_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr)
231 {
232 desc->dsadr = addr;
233 }
234
set_desc_dst_addr_32(struct mmp_pdma_desc_hw * desc,dma_addr_t addr)235 static void set_desc_dst_addr_32(struct mmp_pdma_desc_hw *desc, dma_addr_t addr)
236 {
237 desc->dtadr = addr;
238 }
239
get_desc_src_addr_32(const struct mmp_pdma_desc_hw * desc)240 static u64 get_desc_src_addr_32(const struct mmp_pdma_desc_hw *desc)
241 {
242 return desc->dsadr;
243 }
244
get_desc_dst_addr_32(const struct mmp_pdma_desc_hw * desc)245 static u64 get_desc_dst_addr_32(const struct mmp_pdma_desc_hw *desc)
246 {
247 return desc->dtadr;
248 }
249
250 /* For 64-bit PDMA */
write_next_addr_64(struct mmp_pdma_phy * phy,dma_addr_t addr)251 static void write_next_addr_64(struct mmp_pdma_phy *phy, dma_addr_t addr)
252 {
253 writel(lower_32_bits(addr), phy->base + DDADR(phy->idx));
254 writel(upper_32_bits(addr), phy->base + DDADRH(phy->idx));
255 }
256
read_src_addr_64(struct mmp_pdma_phy * phy)257 static u64 read_src_addr_64(struct mmp_pdma_phy *phy)
258 {
259 u32 low = readl(phy->base + DSADR(phy->idx));
260 u32 high = readl(phy->base + DSADRH(phy->idx));
261
262 return ((u64)high << 32) | low;
263 }
264
read_dst_addr_64(struct mmp_pdma_phy * phy)265 static u64 read_dst_addr_64(struct mmp_pdma_phy *phy)
266 {
267 u32 low = readl(phy->base + DTADR(phy->idx));
268 u32 high = readl(phy->base + DTADRH(phy->idx));
269
270 return ((u64)high << 32) | low;
271 }
272
set_desc_next_addr_64(struct mmp_pdma_desc_hw * desc,dma_addr_t addr)273 static void set_desc_next_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr)
274 {
275 desc->ddadr = lower_32_bits(addr);
276 desc->ddadrh = upper_32_bits(addr);
277 }
278
set_desc_src_addr_64(struct mmp_pdma_desc_hw * desc,dma_addr_t addr)279 static void set_desc_src_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr)
280 {
281 desc->dsadr = lower_32_bits(addr);
282 desc->dsadrh = upper_32_bits(addr);
283 }
284
set_desc_dst_addr_64(struct mmp_pdma_desc_hw * desc,dma_addr_t addr)285 static void set_desc_dst_addr_64(struct mmp_pdma_desc_hw *desc, dma_addr_t addr)
286 {
287 desc->dtadr = lower_32_bits(addr);
288 desc->dtadrh = upper_32_bits(addr);
289 }
290
get_desc_src_addr_64(const struct mmp_pdma_desc_hw * desc)291 static u64 get_desc_src_addr_64(const struct mmp_pdma_desc_hw *desc)
292 {
293 return ((u64)desc->dsadrh << 32) | desc->dsadr;
294 }
295
get_desc_dst_addr_64(const struct mmp_pdma_desc_hw * desc)296 static u64 get_desc_dst_addr_64(const struct mmp_pdma_desc_hw *desc)
297 {
298 return ((u64)desc->dtadrh << 32) | desc->dtadr;
299 }
300
301 static int mmp_pdma_config_write(struct dma_chan *dchan,
302 struct dma_slave_config *cfg,
303 enum dma_transfer_direction direction);
304
enable_chan(struct mmp_pdma_phy * phy)305 static void enable_chan(struct mmp_pdma_phy *phy)
306 {
307 u32 reg, dalgn;
308 struct mmp_pdma_device *pdev;
309
310 if (!phy->vchan)
311 return;
312
313 pdev = to_mmp_pdma_dev(phy->vchan->chan.device);
314
315 reg = mmp_pdma_get_drcmr(pdev, phy->vchan->drcmr);
316 writel(DRCMR_MAPVLD | phy->idx, phy->base + reg);
317
318 dalgn = readl(phy->base + DALGN);
319 if (phy->vchan->byte_align)
320 dalgn |= 1 << phy->idx;
321 else
322 dalgn &= ~(1 << phy->idx);
323 writel(dalgn, phy->base + DALGN);
324
325 reg = (phy->idx << 2) + DCSR;
326 writel(readl(phy->base + reg) | pdev->ops->run_bits,
327 phy->base + reg);
328 }
329
disable_chan(struct mmp_pdma_phy * phy)330 static void disable_chan(struct mmp_pdma_phy *phy)
331 {
332 u32 reg, dcsr;
333
334 if (!phy)
335 return;
336
337 reg = (phy->idx << 2) + DCSR;
338 dcsr = readl(phy->base + reg);
339
340 if (phy->vchan) {
341 struct mmp_pdma_device *pdev;
342
343 pdev = to_mmp_pdma_dev(phy->vchan->chan.device);
344 writel(dcsr & ~pdev->ops->run_bits, phy->base + reg);
345 } else {
346 /* If no vchan, just clear the RUN bit */
347 writel(dcsr & ~DCSR_RUN, phy->base + reg);
348 }
349 }
350
clear_chan_irq(struct mmp_pdma_phy * phy)351 static int clear_chan_irq(struct mmp_pdma_phy *phy)
352 {
353 u32 dcsr;
354 u32 dint = readl(phy->base + DINT);
355 u32 reg = (phy->idx << 2) + DCSR;
356
357 if (!(dint & BIT(phy->idx)))
358 return -EAGAIN;
359
360 /* clear irq */
361 dcsr = readl(phy->base + reg);
362 writel(dcsr, phy->base + reg);
363 if ((dcsr & DCSR_BUSERR) && (phy->vchan))
364 dev_warn(phy->vchan->dev, "DCSR_BUSERR\n");
365
366 return 0;
367 }
368
mmp_pdma_chan_handler(int irq,void * dev_id)369 static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id)
370 {
371 struct mmp_pdma_phy *phy = dev_id;
372
373 if (clear_chan_irq(phy) != 0)
374 return IRQ_NONE;
375
376 tasklet_schedule(&phy->vchan->tasklet);
377 return IRQ_HANDLED;
378 }
379
mmp_pdma_int_handler(int irq,void * dev_id)380 static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id)
381 {
382 struct mmp_pdma_device *pdev = dev_id;
383 struct mmp_pdma_phy *phy;
384 u32 dint = readl(pdev->base + DINT);
385 int i, ret;
386 int irq_num = 0;
387
388 while (dint) {
389 i = __ffs(dint);
390 /* only handle interrupts belonging to pdma driver*/
391 if (i >= pdev->dma_channels)
392 break;
393 dint &= (dint - 1);
394 phy = &pdev->phy[i];
395 ret = mmp_pdma_chan_handler(irq, phy);
396 if (ret == IRQ_HANDLED)
397 irq_num++;
398 }
399
400 if (irq_num)
401 return IRQ_HANDLED;
402
403 return IRQ_NONE;
404 }
405
406 /* lookup free phy channel as descending priority */
lookup_phy(struct mmp_pdma_chan * pchan)407 static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan)
408 {
409 int prio, i;
410 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
411 struct mmp_pdma_phy *phy, *found = NULL;
412 unsigned long flags;
413
414 /*
415 * dma channel priorities
416 * ch 0 - 3, 16 - 19 <--> (0)
417 * ch 4 - 7, 20 - 23 <--> (1)
418 * ch 8 - 11, 24 - 27 <--> (2)
419 * ch 12 - 15, 28 - 31 <--> (3)
420 */
421
422 spin_lock_irqsave(&pdev->phy_lock, flags);
423 for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) {
424 for (i = 0; i < pdev->dma_channels; i++) {
425 if (prio != (i & 0xf) >> 2)
426 continue;
427 phy = &pdev->phy[i];
428 if (!phy->vchan) {
429 phy->vchan = pchan;
430 found = phy;
431 goto out_unlock;
432 }
433 }
434 }
435
436 out_unlock:
437 spin_unlock_irqrestore(&pdev->phy_lock, flags);
438 return found;
439 }
440
mmp_pdma_free_phy(struct mmp_pdma_chan * pchan)441 static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
442 {
443 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(pchan->chan.device);
444 unsigned long flags;
445 u32 reg;
446
447 if (!pchan->phy)
448 return;
449
450 /* clear the channel mapping in DRCMR */
451 reg = mmp_pdma_get_drcmr(pdev, pchan->drcmr);
452 writel(0, pchan->phy->base + reg);
453
454 spin_lock_irqsave(&pdev->phy_lock, flags);
455 pchan->phy->vchan = NULL;
456 pchan->phy = NULL;
457 spin_unlock_irqrestore(&pdev->phy_lock, flags);
458 }
459
460 /*
461 * start_pending_queue - transfer any pending transactions
462 * pending list ==> running list
463 */
start_pending_queue(struct mmp_pdma_chan * chan)464 static void start_pending_queue(struct mmp_pdma_chan *chan)
465 {
466 struct mmp_pdma_desc_sw *desc;
467 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device);
468
469 /* still in running, irq will start the pending list */
470 if (!chan->idle) {
471 dev_dbg(chan->dev, "DMA controller still busy\n");
472 return;
473 }
474
475 if (list_empty(&chan->chain_pending)) {
476 /* chance to re-fetch phy channel with higher prio */
477 mmp_pdma_free_phy(chan);
478 dev_dbg(chan->dev, "no pending list\n");
479 return;
480 }
481
482 if (!chan->phy) {
483 chan->phy = lookup_phy(chan);
484 if (!chan->phy) {
485 dev_dbg(chan->dev, "no free dma channel\n");
486 return;
487 }
488 }
489
490 /*
491 * pending -> running
492 * reintilize pending list
493 */
494 desc = list_first_entry(&chan->chain_pending,
495 struct mmp_pdma_desc_sw, node);
496 list_splice_tail_init(&chan->chain_pending, &chan->chain_running);
497
498 /*
499 * Program the descriptor's address into the DMA controller,
500 * then start the DMA transaction
501 */
502 pdev->ops->write_next_addr(chan->phy, desc->async_tx.phys);
503 enable_chan(chan->phy);
504 chan->idle = false;
505 }
506
507
508 /* desc->tx_list ==> pending list */
mmp_pdma_tx_submit(struct dma_async_tx_descriptor * tx)509 static dma_cookie_t mmp_pdma_tx_submit(struct dma_async_tx_descriptor *tx)
510 {
511 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(tx->chan);
512 struct mmp_pdma_desc_sw *desc = tx_to_mmp_pdma_desc(tx);
513 struct mmp_pdma_desc_sw *child;
514 unsigned long flags;
515 dma_cookie_t cookie = -EBUSY;
516
517 spin_lock_irqsave(&chan->desc_lock, flags);
518
519 list_for_each_entry(child, &desc->tx_list, node) {
520 cookie = dma_cookie_assign(&child->async_tx);
521 }
522
523 /* softly link to pending list - desc->tx_list ==> pending list */
524 list_splice_tail_init(&desc->tx_list, &chan->chain_pending);
525
526 spin_unlock_irqrestore(&chan->desc_lock, flags);
527
528 return cookie;
529 }
530
531 static struct mmp_pdma_desc_sw *
mmp_pdma_alloc_descriptor(struct mmp_pdma_chan * chan)532 mmp_pdma_alloc_descriptor(struct mmp_pdma_chan *chan)
533 {
534 struct mmp_pdma_desc_sw *desc;
535 dma_addr_t pdesc;
536
537 desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &pdesc);
538 if (!desc) {
539 dev_err(chan->dev, "out of memory for link descriptor\n");
540 return NULL;
541 }
542
543 INIT_LIST_HEAD(&desc->tx_list);
544 dma_async_tx_descriptor_init(&desc->async_tx, &chan->chan);
545 /* each desc has submit */
546 desc->async_tx.tx_submit = mmp_pdma_tx_submit;
547 desc->async_tx.phys = pdesc;
548
549 return desc;
550 }
551
552 /*
553 * mmp_pdma_alloc_chan_resources - Allocate resources for DMA channel.
554 *
555 * This function will create a dma pool for descriptor allocation.
556 * Request irq only when channel is requested
557 * Return - The number of allocated descriptors.
558 */
559
mmp_pdma_alloc_chan_resources(struct dma_chan * dchan)560 static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan)
561 {
562 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
563
564 if (chan->desc_pool)
565 return 1;
566
567 chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device),
568 chan->dev,
569 sizeof(struct mmp_pdma_desc_sw),
570 __alignof__(struct mmp_pdma_desc_sw),
571 0);
572 if (!chan->desc_pool) {
573 dev_err(chan->dev, "unable to allocate descriptor pool\n");
574 return -ENOMEM;
575 }
576
577 mmp_pdma_free_phy(chan);
578 chan->idle = true;
579 chan->dev_addr = 0;
580 return 1;
581 }
582
mmp_pdma_free_desc_list(struct mmp_pdma_chan * chan,struct list_head * list)583 static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan,
584 struct list_head *list)
585 {
586 struct mmp_pdma_desc_sw *desc, *_desc;
587
588 list_for_each_entry_safe(desc, _desc, list, node) {
589 list_del(&desc->node);
590 dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys);
591 }
592 }
593
mmp_pdma_free_chan_resources(struct dma_chan * dchan)594 static void mmp_pdma_free_chan_resources(struct dma_chan *dchan)
595 {
596 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
597 unsigned long flags;
598
599 spin_lock_irqsave(&chan->desc_lock, flags);
600 mmp_pdma_free_desc_list(chan, &chan->chain_pending);
601 mmp_pdma_free_desc_list(chan, &chan->chain_running);
602 spin_unlock_irqrestore(&chan->desc_lock, flags);
603
604 dma_pool_destroy(chan->desc_pool);
605 chan->desc_pool = NULL;
606 chan->idle = true;
607 chan->dev_addr = 0;
608 mmp_pdma_free_phy(chan);
609 return;
610 }
611
612 static struct dma_async_tx_descriptor *
mmp_pdma_prep_memcpy(struct dma_chan * dchan,dma_addr_t dma_dst,dma_addr_t dma_src,size_t len,unsigned long flags)613 mmp_pdma_prep_memcpy(struct dma_chan *dchan,
614 dma_addr_t dma_dst, dma_addr_t dma_src,
615 size_t len, unsigned long flags)
616 {
617 struct mmp_pdma_chan *chan;
618 struct mmp_pdma_device *pdev;
619 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
620 size_t copy = 0;
621
622 if (!dchan || !len)
623 return NULL;
624
625 pdev = to_mmp_pdma_dev(dchan->device);
626 chan = to_mmp_pdma_chan(dchan);
627 chan->byte_align = false;
628
629 if (!chan->dir) {
630 chan->dir = DMA_MEM_TO_MEM;
631 chan->dcmd = DCMD_INCTRGADDR | DCMD_INCSRCADDR;
632 chan->dcmd |= DCMD_BURST32;
633 }
634
635 do {
636 /* Allocate the link descriptor from DMA pool */
637 new = mmp_pdma_alloc_descriptor(chan);
638 if (!new) {
639 dev_err(chan->dev, "no memory for desc\n");
640 goto fail;
641 }
642
643 copy = min_t(size_t, len, PDMA_MAX_DESC_BYTES);
644 if (dma_src & 0x7 || dma_dst & 0x7)
645 chan->byte_align = true;
646
647 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & copy);
648 pdev->ops->set_desc_src_addr(&new->desc, dma_src);
649 pdev->ops->set_desc_dst_addr(&new->desc, dma_dst);
650
651 if (!first)
652 first = new;
653 else
654 pdev->ops->set_desc_next_addr(&prev->desc,
655 new->async_tx.phys);
656
657 new->async_tx.cookie = 0;
658 async_tx_ack(&new->async_tx);
659
660 prev = new;
661 len -= copy;
662
663 if (chan->dir == DMA_MEM_TO_DEV) {
664 dma_src += copy;
665 } else if (chan->dir == DMA_DEV_TO_MEM) {
666 dma_dst += copy;
667 } else if (chan->dir == DMA_MEM_TO_MEM) {
668 dma_src += copy;
669 dma_dst += copy;
670 }
671
672 /* Insert the link descriptor to the LD ring */
673 list_add_tail(&new->node, &first->tx_list);
674 } while (len);
675
676 first->async_tx.flags = flags; /* client is in control of this ack */
677 first->async_tx.cookie = -EBUSY;
678
679 /* last desc and fire IRQ */
680 new->desc.ddadr = DDADR_STOP;
681 new->desc.dcmd |= DCMD_ENDIRQEN;
682
683 chan->cyclic_first = NULL;
684
685 return &first->async_tx;
686
687 fail:
688 if (first)
689 mmp_pdma_free_desc_list(chan, &first->tx_list);
690 return NULL;
691 }
692
693 static struct dma_async_tx_descriptor *
mmp_pdma_prep_slave_sg(struct dma_chan * dchan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction dir,unsigned long flags,void * context)694 mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
695 unsigned int sg_len, enum dma_transfer_direction dir,
696 unsigned long flags, void *context)
697 {
698 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
699 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(dchan->device);
700 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL;
701 size_t len, avail;
702 struct scatterlist *sg;
703 dma_addr_t addr;
704 int i;
705
706 if ((sgl == NULL) || (sg_len == 0))
707 return NULL;
708
709 chan->byte_align = false;
710
711 mmp_pdma_config_write(dchan, &chan->slave_config, dir);
712
713 for_each_sg(sgl, sg, sg_len, i) {
714 addr = sg_dma_address(sg);
715 avail = sg_dma_len(sg);
716
717 do {
718 len = min_t(size_t, avail, PDMA_MAX_DESC_BYTES);
719 if (addr & 0x7)
720 chan->byte_align = true;
721
722 /* allocate and populate the descriptor */
723 new = mmp_pdma_alloc_descriptor(chan);
724 if (!new) {
725 dev_err(chan->dev, "no memory for desc\n");
726 goto fail;
727 }
728
729 new->desc.dcmd = chan->dcmd | (DCMD_LENGTH & len);
730 if (dir == DMA_MEM_TO_DEV) {
731 pdev->ops->set_desc_src_addr(&new->desc, addr);
732 new->desc.dtadr = chan->dev_addr;
733 } else {
734 new->desc.dsadr = chan->dev_addr;
735 pdev->ops->set_desc_dst_addr(&new->desc, addr);
736 }
737
738 if (!first)
739 first = new;
740 else
741 pdev->ops->set_desc_next_addr(&prev->desc,
742 new->async_tx.phys);
743
744 new->async_tx.cookie = 0;
745 async_tx_ack(&new->async_tx);
746 prev = new;
747
748 /* Insert the link descriptor to the LD ring */
749 list_add_tail(&new->node, &first->tx_list);
750
751 /* update metadata */
752 addr += len;
753 avail -= len;
754 } while (avail);
755 }
756
757 first->async_tx.cookie = -EBUSY;
758 first->async_tx.flags = flags;
759
760 /* last desc and fire IRQ */
761 new->desc.ddadr = DDADR_STOP;
762 new->desc.dcmd |= DCMD_ENDIRQEN;
763
764 chan->dir = dir;
765 chan->cyclic_first = NULL;
766
767 return &first->async_tx;
768
769 fail:
770 if (first)
771 mmp_pdma_free_desc_list(chan, &first->tx_list);
772 return NULL;
773 }
774
775 static struct dma_async_tx_descriptor *
mmp_pdma_prep_dma_cyclic(struct dma_chan * dchan,dma_addr_t buf_addr,size_t len,size_t period_len,enum dma_transfer_direction direction,unsigned long flags)776 mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan,
777 dma_addr_t buf_addr, size_t len, size_t period_len,
778 enum dma_transfer_direction direction,
779 unsigned long flags)
780 {
781 struct mmp_pdma_chan *chan;
782 struct mmp_pdma_device *pdev;
783 struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new;
784 dma_addr_t dma_src, dma_dst;
785
786 if (!dchan || !len || !period_len)
787 return NULL;
788
789 pdev = to_mmp_pdma_dev(dchan->device);
790
791 /* the buffer length must be a multiple of period_len */
792 if (len % period_len != 0)
793 return NULL;
794
795 if (period_len > PDMA_MAX_DESC_BYTES)
796 return NULL;
797
798 chan = to_mmp_pdma_chan(dchan);
799 mmp_pdma_config_write(dchan, &chan->slave_config, direction);
800
801 switch (direction) {
802 case DMA_MEM_TO_DEV:
803 dma_src = buf_addr;
804 dma_dst = chan->dev_addr;
805 break;
806 case DMA_DEV_TO_MEM:
807 dma_dst = buf_addr;
808 dma_src = chan->dev_addr;
809 break;
810 default:
811 dev_err(chan->dev, "Unsupported direction for cyclic DMA\n");
812 return NULL;
813 }
814
815 chan->dir = direction;
816
817 do {
818 /* Allocate the link descriptor from DMA pool */
819 new = mmp_pdma_alloc_descriptor(chan);
820 if (!new) {
821 dev_err(chan->dev, "no memory for desc\n");
822 goto fail;
823 }
824
825 new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN |
826 (DCMD_LENGTH & period_len));
827 pdev->ops->set_desc_src_addr(&new->desc, dma_src);
828 pdev->ops->set_desc_dst_addr(&new->desc, dma_dst);
829
830 if (!first)
831 first = new;
832 else
833 pdev->ops->set_desc_next_addr(&prev->desc,
834 new->async_tx.phys);
835
836 new->async_tx.cookie = 0;
837 async_tx_ack(&new->async_tx);
838
839 prev = new;
840 len -= period_len;
841
842 if (chan->dir == DMA_MEM_TO_DEV)
843 dma_src += period_len;
844 else
845 dma_dst += period_len;
846
847 /* Insert the link descriptor to the LD ring */
848 list_add_tail(&new->node, &first->tx_list);
849 } while (len);
850
851 first->async_tx.flags = flags; /* client is in control of this ack */
852 first->async_tx.cookie = -EBUSY;
853
854 /* make the cyclic link */
855 pdev->ops->set_desc_next_addr(&new->desc, first->async_tx.phys);
856 chan->cyclic_first = first;
857
858 return &first->async_tx;
859
860 fail:
861 if (first)
862 mmp_pdma_free_desc_list(chan, &first->tx_list);
863 return NULL;
864 }
865
mmp_pdma_config_write(struct dma_chan * dchan,struct dma_slave_config * cfg,enum dma_transfer_direction direction)866 static int mmp_pdma_config_write(struct dma_chan *dchan,
867 struct dma_slave_config *cfg,
868 enum dma_transfer_direction direction)
869 {
870 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
871 u32 maxburst = 0, addr = 0;
872 enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED;
873
874 if (!dchan)
875 return -EINVAL;
876
877 if (direction == DMA_DEV_TO_MEM) {
878 chan->dcmd = DCMD_INCTRGADDR | DCMD_FLOWSRC;
879 maxburst = cfg->src_maxburst;
880 width = cfg->src_addr_width;
881 addr = cfg->src_addr;
882 } else if (direction == DMA_MEM_TO_DEV) {
883 chan->dcmd = DCMD_INCSRCADDR | DCMD_FLOWTRG;
884 maxburst = cfg->dst_maxburst;
885 width = cfg->dst_addr_width;
886 addr = cfg->dst_addr;
887 }
888
889 if (width == DMA_SLAVE_BUSWIDTH_1_BYTE)
890 chan->dcmd |= DCMD_WIDTH1;
891 else if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
892 chan->dcmd |= DCMD_WIDTH2;
893 else if (width == DMA_SLAVE_BUSWIDTH_4_BYTES)
894 chan->dcmd |= DCMD_WIDTH4;
895
896 if (maxburst == 8)
897 chan->dcmd |= DCMD_BURST8;
898 else if (maxburst == 16)
899 chan->dcmd |= DCMD_BURST16;
900 else if (maxburst == 32)
901 chan->dcmd |= DCMD_BURST32;
902
903 chan->dir = direction;
904 chan->dev_addr = addr;
905
906 return 0;
907 }
908
mmp_pdma_config(struct dma_chan * dchan,struct dma_slave_config * cfg)909 static int mmp_pdma_config(struct dma_chan *dchan,
910 struct dma_slave_config *cfg)
911 {
912 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
913
914 memcpy(&chan->slave_config, cfg, sizeof(*cfg));
915 return 0;
916 }
917
mmp_pdma_terminate_all(struct dma_chan * dchan)918 static int mmp_pdma_terminate_all(struct dma_chan *dchan)
919 {
920 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
921 unsigned long flags;
922
923 if (!dchan)
924 return -EINVAL;
925
926 disable_chan(chan->phy);
927 mmp_pdma_free_phy(chan);
928 spin_lock_irqsave(&chan->desc_lock, flags);
929 mmp_pdma_free_desc_list(chan, &chan->chain_pending);
930 mmp_pdma_free_desc_list(chan, &chan->chain_running);
931 spin_unlock_irqrestore(&chan->desc_lock, flags);
932 chan->idle = true;
933
934 return 0;
935 }
936
mmp_pdma_residue(struct mmp_pdma_chan * chan,dma_cookie_t cookie)937 static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan,
938 dma_cookie_t cookie)
939 {
940 struct mmp_pdma_desc_sw *sw;
941 struct mmp_pdma_device *pdev = to_mmp_pdma_dev(chan->chan.device);
942 unsigned long flags;
943 u64 curr;
944 u32 residue = 0;
945 bool passed = false;
946 bool cyclic = chan->cyclic_first != NULL;
947
948 /*
949 * If the channel does not have a phy pointer anymore, it has already
950 * been completed. Therefore, its residue is 0.
951 */
952 if (!chan->phy)
953 return 0;
954
955 if (chan->dir == DMA_DEV_TO_MEM)
956 curr = pdev->ops->read_dst_addr(chan->phy);
957 else
958 curr = pdev->ops->read_src_addr(chan->phy);
959
960 spin_lock_irqsave(&chan->desc_lock, flags);
961
962 list_for_each_entry(sw, &chan->chain_running, node) {
963 u64 start, end;
964 u32 len;
965
966 if (chan->dir == DMA_DEV_TO_MEM)
967 start = pdev->ops->get_desc_dst_addr(&sw->desc);
968 else
969 start = pdev->ops->get_desc_src_addr(&sw->desc);
970
971 len = sw->desc.dcmd & DCMD_LENGTH;
972 end = start + len;
973
974 /*
975 * 'passed' will be latched once we found the descriptor which
976 * lies inside the boundaries of the curr pointer. All
977 * descriptors that occur in the list _after_ we found that
978 * partially handled descriptor are still to be processed and
979 * are hence added to the residual bytes counter.
980 */
981
982 if (passed) {
983 residue += len;
984 } else if (curr >= start && curr <= end) {
985 residue += (u32)(end - curr);
986 passed = true;
987 }
988
989 /*
990 * Descriptors that have the ENDIRQEN bit set mark the end of a
991 * transaction chain, and the cookie assigned with it has been
992 * returned previously from mmp_pdma_tx_submit().
993 *
994 * In case we have multiple transactions in the running chain,
995 * and the cookie does not match the one the user asked us
996 * about, reset the state variables and start over.
997 *
998 * This logic does not apply to cyclic transactions, where all
999 * descriptors have the ENDIRQEN bit set, and for which we
1000 * can't have multiple transactions on one channel anyway.
1001 */
1002 if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN))
1003 continue;
1004
1005 if (sw->async_tx.cookie == cookie) {
1006 spin_unlock_irqrestore(&chan->desc_lock, flags);
1007 return residue;
1008 } else {
1009 residue = 0;
1010 passed = false;
1011 }
1012 }
1013
1014 spin_unlock_irqrestore(&chan->desc_lock, flags);
1015
1016 /* We should only get here in case of cyclic transactions */
1017 return residue;
1018 }
1019
mmp_pdma_tx_status(struct dma_chan * dchan,dma_cookie_t cookie,struct dma_tx_state * txstate)1020 static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan,
1021 dma_cookie_t cookie,
1022 struct dma_tx_state *txstate)
1023 {
1024 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
1025 enum dma_status ret;
1026
1027 ret = dma_cookie_status(dchan, cookie, txstate);
1028 if (likely(ret != DMA_ERROR))
1029 dma_set_residue(txstate, mmp_pdma_residue(chan, cookie));
1030
1031 return ret;
1032 }
1033
1034 /*
1035 * mmp_pdma_issue_pending - Issue the DMA start command
1036 * pending list ==> running list
1037 */
mmp_pdma_issue_pending(struct dma_chan * dchan)1038 static void mmp_pdma_issue_pending(struct dma_chan *dchan)
1039 {
1040 struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan);
1041 unsigned long flags;
1042
1043 spin_lock_irqsave(&chan->desc_lock, flags);
1044 start_pending_queue(chan);
1045 spin_unlock_irqrestore(&chan->desc_lock, flags);
1046 }
1047
1048 /*
1049 * dma_do_tasklet
1050 * Do call back
1051 * Start pending list
1052 */
dma_do_tasklet(struct tasklet_struct * t)1053 static void dma_do_tasklet(struct tasklet_struct *t)
1054 {
1055 struct mmp_pdma_chan *chan = from_tasklet(chan, t, tasklet);
1056 struct mmp_pdma_desc_sw *desc, *_desc;
1057 LIST_HEAD(chain_cleanup);
1058 unsigned long flags;
1059 struct dmaengine_desc_callback cb;
1060
1061 if (chan->cyclic_first) {
1062 spin_lock_irqsave(&chan->desc_lock, flags);
1063 desc = chan->cyclic_first;
1064 dmaengine_desc_get_callback(&desc->async_tx, &cb);
1065 spin_unlock_irqrestore(&chan->desc_lock, flags);
1066
1067 dmaengine_desc_callback_invoke(&cb, NULL);
1068
1069 return;
1070 }
1071
1072 /* submit pending list; callback for each desc; free desc */
1073 spin_lock_irqsave(&chan->desc_lock, flags);
1074
1075 list_for_each_entry_safe(desc, _desc, &chan->chain_running, node) {
1076 /*
1077 * move the descriptors to a temporary list so we can drop
1078 * the lock during the entire cleanup operation
1079 */
1080 list_move(&desc->node, &chain_cleanup);
1081
1082 /*
1083 * Look for the first list entry which has the ENDIRQEN flag
1084 * set. That is the descriptor we got an interrupt for, so
1085 * complete that transaction and its cookie.
1086 */
1087 if (desc->desc.dcmd & DCMD_ENDIRQEN) {
1088 dma_cookie_t cookie = desc->async_tx.cookie;
1089 dma_cookie_complete(&desc->async_tx);
1090 dev_dbg(chan->dev, "completed_cookie=%d\n", cookie);
1091 break;
1092 }
1093 }
1094
1095 /*
1096 * The hardware is idle and ready for more when the
1097 * chain_running list is empty.
1098 */
1099 chan->idle = list_empty(&chan->chain_running);
1100
1101 /* Start any pending transactions automatically */
1102 start_pending_queue(chan);
1103 spin_unlock_irqrestore(&chan->desc_lock, flags);
1104
1105 /* Run the callback for each descriptor, in order */
1106 list_for_each_entry_safe(desc, _desc, &chain_cleanup, node) {
1107 struct dma_async_tx_descriptor *txd = &desc->async_tx;
1108
1109 /* Remove from the list of transactions */
1110 list_del(&desc->node);
1111 /* Run the link descriptor callback function */
1112 dmaengine_desc_get_callback(txd, &cb);
1113 dmaengine_desc_callback_invoke(&cb, NULL);
1114
1115 dma_pool_free(chan->desc_pool, desc, txd->phys);
1116 }
1117 }
1118
mmp_pdma_remove(struct platform_device * op)1119 static void mmp_pdma_remove(struct platform_device *op)
1120 {
1121 struct mmp_pdma_device *pdev = platform_get_drvdata(op);
1122 struct mmp_pdma_phy *phy;
1123 int i, irq = 0, irq_num = 0;
1124
1125 if (op->dev.of_node)
1126 of_dma_controller_free(op->dev.of_node);
1127
1128 for (i = 0; i < pdev->dma_channels; i++) {
1129 if (platform_get_irq(op, i) > 0)
1130 irq_num++;
1131 }
1132
1133 if (irq_num != pdev->dma_channels) {
1134 irq = platform_get_irq(op, 0);
1135 devm_free_irq(&op->dev, irq, pdev);
1136 } else {
1137 for (i = 0; i < pdev->dma_channels; i++) {
1138 phy = &pdev->phy[i];
1139 irq = platform_get_irq(op, i);
1140 devm_free_irq(&op->dev, irq, phy);
1141 }
1142 }
1143
1144 dma_async_device_unregister(&pdev->device);
1145 }
1146
mmp_pdma_chan_init(struct mmp_pdma_device * pdev,int idx,int irq)1147 static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq)
1148 {
1149 struct mmp_pdma_phy *phy = &pdev->phy[idx];
1150 struct mmp_pdma_chan *chan;
1151 int ret;
1152
1153 chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL);
1154 if (chan == NULL)
1155 return -ENOMEM;
1156
1157 phy->idx = idx;
1158 phy->base = pdev->base;
1159
1160 if (irq) {
1161 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler,
1162 IRQF_SHARED, "pdma", phy);
1163 if (ret) {
1164 dev_err(pdev->dev, "channel request irq fail!\n");
1165 return ret;
1166 }
1167 }
1168
1169 spin_lock_init(&chan->desc_lock);
1170 chan->dev = pdev->dev;
1171 chan->chan.device = &pdev->device;
1172 tasklet_setup(&chan->tasklet, dma_do_tasklet);
1173 INIT_LIST_HEAD(&chan->chain_pending);
1174 INIT_LIST_HEAD(&chan->chain_running);
1175
1176 /* register virt channel to dma engine */
1177 list_add_tail(&chan->chan.device_node, &pdev->device.channels);
1178
1179 return 0;
1180 }
1181
1182 static const struct mmp_pdma_ops marvell_pdma_v1_ops = {
1183 .write_next_addr = write_next_addr_32,
1184 .read_src_addr = read_src_addr_32,
1185 .read_dst_addr = read_dst_addr_32,
1186 .set_desc_next_addr = set_desc_next_addr_32,
1187 .set_desc_src_addr = set_desc_src_addr_32,
1188 .set_desc_dst_addr = set_desc_dst_addr_32,
1189 .get_desc_src_addr = get_desc_src_addr_32,
1190 .get_desc_dst_addr = get_desc_dst_addr_32,
1191 .run_bits = (DCSR_RUN),
1192 .dma_width = 32,
1193 .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
1194 };
1195
1196 static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
1197 .write_next_addr = write_next_addr_64,
1198 .read_src_addr = read_src_addr_64,
1199 .read_dst_addr = read_dst_addr_64,
1200 .set_desc_next_addr = set_desc_next_addr_64,
1201 .set_desc_src_addr = set_desc_src_addr_64,
1202 .set_desc_dst_addr = set_desc_dst_addr_64,
1203 .get_desc_src_addr = get_desc_src_addr_64,
1204 .get_desc_dst_addr = get_desc_dst_addr_64,
1205 .run_bits = (DCSR_RUN | DCSR_LPAEEN),
1206 .dma_width = 64,
1207 .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
1208 };
1209
1210 static const struct mmp_pdma_ops spacemit_k3_pdma_ops = {
1211 .write_next_addr = write_next_addr_64,
1212 .read_src_addr = read_src_addr_64,
1213 .read_dst_addr = read_dst_addr_64,
1214 .set_desc_next_addr = set_desc_next_addr_64,
1215 .set_desc_src_addr = set_desc_src_addr_64,
1216 .set_desc_dst_addr = set_desc_dst_addr_64,
1217 .get_desc_src_addr = get_desc_src_addr_64,
1218 .get_desc_dst_addr = get_desc_dst_addr_64,
1219 .run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN),
1220 .dma_width = 64,
1221 .drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
1222 };
1223
1224 static const struct of_device_id mmp_pdma_dt_ids[] = {
1225 {
1226 .compatible = "marvell,pdma-1.0",
1227 .data = &marvell_pdma_v1_ops
1228 }, {
1229 .compatible = "spacemit,k1-pdma",
1230 .data = &spacemit_k1_pdma_ops
1231 }, {
1232 .compatible = "spacemit,k3-pdma",
1233 .data = &spacemit_k3_pdma_ops
1234 }, {
1235 /* sentinel */
1236 }
1237 };
1238 MODULE_DEVICE_TABLE(of, mmp_pdma_dt_ids);
1239
mmp_pdma_dma_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)1240 static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec,
1241 struct of_dma *ofdma)
1242 {
1243 struct mmp_pdma_device *d = ofdma->of_dma_data;
1244 struct dma_chan *chan;
1245
1246 chan = dma_get_any_slave_channel(&d->device);
1247 if (!chan)
1248 return NULL;
1249
1250 to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0];
1251
1252 return chan;
1253 }
1254
mmp_pdma_probe(struct platform_device * op)1255 static int mmp_pdma_probe(struct platform_device *op)
1256 {
1257 struct mmp_pdma_device *pdev;
1258 struct mmp_dma_platdata *pdata = dev_get_platdata(&op->dev);
1259 struct clk *clk;
1260 struct reset_control *rst;
1261 int i, ret, irq = 0;
1262 int dma_channels = 0, irq_num = 0;
1263 const enum dma_slave_buswidth widths =
1264 DMA_SLAVE_BUSWIDTH_1_BYTE | DMA_SLAVE_BUSWIDTH_2_BYTES |
1265 DMA_SLAVE_BUSWIDTH_4_BYTES;
1266
1267 pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL);
1268 if (!pdev)
1269 return -ENOMEM;
1270
1271 pdev->dev = &op->dev;
1272
1273 spin_lock_init(&pdev->phy_lock);
1274
1275 pdev->base = devm_platform_ioremap_resource(op, 0);
1276 if (IS_ERR(pdev->base))
1277 return PTR_ERR(pdev->base);
1278
1279 clk = devm_clk_get_optional_enabled(pdev->dev, NULL);
1280 if (IS_ERR(clk))
1281 return PTR_ERR(clk);
1282
1283 rst = devm_reset_control_get_optional_exclusive_deasserted(pdev->dev,
1284 NULL);
1285 if (IS_ERR(rst))
1286 return PTR_ERR(rst);
1287
1288 pdev->ops = of_device_get_match_data(&op->dev);
1289 if (!pdev->ops)
1290 return -ENODEV;
1291
1292 if (pdev->dev->of_node) {
1293 /* Parse new and deprecated dma-channels properties */
1294 if (of_property_read_u32(pdev->dev->of_node, "dma-channels",
1295 &dma_channels))
1296 of_property_read_u32(pdev->dev->of_node, "#dma-channels",
1297 &dma_channels);
1298 } else if (pdata && pdata->dma_channels) {
1299 dma_channels = pdata->dma_channels;
1300 } else {
1301 dma_channels = 32; /* default 32 channel */
1302 }
1303 pdev->dma_channels = dma_channels;
1304
1305 for (i = 0; i < dma_channels; i++) {
1306 if (platform_get_irq_optional(op, i) > 0)
1307 irq_num++;
1308 }
1309
1310 pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy),
1311 GFP_KERNEL);
1312 if (pdev->phy == NULL)
1313 return -ENOMEM;
1314
1315 INIT_LIST_HEAD(&pdev->device.channels);
1316
1317 if (irq_num != dma_channels) {
1318 /* all chan share one irq, demux inside */
1319 irq = platform_get_irq(op, 0);
1320 ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler,
1321 IRQF_SHARED, "pdma", pdev);
1322 if (ret)
1323 return ret;
1324 }
1325
1326 for (i = 0; i < dma_channels; i++) {
1327 irq = (irq_num != dma_channels) ? 0 : platform_get_irq(op, i);
1328 ret = mmp_pdma_chan_init(pdev, i, irq);
1329 if (ret)
1330 return ret;
1331 }
1332
1333 dma_cap_set(DMA_SLAVE, pdev->device.cap_mask);
1334 dma_cap_set(DMA_MEMCPY, pdev->device.cap_mask);
1335 dma_cap_set(DMA_CYCLIC, pdev->device.cap_mask);
1336 dma_cap_set(DMA_PRIVATE, pdev->device.cap_mask);
1337 pdev->device.dev = &op->dev;
1338 pdev->device.device_alloc_chan_resources = mmp_pdma_alloc_chan_resources;
1339 pdev->device.device_free_chan_resources = mmp_pdma_free_chan_resources;
1340 pdev->device.device_tx_status = mmp_pdma_tx_status;
1341 pdev->device.device_prep_dma_memcpy = mmp_pdma_prep_memcpy;
1342 pdev->device.device_prep_slave_sg = mmp_pdma_prep_slave_sg;
1343 pdev->device.device_prep_dma_cyclic = mmp_pdma_prep_dma_cyclic;
1344 pdev->device.device_issue_pending = mmp_pdma_issue_pending;
1345 pdev->device.device_config = mmp_pdma_config;
1346 pdev->device.device_terminate_all = mmp_pdma_terminate_all;
1347 pdev->device.copy_align = DMAENGINE_ALIGN_8_BYTES;
1348 pdev->device.src_addr_widths = widths;
1349 pdev->device.dst_addr_widths = widths;
1350 pdev->device.directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM);
1351 pdev->device.residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
1352
1353 /* Set DMA mask based on controller hardware capabilities */
1354 dma_set_mask_and_coherent(pdev->dev,
1355 DMA_BIT_MASK(pdev->ops->dma_width));
1356
1357 ret = dma_async_device_register(&pdev->device);
1358 if (ret) {
1359 dev_err(pdev->device.dev, "unable to register\n");
1360 return ret;
1361 }
1362
1363 if (op->dev.of_node) {
1364 /* Device-tree DMA controller registration */
1365 ret = of_dma_controller_register(op->dev.of_node,
1366 mmp_pdma_dma_xlate, pdev);
1367 if (ret < 0) {
1368 dev_err(&op->dev, "of_dma_controller_register failed\n");
1369 dma_async_device_unregister(&pdev->device);
1370 return ret;
1371 }
1372 }
1373
1374 platform_set_drvdata(op, pdev);
1375 dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels);
1376 return 0;
1377 }
1378
1379 static const struct platform_device_id mmp_pdma_id_table[] = {
1380 { "mmp-pdma", },
1381 { },
1382 };
1383
1384 static struct platform_driver mmp_pdma_driver = {
1385 .driver = {
1386 .name = "mmp-pdma",
1387 .of_match_table = mmp_pdma_dt_ids,
1388 },
1389 .id_table = mmp_pdma_id_table,
1390 .probe = mmp_pdma_probe,
1391 .remove = mmp_pdma_remove,
1392 };
1393
1394 module_platform_driver(mmp_pdma_driver);
1395
1396 MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver");
1397 MODULE_AUTHOR("Marvell International Ltd.");
1398 MODULE_LICENSE("GPL v2");
1399