1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2013-2014 Renesas Electronics Europe Ltd.
4 * Author: Guennadi Liakhovetski <g.liakhovetski@gmx.de>
5 */
6
7 #include <linux/bitmap.h>
8 #include <linux/bitops.h>
9 #include <linux/clk.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmaengine.h>
12 #include <linux/err.h>
13 #include <linux/interrupt.h>
14 #include <linux/io.h>
15 #include <linux/log2.h>
16 #include <linux/module.h>
17 #include <linux/of.h>
18 #include <linux/of_dma.h>
19 #include <linux/platform_device.h>
20 #include <linux/slab.h>
21
22 #include <dt-bindings/dma/nbpfaxi.h>
23
24 #include "dmaengine.h"
25
26 #define NBPF_REG_CHAN_OFFSET 0
27 #define NBPF_REG_CHAN_SIZE 0x40
28
29 /* Channel Current Transaction Byte register */
30 #define NBPF_CHAN_CUR_TR_BYTE 0x20
31
32 /* Channel Status register */
33 #define NBPF_CHAN_STAT 0x24
34 #define NBPF_CHAN_STAT_EN 1
35 #define NBPF_CHAN_STAT_TACT 4
36 #define NBPF_CHAN_STAT_ERR 0x10
37 #define NBPF_CHAN_STAT_END 0x20
38 #define NBPF_CHAN_STAT_TC 0x40
39 #define NBPF_CHAN_STAT_DER 0x400
40
41 /* Channel Control register */
42 #define NBPF_CHAN_CTRL 0x28
43 #define NBPF_CHAN_CTRL_SETEN 1
44 #define NBPF_CHAN_CTRL_CLREN 2
45 #define NBPF_CHAN_CTRL_STG 4
46 #define NBPF_CHAN_CTRL_SWRST 8
47 #define NBPF_CHAN_CTRL_CLRRQ 0x10
48 #define NBPF_CHAN_CTRL_CLREND 0x20
49 #define NBPF_CHAN_CTRL_CLRTC 0x40
50 #define NBPF_CHAN_CTRL_SETSUS 0x100
51 #define NBPF_CHAN_CTRL_CLRSUS 0x200
52
53 /* Channel Configuration register */
54 #define NBPF_CHAN_CFG 0x2c
55 #define NBPF_CHAN_CFG_SEL 7 /* terminal SELect: 0..7 */
56 #define NBPF_CHAN_CFG_REQD 8 /* REQuest Direction: DMAREQ is 0: input, 1: output */
57 #define NBPF_CHAN_CFG_LOEN 0x10 /* LOw ENable: low DMA request line is: 0: inactive, 1: active */
58 #define NBPF_CHAN_CFG_HIEN 0x20 /* HIgh ENable: high DMA request line is: 0: inactive, 1: active */
59 #define NBPF_CHAN_CFG_LVL 0x40 /* LeVeL: DMA request line is sensed as 0: edge, 1: level */
60 #define NBPF_CHAN_CFG_AM 0x700 /* ACK Mode: 0: Pulse mode, 1: Level mode, b'1x: Bus Cycle */
61 #define NBPF_CHAN_CFG_SDS 0xf000 /* Source Data Size: 0: 8 bits,... , 7: 1024 bits */
62 #define NBPF_CHAN_CFG_DDS 0xf0000 /* Destination Data Size: as above */
63 #define NBPF_CHAN_CFG_SAD 0x100000 /* Source ADdress counting: 0: increment, 1: fixed */
64 #define NBPF_CHAN_CFG_DAD 0x200000 /* Destination ADdress counting: 0: increment, 1: fixed */
65 #define NBPF_CHAN_CFG_TM 0x400000 /* Transfer Mode: 0: single, 1: block TM */
66 #define NBPF_CHAN_CFG_DEM 0x1000000 /* DMAEND interrupt Mask */
67 #define NBPF_CHAN_CFG_TCM 0x2000000 /* DMATCO interrupt Mask */
68 #define NBPF_CHAN_CFG_SBE 0x8000000 /* Sweep Buffer Enable */
69 #define NBPF_CHAN_CFG_RSEL 0x10000000 /* RM: Register Set sELect */
70 #define NBPF_CHAN_CFG_RSW 0x20000000 /* RM: Register Select sWitch */
71 #define NBPF_CHAN_CFG_REN 0x40000000 /* RM: Register Set Enable */
72 #define NBPF_CHAN_CFG_DMS 0x80000000 /* 0: register mode (RM), 1: link mode (LM) */
73
74 #define NBPF_CHAN_NXLA 0x38
75 #define NBPF_CHAN_CRLA 0x3c
76
77 /* Link Header field */
78 #define NBPF_HEADER_LV 1
79 #define NBPF_HEADER_LE 2
80 #define NBPF_HEADER_WBD 4
81 #define NBPF_HEADER_DIM 8
82
83 #define NBPF_CTRL 0x300
84 #define NBPF_CTRL_PR 1 /* 0: fixed priority, 1: round robin */
85 #define NBPF_CTRL_LVINT 2 /* DMAEND and DMAERR signalling: 0: pulse, 1: level */
86
87 #define NBPF_DSTAT_ER 0x314
88 #define NBPF_DSTAT_END 0x318
89
90 #define NBPF_DMA_BUSWIDTHS \
91 (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \
92 BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
93 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
94 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \
95 BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
96
97 struct nbpf_config {
98 int num_channels;
99 int buffer_size;
100 };
101
102 /*
103 * We've got 3 types of objects, used to describe DMA transfers:
104 * 1. high-level descriptor, containing a struct dma_async_tx_descriptor object
105 * in it, used to communicate with the user
106 * 2. hardware DMA link descriptors, that we pass to DMAC for DMA transfer
107 * queuing, these must be DMAable, using either the streaming DMA API or
108 * allocated from coherent memory - one per SG segment
109 * 3. one per SG segment descriptors, used to manage HW link descriptors from
110 * (2). They do not have to be DMAable. They can either be (a) allocated
111 * together with link descriptors as mixed (DMA / CPU) objects, or (b)
112 * separately. Even if allocated separately it would be best to link them
113 * to link descriptors once during channel resource allocation and always
114 * use them as a single object.
115 * Therefore for both cases (a) and (b) at run-time objects (2) and (3) shall be
116 * treated as a single SG segment descriptor.
117 */
118
119 struct nbpf_link_reg {
120 u32 header;
121 u32 src_addr;
122 u32 dst_addr;
123 u32 transaction_size;
124 u32 config;
125 u32 interval;
126 u32 extension;
127 u32 next;
128 } __packed;
129
130 struct nbpf_device;
131 struct nbpf_channel;
132 struct nbpf_desc;
133
134 struct nbpf_link_desc {
135 struct nbpf_link_reg *hwdesc;
136 dma_addr_t hwdesc_dma_addr;
137 struct nbpf_desc *desc;
138 struct list_head node;
139 };
140
141 /**
142 * struct nbpf_desc - DMA transfer descriptor
143 * @async_tx: dmaengine object
144 * @user_wait: waiting for a user ack
145 * @length: total transfer length
146 * @chan: associated DMAC channel
147 * @sg: list of hardware descriptors, represented by struct nbpf_link_desc
148 * @node: member in channel descriptor lists
149 */
150 struct nbpf_desc {
151 struct dma_async_tx_descriptor async_tx;
152 bool user_wait;
153 size_t length;
154 struct nbpf_channel *chan;
155 struct list_head sg;
156 struct list_head node;
157 };
158
159 /* Take a wild guess: allocate 4 segments per descriptor */
160 #define NBPF_SEGMENTS_PER_DESC 4
161 #define NBPF_DESCS_PER_PAGE ((PAGE_SIZE - sizeof(struct list_head)) / \
162 (sizeof(struct nbpf_desc) + \
163 NBPF_SEGMENTS_PER_DESC * \
164 (sizeof(struct nbpf_link_desc) + sizeof(struct nbpf_link_reg))))
165 #define NBPF_SEGMENTS_PER_PAGE (NBPF_SEGMENTS_PER_DESC * NBPF_DESCS_PER_PAGE)
166
167 struct nbpf_desc_page {
168 struct list_head node;
169 struct nbpf_desc desc[NBPF_DESCS_PER_PAGE];
170 struct nbpf_link_desc ldesc[NBPF_SEGMENTS_PER_PAGE];
171 struct nbpf_link_reg hwdesc[NBPF_SEGMENTS_PER_PAGE];
172 };
173
174 /**
175 * struct nbpf_channel - one DMAC channel
176 * @dma_chan: standard dmaengine channel object
177 * @tasklet: channel specific tasklet used for callbacks
178 * @base: register address base
179 * @nbpf: DMAC
180 * @name: IRQ name
181 * @irq: IRQ number
182 * @slave_src_addr: source address for slave DMA
183 * @slave_src_width: source slave data size in bytes
184 * @slave_src_burst: maximum source slave burst size in bytes
185 * @slave_dst_addr: destination address for slave DMA
186 * @slave_dst_width: destination slave data size in bytes
187 * @slave_dst_burst: maximum destination slave burst size in bytes
188 * @terminal: DMA terminal, assigned to this channel
189 * @dmarq_cfg: DMA request line configuration - high / low, edge / level for NBPF_CHAN_CFG
190 * @flags: configuration flags from DT
191 * @lock: protect descriptor lists
192 * @free_links: list of free link descriptors
193 * @free: list of free descriptors
194 * @queued: list of queued descriptors
195 * @active: list of descriptors, scheduled for processing
196 * @done: list of completed descriptors, waiting post-processing
197 * @desc_page: list of additionally allocated descriptor pages - if any
198 * @running: linked descriptor of running transaction
199 * @paused: are translations on this channel paused?
200 */
201 struct nbpf_channel {
202 struct dma_chan dma_chan;
203 struct tasklet_struct tasklet;
204 void __iomem *base;
205 struct nbpf_device *nbpf;
206 char name[16];
207 int irq;
208 dma_addr_t slave_src_addr;
209 size_t slave_src_width;
210 size_t slave_src_burst;
211 dma_addr_t slave_dst_addr;
212 size_t slave_dst_width;
213 size_t slave_dst_burst;
214 unsigned int terminal;
215 u32 dmarq_cfg;
216 unsigned long flags;
217 spinlock_t lock;
218 struct list_head free_links;
219 struct list_head free;
220 struct list_head queued;
221 struct list_head active;
222 struct list_head done;
223 struct list_head desc_page;
224 struct nbpf_desc *running;
225 bool paused;
226 };
227
228 struct nbpf_device {
229 struct dma_device dma_dev;
230 void __iomem *base;
231 u32 max_burst_mem_read;
232 u32 max_burst_mem_write;
233 struct clk *clk;
234 const struct nbpf_config *config;
235 unsigned int eirq;
236 struct nbpf_channel chan[];
237 };
238
239 enum nbpf_model {
240 NBPF1B4,
241 NBPF1B8,
242 NBPF1B16,
243 NBPF4B4,
244 NBPF4B8,
245 NBPF4B16,
246 NBPF8B4,
247 NBPF8B8,
248 NBPF8B16,
249 };
250
251 static struct nbpf_config nbpf_cfg[] = {
252 [NBPF1B4] = {
253 .num_channels = 1,
254 .buffer_size = 4,
255 },
256 [NBPF1B8] = {
257 .num_channels = 1,
258 .buffer_size = 8,
259 },
260 [NBPF1B16] = {
261 .num_channels = 1,
262 .buffer_size = 16,
263 },
264 [NBPF4B4] = {
265 .num_channels = 4,
266 .buffer_size = 4,
267 },
268 [NBPF4B8] = {
269 .num_channels = 4,
270 .buffer_size = 8,
271 },
272 [NBPF4B16] = {
273 .num_channels = 4,
274 .buffer_size = 16,
275 },
276 [NBPF8B4] = {
277 .num_channels = 8,
278 .buffer_size = 4,
279 },
280 [NBPF8B8] = {
281 .num_channels = 8,
282 .buffer_size = 8,
283 },
284 [NBPF8B16] = {
285 .num_channels = 8,
286 .buffer_size = 16,
287 },
288 };
289
290 #define nbpf_to_chan(d) container_of(d, struct nbpf_channel, dma_chan)
291
292 /*
293 * dmaengine drivers seem to have a lot in common and instead of sharing more
294 * code, they reimplement those common algorithms independently. In this driver
295 * we try to separate the hardware-specific part from the (largely) generic
296 * part. This improves code readability and makes it possible in the future to
297 * reuse the generic code in form of a helper library. That generic code should
298 * be suitable for various DMA controllers, using transfer descriptors in RAM
299 * and pushing one SG list at a time to the DMA controller.
300 */
301
302 /* Hardware-specific part */
303
nbpf_chan_read(struct nbpf_channel * chan,unsigned int offset)304 static inline u32 nbpf_chan_read(struct nbpf_channel *chan,
305 unsigned int offset)
306 {
307 u32 data = ioread32(chan->base + offset);
308 dev_dbg(chan->dma_chan.device->dev, "%s(0x%p + 0x%x) = 0x%x\n",
309 __func__, chan->base, offset, data);
310 return data;
311 }
312
nbpf_chan_write(struct nbpf_channel * chan,unsigned int offset,u32 data)313 static inline void nbpf_chan_write(struct nbpf_channel *chan,
314 unsigned int offset, u32 data)
315 {
316 iowrite32(data, chan->base + offset);
317 dev_dbg(chan->dma_chan.device->dev, "%s(0x%p + 0x%x) = 0x%x\n",
318 __func__, chan->base, offset, data);
319 }
320
nbpf_read(struct nbpf_device * nbpf,unsigned int offset)321 static inline u32 nbpf_read(struct nbpf_device *nbpf,
322 unsigned int offset)
323 {
324 u32 data = ioread32(nbpf->base + offset);
325 dev_dbg(nbpf->dma_dev.dev, "%s(0x%p + 0x%x) = 0x%x\n",
326 __func__, nbpf->base, offset, data);
327 return data;
328 }
329
nbpf_write(struct nbpf_device * nbpf,unsigned int offset,u32 data)330 static inline void nbpf_write(struct nbpf_device *nbpf,
331 unsigned int offset, u32 data)
332 {
333 iowrite32(data, nbpf->base + offset);
334 dev_dbg(nbpf->dma_dev.dev, "%s(0x%p + 0x%x) = 0x%x\n",
335 __func__, nbpf->base, offset, data);
336 }
337
nbpf_chan_halt(struct nbpf_channel * chan)338 static void nbpf_chan_halt(struct nbpf_channel *chan)
339 {
340 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREN);
341 }
342
nbpf_status_get(struct nbpf_channel * chan)343 static bool nbpf_status_get(struct nbpf_channel *chan)
344 {
345 u32 status = nbpf_read(chan->nbpf, NBPF_DSTAT_END);
346
347 return status & BIT(chan - chan->nbpf->chan);
348 }
349
nbpf_status_ack(struct nbpf_channel * chan)350 static void nbpf_status_ack(struct nbpf_channel *chan)
351 {
352 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREND);
353 }
354
nbpf_error_get(struct nbpf_device * nbpf)355 static u32 nbpf_error_get(struct nbpf_device *nbpf)
356 {
357 return nbpf_read(nbpf, NBPF_DSTAT_ER);
358 }
359
nbpf_error_get_channel(struct nbpf_device * nbpf,u32 error)360 static struct nbpf_channel *nbpf_error_get_channel(struct nbpf_device *nbpf, u32 error)
361 {
362 return nbpf->chan + __ffs(error);
363 }
364
nbpf_error_clear(struct nbpf_channel * chan)365 static void nbpf_error_clear(struct nbpf_channel *chan)
366 {
367 u32 status;
368 int i;
369
370 /* Stop the channel, make sure DMA has been aborted */
371 nbpf_chan_halt(chan);
372
373 for (i = 1000; i; i--) {
374 status = nbpf_chan_read(chan, NBPF_CHAN_STAT);
375 if (!(status & NBPF_CHAN_STAT_TACT))
376 break;
377 cpu_relax();
378 }
379
380 if (!i)
381 dev_err(chan->dma_chan.device->dev,
382 "%s(): abort timeout, channel status 0x%x\n", __func__, status);
383
384 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SWRST);
385 }
386
nbpf_start(struct nbpf_desc * desc)387 static int nbpf_start(struct nbpf_desc *desc)
388 {
389 struct nbpf_channel *chan = desc->chan;
390 struct nbpf_link_desc *ldesc = list_first_entry(&desc->sg, struct nbpf_link_desc, node);
391
392 nbpf_chan_write(chan, NBPF_CHAN_NXLA, (u32)ldesc->hwdesc_dma_addr);
393 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SETEN | NBPF_CHAN_CTRL_CLRSUS);
394 chan->paused = false;
395
396 /* Software trigger MEMCPY - only MEMCPY uses the block mode */
397 if (ldesc->hwdesc->config & NBPF_CHAN_CFG_TM)
398 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_STG);
399
400 dev_dbg(chan->nbpf->dma_dev.dev, "%s(): next 0x%x, cur 0x%x\n", __func__,
401 nbpf_chan_read(chan, NBPF_CHAN_NXLA), nbpf_chan_read(chan, NBPF_CHAN_CRLA));
402
403 return 0;
404 }
405
nbpf_chan_prepare(struct nbpf_channel * chan)406 static void nbpf_chan_prepare(struct nbpf_channel *chan)
407 {
408 chan->dmarq_cfg = (chan->flags & NBPF_SLAVE_RQ_HIGH ? NBPF_CHAN_CFG_HIEN : 0) |
409 (chan->flags & NBPF_SLAVE_RQ_LOW ? NBPF_CHAN_CFG_LOEN : 0) |
410 (chan->flags & NBPF_SLAVE_RQ_LEVEL ?
411 NBPF_CHAN_CFG_LVL | (NBPF_CHAN_CFG_AM & 0x200) : 0) |
412 chan->terminal;
413 }
414
nbpf_chan_prepare_default(struct nbpf_channel * chan)415 static void nbpf_chan_prepare_default(struct nbpf_channel *chan)
416 {
417 /* Don't output DMAACK */
418 chan->dmarq_cfg = NBPF_CHAN_CFG_AM & 0x400;
419 chan->terminal = 0;
420 chan->flags = 0;
421 }
422
nbpf_chan_configure(struct nbpf_channel * chan)423 static void nbpf_chan_configure(struct nbpf_channel *chan)
424 {
425 /*
426 * We assume, that only the link mode and DMA request line configuration
427 * have to be set in the configuration register manually. Dynamic
428 * per-transfer configuration will be loaded from transfer descriptors.
429 */
430 nbpf_chan_write(chan, NBPF_CHAN_CFG, NBPF_CHAN_CFG_DMS | chan->dmarq_cfg);
431 }
432
nbpf_xfer_ds(struct nbpf_device * nbpf,size_t size,enum dma_transfer_direction direction)433 static u32 nbpf_xfer_ds(struct nbpf_device *nbpf, size_t size,
434 enum dma_transfer_direction direction)
435 {
436 int max_burst = nbpf->config->buffer_size * 8;
437
438 if (nbpf->max_burst_mem_read || nbpf->max_burst_mem_write) {
439 switch (direction) {
440 case DMA_MEM_TO_MEM:
441 max_burst = min_not_zero(nbpf->max_burst_mem_read,
442 nbpf->max_burst_mem_write);
443 break;
444 case DMA_MEM_TO_DEV:
445 if (nbpf->max_burst_mem_read)
446 max_burst = nbpf->max_burst_mem_read;
447 break;
448 case DMA_DEV_TO_MEM:
449 if (nbpf->max_burst_mem_write)
450 max_burst = nbpf->max_burst_mem_write;
451 break;
452 case DMA_DEV_TO_DEV:
453 default:
454 break;
455 }
456 }
457
458 /* Maximum supported bursts depend on the buffer size */
459 return min_t(int, __ffs(size), ilog2(max_burst));
460 }
461
nbpf_xfer_size(struct nbpf_device * nbpf,enum dma_slave_buswidth width,u32 burst)462 static size_t nbpf_xfer_size(struct nbpf_device *nbpf,
463 enum dma_slave_buswidth width, u32 burst)
464 {
465 size_t size;
466
467 if (!burst)
468 burst = 1;
469
470 switch (width) {
471 case DMA_SLAVE_BUSWIDTH_8_BYTES:
472 size = 8 * burst;
473 break;
474
475 case DMA_SLAVE_BUSWIDTH_4_BYTES:
476 size = 4 * burst;
477 break;
478
479 case DMA_SLAVE_BUSWIDTH_2_BYTES:
480 size = 2 * burst;
481 break;
482
483 default:
484 pr_warn("%s(): invalid bus width %u\n", __func__, width);
485 fallthrough;
486 case DMA_SLAVE_BUSWIDTH_1_BYTE:
487 size = burst;
488 }
489
490 return nbpf_xfer_ds(nbpf, size, DMA_TRANS_NONE);
491 }
492
493 /*
494 * We need a way to recognise slaves, whose data is sent "raw" over the bus,
495 * i.e. it isn't known in advance how many bytes will be received. Therefore
496 * the slave driver has to provide a "large enough" buffer and either read the
497 * buffer, when it is full, or detect, that some data has arrived, then wait for
498 * a timeout, if no more data arrives - receive what's already there. We want to
499 * handle such slaves in a special way to allow an optimised mode for other
500 * users, for whom the amount of data is known in advance. So far there's no way
501 * to recognise such slaves. We use a data-width check to distinguish between
502 * the SD host and the PL011 UART.
503 */
504
nbpf_prep_one(struct nbpf_link_desc * ldesc,enum dma_transfer_direction direction,dma_addr_t src,dma_addr_t dst,size_t size,bool last)505 static int nbpf_prep_one(struct nbpf_link_desc *ldesc,
506 enum dma_transfer_direction direction,
507 dma_addr_t src, dma_addr_t dst, size_t size, bool last)
508 {
509 struct nbpf_link_reg *hwdesc = ldesc->hwdesc;
510 struct nbpf_desc *desc = ldesc->desc;
511 struct nbpf_channel *chan = desc->chan;
512 struct device *dev = chan->dma_chan.device->dev;
513 size_t mem_xfer, slave_xfer;
514 bool can_burst;
515
516 hwdesc->header = NBPF_HEADER_WBD | NBPF_HEADER_LV |
517 (last ? NBPF_HEADER_LE : 0);
518
519 hwdesc->src_addr = src;
520 hwdesc->dst_addr = dst;
521 hwdesc->transaction_size = size;
522
523 /*
524 * set config: SAD, DAD, DDS, SDS, etc.
525 * Note on transfer sizes: the DMAC can perform unaligned DMA transfers,
526 * but it is important to have transaction size a multiple of both
527 * receiver and transmitter transfer sizes. It is also possible to use
528 * different RAM and device transfer sizes, and it does work well with
529 * some devices, e.g. with V08R07S01E SD host controllers, which can use
530 * 128 byte transfers. But this doesn't work with other devices,
531 * especially when the transaction size is unknown. This is the case,
532 * e.g. with serial drivers like amba-pl011.c. For reception it sets up
533 * the transaction size of 4K and if fewer bytes are received, it
534 * pauses DMA and reads out data received via DMA as well as those left
535 * in the Rx FIFO. For this to work with the RAM side using burst
536 * transfers we enable the SBE bit and terminate the transfer in our
537 * .device_pause handler.
538 */
539 mem_xfer = nbpf_xfer_ds(chan->nbpf, size, direction);
540
541 switch (direction) {
542 case DMA_DEV_TO_MEM:
543 can_burst = chan->slave_src_width >= 3;
544 slave_xfer = min(mem_xfer, can_burst ?
545 chan->slave_src_burst : chan->slave_src_width);
546 /*
547 * Is the slave narrower than 64 bits, i.e. isn't using the full
548 * bus width and cannot use bursts?
549 */
550 if (mem_xfer > chan->slave_src_burst && !can_burst)
551 mem_xfer = chan->slave_src_burst;
552 /* Device-to-RAM DMA is unreliable without REQD set */
553 hwdesc->config = NBPF_CHAN_CFG_SAD | (NBPF_CHAN_CFG_DDS & (mem_xfer << 16)) |
554 (NBPF_CHAN_CFG_SDS & (slave_xfer << 12)) | NBPF_CHAN_CFG_REQD |
555 NBPF_CHAN_CFG_SBE;
556 break;
557
558 case DMA_MEM_TO_DEV:
559 slave_xfer = min(mem_xfer, chan->slave_dst_width >= 3 ?
560 chan->slave_dst_burst : chan->slave_dst_width);
561 hwdesc->config = NBPF_CHAN_CFG_DAD | (NBPF_CHAN_CFG_SDS & (mem_xfer << 12)) |
562 (NBPF_CHAN_CFG_DDS & (slave_xfer << 16)) | NBPF_CHAN_CFG_REQD;
563 break;
564
565 case DMA_MEM_TO_MEM:
566 hwdesc->config = NBPF_CHAN_CFG_TCM | NBPF_CHAN_CFG_TM |
567 (NBPF_CHAN_CFG_SDS & (mem_xfer << 12)) |
568 (NBPF_CHAN_CFG_DDS & (mem_xfer << 16));
569 break;
570
571 default:
572 return -EINVAL;
573 }
574
575 hwdesc->config |= chan->dmarq_cfg | (last ? 0 : NBPF_CHAN_CFG_DEM) |
576 NBPF_CHAN_CFG_DMS;
577
578 dev_dbg(dev, "%s(): desc @ %pad: hdr 0x%x, cfg 0x%x, %zu @ %pad -> %pad\n",
579 __func__, &ldesc->hwdesc_dma_addr, hwdesc->header,
580 hwdesc->config, size, &src, &dst);
581
582 dma_sync_single_for_device(dev, ldesc->hwdesc_dma_addr, sizeof(*hwdesc),
583 DMA_TO_DEVICE);
584
585 return 0;
586 }
587
nbpf_bytes_left(struct nbpf_channel * chan)588 static size_t nbpf_bytes_left(struct nbpf_channel *chan)
589 {
590 return nbpf_chan_read(chan, NBPF_CHAN_CUR_TR_BYTE);
591 }
592
nbpf_configure(struct nbpf_device * nbpf)593 static void nbpf_configure(struct nbpf_device *nbpf)
594 {
595 nbpf_write(nbpf, NBPF_CTRL, NBPF_CTRL_LVINT);
596 }
597
598 /* Generic part */
599
600 /* DMA ENGINE functions */
nbpf_issue_pending(struct dma_chan * dchan)601 static void nbpf_issue_pending(struct dma_chan *dchan)
602 {
603 struct nbpf_channel *chan = nbpf_to_chan(dchan);
604 unsigned long flags;
605
606 dev_dbg(dchan->device->dev, "Entry %s()\n", __func__);
607
608 spin_lock_irqsave(&chan->lock, flags);
609 if (list_empty(&chan->queued))
610 goto unlock;
611
612 list_splice_tail_init(&chan->queued, &chan->active);
613
614 if (!chan->running) {
615 struct nbpf_desc *desc = list_first_entry(&chan->active,
616 struct nbpf_desc, node);
617 if (!nbpf_start(desc))
618 chan->running = desc;
619 }
620
621 unlock:
622 spin_unlock_irqrestore(&chan->lock, flags);
623 }
624
nbpf_tx_status(struct dma_chan * dchan,dma_cookie_t cookie,struct dma_tx_state * state)625 static enum dma_status nbpf_tx_status(struct dma_chan *dchan,
626 dma_cookie_t cookie, struct dma_tx_state *state)
627 {
628 struct nbpf_channel *chan = nbpf_to_chan(dchan);
629 enum dma_status status = dma_cookie_status(dchan, cookie, state);
630
631 if (state) {
632 dma_cookie_t running;
633 unsigned long flags;
634
635 spin_lock_irqsave(&chan->lock, flags);
636 running = chan->running ? chan->running->async_tx.cookie : -EINVAL;
637
638 if (cookie == running) {
639 state->residue = nbpf_bytes_left(chan);
640 dev_dbg(dchan->device->dev, "%s(): residue %u\n", __func__,
641 state->residue);
642 } else if (status == DMA_IN_PROGRESS) {
643 struct nbpf_desc *desc;
644 bool found = false;
645
646 list_for_each_entry(desc, &chan->active, node)
647 if (desc->async_tx.cookie == cookie) {
648 found = true;
649 break;
650 }
651
652 if (!found)
653 list_for_each_entry(desc, &chan->queued, node)
654 if (desc->async_tx.cookie == cookie) {
655 found = true;
656 break;
657
658 }
659
660 state->residue = found ? desc->length : 0;
661 }
662
663 spin_unlock_irqrestore(&chan->lock, flags);
664 }
665
666 if (chan->paused)
667 status = DMA_PAUSED;
668
669 return status;
670 }
671
nbpf_tx_submit(struct dma_async_tx_descriptor * tx)672 static dma_cookie_t nbpf_tx_submit(struct dma_async_tx_descriptor *tx)
673 {
674 struct nbpf_desc *desc = container_of(tx, struct nbpf_desc, async_tx);
675 struct nbpf_channel *chan = desc->chan;
676 unsigned long flags;
677 dma_cookie_t cookie;
678
679 spin_lock_irqsave(&chan->lock, flags);
680 cookie = dma_cookie_assign(tx);
681 list_add_tail(&desc->node, &chan->queued);
682 spin_unlock_irqrestore(&chan->lock, flags);
683
684 dev_dbg(chan->dma_chan.device->dev, "Entry %s(%d)\n", __func__, cookie);
685
686 return cookie;
687 }
688
nbpf_desc_page_alloc(struct nbpf_channel * chan)689 static int nbpf_desc_page_alloc(struct nbpf_channel *chan)
690 {
691 struct dma_chan *dchan = &chan->dma_chan;
692 struct nbpf_desc_page *dpage = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
693 struct nbpf_link_desc *ldesc;
694 struct nbpf_link_reg *hwdesc;
695 struct nbpf_desc *desc;
696 LIST_HEAD(head);
697 LIST_HEAD(lhead);
698 int i;
699 struct device *dev = dchan->device->dev;
700
701 if (!dpage)
702 return -ENOMEM;
703
704 dev_dbg(dev, "%s(): alloc %lu descriptors, %lu segments, total alloc %zu\n",
705 __func__, NBPF_DESCS_PER_PAGE, NBPF_SEGMENTS_PER_PAGE, sizeof(*dpage));
706
707 for (i = 0, ldesc = dpage->ldesc, hwdesc = dpage->hwdesc;
708 i < ARRAY_SIZE(dpage->ldesc);
709 i++, ldesc++, hwdesc++) {
710 ldesc->hwdesc = hwdesc;
711 list_add_tail(&ldesc->node, &lhead);
712 ldesc->hwdesc_dma_addr = dma_map_single(dchan->device->dev,
713 hwdesc, sizeof(*hwdesc), DMA_TO_DEVICE);
714 if (dma_mapping_error(dchan->device->dev,
715 ldesc->hwdesc_dma_addr))
716 goto unmap_error;
717
718 dev_dbg(dev, "%s(): mapped 0x%p to %pad\n", __func__,
719 hwdesc, &ldesc->hwdesc_dma_addr);
720 }
721
722 for (i = 0, desc = dpage->desc;
723 i < ARRAY_SIZE(dpage->desc);
724 i++, desc++) {
725 dma_async_tx_descriptor_init(&desc->async_tx, dchan);
726 desc->async_tx.tx_submit = nbpf_tx_submit;
727 desc->chan = chan;
728 INIT_LIST_HEAD(&desc->sg);
729 list_add_tail(&desc->node, &head);
730 }
731
732 /*
733 * This function cannot be called from interrupt context, so, no need to
734 * save flags
735 */
736 spin_lock_irq(&chan->lock);
737 list_splice_tail(&lhead, &chan->free_links);
738 list_splice_tail(&head, &chan->free);
739 list_add(&dpage->node, &chan->desc_page);
740 spin_unlock_irq(&chan->lock);
741
742 return ARRAY_SIZE(dpage->desc);
743
744 unmap_error:
745 while (i--) {
746 ldesc--; hwdesc--;
747
748 dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr,
749 sizeof(hwdesc), DMA_TO_DEVICE);
750 }
751
752 return -ENOMEM;
753 }
754
nbpf_desc_put(struct nbpf_desc * desc)755 static void nbpf_desc_put(struct nbpf_desc *desc)
756 {
757 struct nbpf_channel *chan = desc->chan;
758 struct nbpf_link_desc *ldesc, *tmp;
759 unsigned long flags;
760
761 spin_lock_irqsave(&chan->lock, flags);
762 list_for_each_entry_safe(ldesc, tmp, &desc->sg, node)
763 list_move(&ldesc->node, &chan->free_links);
764
765 list_add(&desc->node, &chan->free);
766 spin_unlock_irqrestore(&chan->lock, flags);
767 }
768
nbpf_scan_acked(struct nbpf_channel * chan)769 static void nbpf_scan_acked(struct nbpf_channel *chan)
770 {
771 struct nbpf_desc *desc, *tmp;
772 unsigned long flags;
773 LIST_HEAD(head);
774
775 spin_lock_irqsave(&chan->lock, flags);
776 list_for_each_entry_safe(desc, tmp, &chan->done, node)
777 if (async_tx_test_ack(&desc->async_tx) && desc->user_wait) {
778 list_move(&desc->node, &head);
779 desc->user_wait = false;
780 }
781 spin_unlock_irqrestore(&chan->lock, flags);
782
783 list_for_each_entry_safe(desc, tmp, &head, node) {
784 list_del(&desc->node);
785 nbpf_desc_put(desc);
786 }
787 }
788
789 /*
790 * We have to allocate descriptors with the channel lock dropped. This means,
791 * before we re-acquire the lock buffers can be taken already, so we have to
792 * re-check after re-acquiring the lock and possibly retry, if buffers are gone
793 * again.
794 */
nbpf_desc_get(struct nbpf_channel * chan,size_t len)795 static struct nbpf_desc *nbpf_desc_get(struct nbpf_channel *chan, size_t len)
796 {
797 struct nbpf_desc *desc = NULL;
798 struct nbpf_link_desc *ldesc, *prev = NULL;
799
800 nbpf_scan_acked(chan);
801
802 spin_lock_irq(&chan->lock);
803
804 do {
805 int i = 0, ret;
806
807 if (list_empty(&chan->free)) {
808 /* No more free descriptors */
809 spin_unlock_irq(&chan->lock);
810 ret = nbpf_desc_page_alloc(chan);
811 if (ret < 0)
812 return NULL;
813 spin_lock_irq(&chan->lock);
814 continue;
815 }
816 desc = list_first_entry(&chan->free, struct nbpf_desc, node);
817 list_del(&desc->node);
818
819 do {
820 if (list_empty(&chan->free_links)) {
821 /* No more free link descriptors */
822 spin_unlock_irq(&chan->lock);
823 ret = nbpf_desc_page_alloc(chan);
824 if (ret < 0) {
825 nbpf_desc_put(desc);
826 return NULL;
827 }
828 spin_lock_irq(&chan->lock);
829 continue;
830 }
831
832 ldesc = list_first_entry(&chan->free_links,
833 struct nbpf_link_desc, node);
834 ldesc->desc = desc;
835 if (prev)
836 prev->hwdesc->next = (u32)ldesc->hwdesc_dma_addr;
837
838 prev = ldesc;
839 list_move_tail(&ldesc->node, &desc->sg);
840
841 i++;
842 } while (i < len);
843 } while (!desc);
844
845 prev->hwdesc->next = 0;
846
847 spin_unlock_irq(&chan->lock);
848
849 return desc;
850 }
851
nbpf_chan_idle(struct nbpf_channel * chan)852 static void nbpf_chan_idle(struct nbpf_channel *chan)
853 {
854 struct nbpf_desc *desc, *tmp;
855 unsigned long flags;
856 LIST_HEAD(head);
857
858 spin_lock_irqsave(&chan->lock, flags);
859
860 list_splice_init(&chan->done, &head);
861 list_splice_init(&chan->active, &head);
862 list_splice_init(&chan->queued, &head);
863
864 chan->running = NULL;
865
866 spin_unlock_irqrestore(&chan->lock, flags);
867
868 list_for_each_entry_safe(desc, tmp, &head, node) {
869 dev_dbg(chan->nbpf->dma_dev.dev, "%s(): force-free desc %p cookie %d\n",
870 __func__, desc, desc->async_tx.cookie);
871 list_del(&desc->node);
872 nbpf_desc_put(desc);
873 }
874 }
875
nbpf_pause(struct dma_chan * dchan)876 static int nbpf_pause(struct dma_chan *dchan)
877 {
878 struct nbpf_channel *chan = nbpf_to_chan(dchan);
879
880 dev_dbg(dchan->device->dev, "Entry %s\n", __func__);
881
882 chan->paused = true;
883 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_SETSUS);
884 /* See comment in nbpf_prep_one() */
885 nbpf_chan_write(chan, NBPF_CHAN_CTRL, NBPF_CHAN_CTRL_CLREN);
886
887 return 0;
888 }
889
nbpf_terminate_all(struct dma_chan * dchan)890 static int nbpf_terminate_all(struct dma_chan *dchan)
891 {
892 struct nbpf_channel *chan = nbpf_to_chan(dchan);
893
894 dev_dbg(dchan->device->dev, "Entry %s\n", __func__);
895 dev_dbg(dchan->device->dev, "Terminating\n");
896
897 nbpf_chan_halt(chan);
898 nbpf_chan_idle(chan);
899
900 return 0;
901 }
902
nbpf_config(struct dma_chan * dchan,struct dma_slave_config * config)903 static int nbpf_config(struct dma_chan *dchan,
904 struct dma_slave_config *config)
905 {
906 struct nbpf_channel *chan = nbpf_to_chan(dchan);
907
908 dev_dbg(dchan->device->dev, "Entry %s\n", __func__);
909
910 /*
911 * We could check config->slave_id to match chan->terminal here,
912 * but with DT they would be coming from the same source, so
913 * such a check would be superfluous
914 */
915
916 chan->slave_dst_addr = config->dst_addr;
917 chan->slave_dst_width = nbpf_xfer_size(chan->nbpf,
918 config->dst_addr_width, 1);
919 chan->slave_dst_burst = nbpf_xfer_size(chan->nbpf,
920 config->dst_addr_width,
921 config->dst_maxburst);
922 chan->slave_src_addr = config->src_addr;
923 chan->slave_src_width = nbpf_xfer_size(chan->nbpf,
924 config->src_addr_width, 1);
925 chan->slave_src_burst = nbpf_xfer_size(chan->nbpf,
926 config->src_addr_width,
927 config->src_maxburst);
928
929 return 0;
930 }
931
nbpf_prep_sg(struct nbpf_channel * chan,struct scatterlist * src_sg,struct scatterlist * dst_sg,size_t len,enum dma_transfer_direction direction,unsigned long flags)932 static struct dma_async_tx_descriptor *nbpf_prep_sg(struct nbpf_channel *chan,
933 struct scatterlist *src_sg, struct scatterlist *dst_sg,
934 size_t len, enum dma_transfer_direction direction,
935 unsigned long flags)
936 {
937 struct nbpf_link_desc *ldesc;
938 struct scatterlist *mem_sg;
939 struct nbpf_desc *desc;
940 bool inc_src, inc_dst;
941 size_t data_len = 0;
942 int i = 0;
943
944 switch (direction) {
945 case DMA_DEV_TO_MEM:
946 mem_sg = dst_sg;
947 inc_src = false;
948 inc_dst = true;
949 break;
950
951 case DMA_MEM_TO_DEV:
952 mem_sg = src_sg;
953 inc_src = true;
954 inc_dst = false;
955 break;
956
957 default:
958 case DMA_MEM_TO_MEM:
959 mem_sg = src_sg;
960 inc_src = true;
961 inc_dst = true;
962 }
963
964 desc = nbpf_desc_get(chan, len);
965 if (!desc)
966 return NULL;
967
968 desc->async_tx.flags = flags;
969 desc->async_tx.cookie = -EBUSY;
970 desc->user_wait = false;
971
972 /*
973 * This is a private descriptor list, and we own the descriptor. No need
974 * to lock.
975 */
976 list_for_each_entry(ldesc, &desc->sg, node) {
977 int ret = nbpf_prep_one(ldesc, direction,
978 sg_dma_address(src_sg),
979 sg_dma_address(dst_sg),
980 sg_dma_len(mem_sg),
981 i == len - 1);
982 if (ret < 0) {
983 nbpf_desc_put(desc);
984 return NULL;
985 }
986 data_len += sg_dma_len(mem_sg);
987 if (inc_src)
988 src_sg = sg_next(src_sg);
989 if (inc_dst)
990 dst_sg = sg_next(dst_sg);
991 mem_sg = direction == DMA_DEV_TO_MEM ? dst_sg : src_sg;
992 i++;
993 }
994
995 desc->length = data_len;
996
997 /* The user has to return the descriptor to us ASAP via .tx_submit() */
998 return &desc->async_tx;
999 }
1000
nbpf_prep_memcpy(struct dma_chan * dchan,dma_addr_t dst,dma_addr_t src,size_t len,unsigned long flags)1001 static struct dma_async_tx_descriptor *nbpf_prep_memcpy(
1002 struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src,
1003 size_t len, unsigned long flags)
1004 {
1005 struct nbpf_channel *chan = nbpf_to_chan(dchan);
1006 struct scatterlist dst_sg;
1007 struct scatterlist src_sg;
1008
1009 sg_init_table(&dst_sg, 1);
1010 sg_init_table(&src_sg, 1);
1011
1012 sg_dma_address(&dst_sg) = dst;
1013 sg_dma_address(&src_sg) = src;
1014
1015 sg_dma_len(&dst_sg) = len;
1016 sg_dma_len(&src_sg) = len;
1017
1018 dev_dbg(dchan->device->dev, "%s(): %zu @ %pad -> %pad\n",
1019 __func__, len, &src, &dst);
1020
1021 return nbpf_prep_sg(chan, &src_sg, &dst_sg, 1,
1022 DMA_MEM_TO_MEM, flags);
1023 }
1024
nbpf_prep_slave_sg(struct dma_chan * dchan,struct scatterlist * sgl,unsigned int sg_len,enum dma_transfer_direction direction,unsigned long flags,void * context)1025 static struct dma_async_tx_descriptor *nbpf_prep_slave_sg(
1026 struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len,
1027 enum dma_transfer_direction direction, unsigned long flags, void *context)
1028 {
1029 struct nbpf_channel *chan = nbpf_to_chan(dchan);
1030 struct scatterlist slave_sg;
1031
1032 dev_dbg(dchan->device->dev, "Entry %s()\n", __func__);
1033
1034 sg_init_table(&slave_sg, 1);
1035
1036 switch (direction) {
1037 case DMA_MEM_TO_DEV:
1038 sg_dma_address(&slave_sg) = chan->slave_dst_addr;
1039 return nbpf_prep_sg(chan, sgl, &slave_sg, sg_len,
1040 direction, flags);
1041
1042 case DMA_DEV_TO_MEM:
1043 sg_dma_address(&slave_sg) = chan->slave_src_addr;
1044 return nbpf_prep_sg(chan, &slave_sg, sgl, sg_len,
1045 direction, flags);
1046
1047 default:
1048 return NULL;
1049 }
1050 }
1051
nbpf_alloc_chan_resources(struct dma_chan * dchan)1052 static int nbpf_alloc_chan_resources(struct dma_chan *dchan)
1053 {
1054 struct nbpf_channel *chan = nbpf_to_chan(dchan);
1055 int ret;
1056
1057 INIT_LIST_HEAD(&chan->free);
1058 INIT_LIST_HEAD(&chan->free_links);
1059 INIT_LIST_HEAD(&chan->queued);
1060 INIT_LIST_HEAD(&chan->active);
1061 INIT_LIST_HEAD(&chan->done);
1062
1063 ret = nbpf_desc_page_alloc(chan);
1064 if (ret < 0)
1065 return ret;
1066
1067 dev_dbg(dchan->device->dev, "Entry %s(): terminal %u\n", __func__,
1068 chan->terminal);
1069
1070 nbpf_chan_configure(chan);
1071
1072 return ret;
1073 }
1074
nbpf_free_chan_resources(struct dma_chan * dchan)1075 static void nbpf_free_chan_resources(struct dma_chan *dchan)
1076 {
1077 struct nbpf_channel *chan = nbpf_to_chan(dchan);
1078 struct nbpf_desc_page *dpage, *tmp;
1079
1080 dev_dbg(dchan->device->dev, "Entry %s()\n", __func__);
1081
1082 nbpf_chan_halt(chan);
1083 nbpf_chan_idle(chan);
1084 /* Clean up for if a channel is re-used for MEMCPY after slave DMA */
1085 nbpf_chan_prepare_default(chan);
1086
1087 list_for_each_entry_safe(dpage, tmp, &chan->desc_page, node) {
1088 struct nbpf_link_desc *ldesc;
1089 int i;
1090 list_del(&dpage->node);
1091 for (i = 0, ldesc = dpage->ldesc;
1092 i < ARRAY_SIZE(dpage->ldesc);
1093 i++, ldesc++)
1094 dma_unmap_single(dchan->device->dev, ldesc->hwdesc_dma_addr,
1095 sizeof(*ldesc->hwdesc), DMA_TO_DEVICE);
1096 free_page((unsigned long)dpage);
1097 }
1098 }
1099
nbpf_of_xlate(struct of_phandle_args * dma_spec,struct of_dma * ofdma)1100 static struct dma_chan *nbpf_of_xlate(struct of_phandle_args *dma_spec,
1101 struct of_dma *ofdma)
1102 {
1103 struct nbpf_device *nbpf = ofdma->of_dma_data;
1104 struct dma_chan *dchan;
1105 struct nbpf_channel *chan;
1106
1107 if (dma_spec->args_count != 2)
1108 return NULL;
1109
1110 dchan = dma_get_any_slave_channel(&nbpf->dma_dev);
1111 if (!dchan)
1112 return NULL;
1113
1114 dev_dbg(dchan->device->dev, "Entry %s(%pOFn)\n", __func__,
1115 dma_spec->np);
1116
1117 chan = nbpf_to_chan(dchan);
1118
1119 chan->terminal = dma_spec->args[0];
1120 chan->flags = dma_spec->args[1];
1121
1122 nbpf_chan_prepare(chan);
1123 nbpf_chan_configure(chan);
1124
1125 return dchan;
1126 }
1127
nbpf_chan_tasklet(struct tasklet_struct * t)1128 static void nbpf_chan_tasklet(struct tasklet_struct *t)
1129 {
1130 struct nbpf_channel *chan = from_tasklet(chan, t, tasklet);
1131 struct nbpf_desc *desc, *tmp;
1132 struct dmaengine_desc_callback cb;
1133
1134 while (!list_empty(&chan->done)) {
1135 bool found = false, must_put, recycling = false;
1136
1137 spin_lock_irq(&chan->lock);
1138
1139 list_for_each_entry_safe(desc, tmp, &chan->done, node) {
1140 if (!desc->user_wait) {
1141 /* Newly completed descriptor, have to process */
1142 found = true;
1143 break;
1144 } else if (async_tx_test_ack(&desc->async_tx)) {
1145 /*
1146 * This descriptor was waiting for a user ACK,
1147 * it can be recycled now.
1148 */
1149 list_del(&desc->node);
1150 spin_unlock_irq(&chan->lock);
1151 nbpf_desc_put(desc);
1152 recycling = true;
1153 break;
1154 }
1155 }
1156
1157 if (recycling)
1158 continue;
1159
1160 if (!found) {
1161 /* This can happen if TERMINATE_ALL has been called */
1162 spin_unlock_irq(&chan->lock);
1163 break;
1164 }
1165
1166 dma_cookie_complete(&desc->async_tx);
1167
1168 /*
1169 * With released lock we cannot dereference desc, maybe it's
1170 * still on the "done" list
1171 */
1172 if (async_tx_test_ack(&desc->async_tx)) {
1173 list_del(&desc->node);
1174 must_put = true;
1175 } else {
1176 desc->user_wait = true;
1177 must_put = false;
1178 }
1179
1180 dmaengine_desc_get_callback(&desc->async_tx, &cb);
1181
1182 /* ack and callback completed descriptor */
1183 spin_unlock_irq(&chan->lock);
1184
1185 dmaengine_desc_callback_invoke(&cb, NULL);
1186
1187 if (must_put)
1188 nbpf_desc_put(desc);
1189 }
1190 }
1191
nbpf_chan_irq(int irq,void * dev)1192 static irqreturn_t nbpf_chan_irq(int irq, void *dev)
1193 {
1194 struct nbpf_channel *chan = dev;
1195 bool done = nbpf_status_get(chan);
1196 struct nbpf_desc *desc;
1197 irqreturn_t ret;
1198 bool bh = false;
1199
1200 if (!done)
1201 return IRQ_NONE;
1202
1203 nbpf_status_ack(chan);
1204
1205 dev_dbg(&chan->dma_chan.dev->device, "%s()\n", __func__);
1206
1207 spin_lock(&chan->lock);
1208 desc = chan->running;
1209 if (WARN_ON(!desc)) {
1210 ret = IRQ_NONE;
1211 goto unlock;
1212 } else {
1213 ret = IRQ_HANDLED;
1214 bh = true;
1215 }
1216
1217 list_move_tail(&desc->node, &chan->done);
1218 chan->running = NULL;
1219
1220 if (!list_empty(&chan->active)) {
1221 desc = list_first_entry(&chan->active,
1222 struct nbpf_desc, node);
1223 if (!nbpf_start(desc))
1224 chan->running = desc;
1225 }
1226
1227 unlock:
1228 spin_unlock(&chan->lock);
1229
1230 if (bh)
1231 tasklet_schedule(&chan->tasklet);
1232
1233 return ret;
1234 }
1235
nbpf_err_irq(int irq,void * dev)1236 static irqreturn_t nbpf_err_irq(int irq, void *dev)
1237 {
1238 struct nbpf_device *nbpf = dev;
1239 u32 error = nbpf_error_get(nbpf);
1240
1241 dev_warn(nbpf->dma_dev.dev, "DMA error IRQ %u\n", irq);
1242
1243 if (!error)
1244 return IRQ_NONE;
1245
1246 do {
1247 struct nbpf_channel *chan = nbpf_error_get_channel(nbpf, error);
1248 /* On error: abort all queued transfers, no callback */
1249 nbpf_error_clear(chan);
1250 nbpf_chan_idle(chan);
1251 error = nbpf_error_get(nbpf);
1252 } while (error);
1253
1254 return IRQ_HANDLED;
1255 }
1256
nbpf_chan_probe(struct nbpf_device * nbpf,int n)1257 static int nbpf_chan_probe(struct nbpf_device *nbpf, int n)
1258 {
1259 struct dma_device *dma_dev = &nbpf->dma_dev;
1260 struct nbpf_channel *chan = nbpf->chan + n;
1261 int ret;
1262
1263 chan->nbpf = nbpf;
1264 chan->base = nbpf->base + NBPF_REG_CHAN_OFFSET + NBPF_REG_CHAN_SIZE * n;
1265 INIT_LIST_HEAD(&chan->desc_page);
1266 spin_lock_init(&chan->lock);
1267 chan->dma_chan.device = dma_dev;
1268 dma_cookie_init(&chan->dma_chan);
1269 nbpf_chan_prepare_default(chan);
1270
1271 dev_dbg(dma_dev->dev, "%s(): channel %d: -> %p\n", __func__, n, chan->base);
1272
1273 snprintf(chan->name, sizeof(chan->name), "nbpf %d", n);
1274
1275 tasklet_setup(&chan->tasklet, nbpf_chan_tasklet);
1276 ret = devm_request_irq(dma_dev->dev, chan->irq,
1277 nbpf_chan_irq, IRQF_SHARED,
1278 chan->name, chan);
1279 if (ret < 0)
1280 return ret;
1281
1282 /* Add the channel to DMA device channel list */
1283 list_add_tail(&chan->dma_chan.device_node,
1284 &dma_dev->channels);
1285
1286 return 0;
1287 }
1288
1289 static const struct of_device_id nbpf_match[] = {
1290 {.compatible = "renesas,nbpfaxi64dmac1b4", .data = &nbpf_cfg[NBPF1B4]},
1291 {.compatible = "renesas,nbpfaxi64dmac1b8", .data = &nbpf_cfg[NBPF1B8]},
1292 {.compatible = "renesas,nbpfaxi64dmac1b16", .data = &nbpf_cfg[NBPF1B16]},
1293 {.compatible = "renesas,nbpfaxi64dmac4b4", .data = &nbpf_cfg[NBPF4B4]},
1294 {.compatible = "renesas,nbpfaxi64dmac4b8", .data = &nbpf_cfg[NBPF4B8]},
1295 {.compatible = "renesas,nbpfaxi64dmac4b16", .data = &nbpf_cfg[NBPF4B16]},
1296 {.compatible = "renesas,nbpfaxi64dmac8b4", .data = &nbpf_cfg[NBPF8B4]},
1297 {.compatible = "renesas,nbpfaxi64dmac8b8", .data = &nbpf_cfg[NBPF8B8]},
1298 {.compatible = "renesas,nbpfaxi64dmac8b16", .data = &nbpf_cfg[NBPF8B16]},
1299 {}
1300 };
1301 MODULE_DEVICE_TABLE(of, nbpf_match);
1302
nbpf_probe(struct platform_device * pdev)1303 static int nbpf_probe(struct platform_device *pdev)
1304 {
1305 struct device *dev = &pdev->dev;
1306 struct device_node *np = dev->of_node;
1307 struct nbpf_device *nbpf;
1308 struct dma_device *dma_dev;
1309 const struct nbpf_config *cfg;
1310 int num_channels;
1311 int ret, irq, eirq, i;
1312 int irqbuf[9] /* maximum 8 channels + error IRQ */;
1313 unsigned int irqs = 0;
1314
1315 BUILD_BUG_ON(sizeof(struct nbpf_desc_page) > PAGE_SIZE);
1316
1317 /* DT only */
1318 if (!np)
1319 return -ENODEV;
1320
1321 cfg = of_device_get_match_data(dev);
1322 num_channels = cfg->num_channels;
1323
1324 nbpf = devm_kzalloc(dev, struct_size(nbpf, chan, num_channels),
1325 GFP_KERNEL);
1326 if (!nbpf)
1327 return -ENOMEM;
1328
1329 dma_dev = &nbpf->dma_dev;
1330 dma_dev->dev = dev;
1331
1332 nbpf->base = devm_platform_ioremap_resource(pdev, 0);
1333 if (IS_ERR(nbpf->base))
1334 return PTR_ERR(nbpf->base);
1335
1336 nbpf->clk = devm_clk_get(dev, NULL);
1337 if (IS_ERR(nbpf->clk))
1338 return PTR_ERR(nbpf->clk);
1339
1340 of_property_read_u32(np, "max-burst-mem-read",
1341 &nbpf->max_burst_mem_read);
1342 of_property_read_u32(np, "max-burst-mem-write",
1343 &nbpf->max_burst_mem_write);
1344
1345 nbpf->config = cfg;
1346
1347 for (i = 0; irqs < ARRAY_SIZE(irqbuf); i++) {
1348 irq = platform_get_irq_optional(pdev, i);
1349 if (irq < 0 && irq != -ENXIO)
1350 return irq;
1351 if (irq > 0)
1352 irqbuf[irqs++] = irq;
1353 }
1354
1355 /*
1356 * 3 IRQ resource schemes are supported:
1357 * 1. 1 shared IRQ for error and all channels
1358 * 2. 2 IRQs: one for error and one shared for all channels
1359 * 3. 1 IRQ for error and an own IRQ for each channel
1360 */
1361 if (irqs != 1 && irqs != 2 && irqs != num_channels + 1)
1362 return -ENXIO;
1363
1364 if (irqs == 1) {
1365 eirq = irqbuf[0];
1366
1367 for (i = 0; i < num_channels; i++)
1368 nbpf->chan[i].irq = irqbuf[0];
1369 } else {
1370 eirq = platform_get_irq_byname(pdev, "error");
1371 if (eirq < 0)
1372 return eirq;
1373
1374 if (irqs == num_channels + 1) {
1375 struct nbpf_channel *chan;
1376
1377 for (i = 0, chan = nbpf->chan; i < num_channels;
1378 i++, chan++) {
1379 /* Skip the error IRQ */
1380 if (irqbuf[i] == eirq)
1381 i++;
1382 if (i >= ARRAY_SIZE(irqbuf))
1383 return -EINVAL;
1384 chan->irq = irqbuf[i];
1385 }
1386 } else {
1387 /* 2 IRQs and more than one channel */
1388 if (irqbuf[0] == eirq)
1389 irq = irqbuf[1];
1390 else
1391 irq = irqbuf[0];
1392
1393 for (i = 0; i < num_channels; i++)
1394 nbpf->chan[i].irq = irq;
1395 }
1396 }
1397
1398 ret = devm_request_irq(dev, eirq, nbpf_err_irq,
1399 IRQF_SHARED, "dma error", nbpf);
1400 if (ret < 0)
1401 return ret;
1402 nbpf->eirq = eirq;
1403
1404 INIT_LIST_HEAD(&dma_dev->channels);
1405
1406 /* Create DMA Channel */
1407 for (i = 0; i < num_channels; i++) {
1408 ret = nbpf_chan_probe(nbpf, i);
1409 if (ret < 0)
1410 return ret;
1411 }
1412
1413 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1414 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1415 dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1416
1417 /* Common and MEMCPY operations */
1418 dma_dev->device_alloc_chan_resources
1419 = nbpf_alloc_chan_resources;
1420 dma_dev->device_free_chan_resources = nbpf_free_chan_resources;
1421 dma_dev->device_prep_dma_memcpy = nbpf_prep_memcpy;
1422 dma_dev->device_tx_status = nbpf_tx_status;
1423 dma_dev->device_issue_pending = nbpf_issue_pending;
1424
1425 /*
1426 * If we drop support for unaligned MEMCPY buffer addresses and / or
1427 * lengths by setting
1428 * dma_dev->copy_align = 4;
1429 * then we can set transfer length to 4 bytes in nbpf_prep_one() for
1430 * DMA_MEM_TO_MEM
1431 */
1432
1433 /* Compulsory for DMA_SLAVE fields */
1434 dma_dev->device_prep_slave_sg = nbpf_prep_slave_sg;
1435 dma_dev->device_config = nbpf_config;
1436 dma_dev->device_pause = nbpf_pause;
1437 dma_dev->device_terminate_all = nbpf_terminate_all;
1438
1439 dma_dev->src_addr_widths = NBPF_DMA_BUSWIDTHS;
1440 dma_dev->dst_addr_widths = NBPF_DMA_BUSWIDTHS;
1441 dma_dev->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
1442
1443 platform_set_drvdata(pdev, nbpf);
1444
1445 ret = clk_prepare_enable(nbpf->clk);
1446 if (ret < 0)
1447 return ret;
1448
1449 nbpf_configure(nbpf);
1450
1451 ret = dma_async_device_register(dma_dev);
1452 if (ret < 0)
1453 goto e_clk_off;
1454
1455 ret = of_dma_controller_register(np, nbpf_of_xlate, nbpf);
1456 if (ret < 0)
1457 goto e_dma_dev_unreg;
1458
1459 return 0;
1460
1461 e_dma_dev_unreg:
1462 dma_async_device_unregister(dma_dev);
1463 e_clk_off:
1464 clk_disable_unprepare(nbpf->clk);
1465
1466 return ret;
1467 }
1468
nbpf_remove(struct platform_device * pdev)1469 static void nbpf_remove(struct platform_device *pdev)
1470 {
1471 struct nbpf_device *nbpf = platform_get_drvdata(pdev);
1472 int i;
1473
1474 devm_free_irq(&pdev->dev, nbpf->eirq, nbpf);
1475
1476 for (i = 0; i < nbpf->config->num_channels; i++) {
1477 struct nbpf_channel *chan = nbpf->chan + i;
1478
1479 devm_free_irq(&pdev->dev, chan->irq, chan);
1480
1481 tasklet_kill(&chan->tasklet);
1482 }
1483
1484 of_dma_controller_free(pdev->dev.of_node);
1485 dma_async_device_unregister(&nbpf->dma_dev);
1486 clk_disable_unprepare(nbpf->clk);
1487 }
1488
1489 static const struct platform_device_id nbpf_ids[] = {
1490 {"nbpfaxi64dmac1b4", (kernel_ulong_t)&nbpf_cfg[NBPF1B4]},
1491 {"nbpfaxi64dmac1b8", (kernel_ulong_t)&nbpf_cfg[NBPF1B8]},
1492 {"nbpfaxi64dmac1b16", (kernel_ulong_t)&nbpf_cfg[NBPF1B16]},
1493 {"nbpfaxi64dmac4b4", (kernel_ulong_t)&nbpf_cfg[NBPF4B4]},
1494 {"nbpfaxi64dmac4b8", (kernel_ulong_t)&nbpf_cfg[NBPF4B8]},
1495 {"nbpfaxi64dmac4b16", (kernel_ulong_t)&nbpf_cfg[NBPF4B16]},
1496 {"nbpfaxi64dmac8b4", (kernel_ulong_t)&nbpf_cfg[NBPF8B4]},
1497 {"nbpfaxi64dmac8b8", (kernel_ulong_t)&nbpf_cfg[NBPF8B8]},
1498 {"nbpfaxi64dmac8b16", (kernel_ulong_t)&nbpf_cfg[NBPF8B16]},
1499 {},
1500 };
1501 MODULE_DEVICE_TABLE(platform, nbpf_ids);
1502
1503 #ifdef CONFIG_PM
nbpf_runtime_suspend(struct device * dev)1504 static int nbpf_runtime_suspend(struct device *dev)
1505 {
1506 struct nbpf_device *nbpf = dev_get_drvdata(dev);
1507 clk_disable_unprepare(nbpf->clk);
1508 return 0;
1509 }
1510
nbpf_runtime_resume(struct device * dev)1511 static int nbpf_runtime_resume(struct device *dev)
1512 {
1513 struct nbpf_device *nbpf = dev_get_drvdata(dev);
1514 return clk_prepare_enable(nbpf->clk);
1515 }
1516 #endif
1517
1518 static const struct dev_pm_ops nbpf_pm_ops = {
1519 SET_RUNTIME_PM_OPS(nbpf_runtime_suspend, nbpf_runtime_resume, NULL)
1520 };
1521
1522 static struct platform_driver nbpf_driver = {
1523 .driver = {
1524 .name = "dma-nbpf",
1525 .of_match_table = nbpf_match,
1526 .pm = &nbpf_pm_ops,
1527 },
1528 .id_table = nbpf_ids,
1529 .probe = nbpf_probe,
1530 .remove = nbpf_remove,
1531 };
1532
1533 module_platform_driver(nbpf_driver);
1534
1535 MODULE_AUTHOR("Guennadi Liakhovetski <g.liakhovetski@gmx.de>");
1536 MODULE_DESCRIPTION("dmaengine driver for NBPFAXI64* DMACs");
1537 MODULE_LICENSE("GPL v2");
1538