1 /*-
2 * Copyright (c) 2017-2018 Ruslan Bukin <br@bsdpad.com>
3 * All rights reserved.
4 *
5 * This software was developed by SRI International and the University of
6 * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
7 * ("CTSRD"), as part of the DARPA CRASH research programme.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 /* ARM PrimeCell DMA Controller (PL330) driver. */
32
33 #include <sys/cdefs.h>
34 #include "opt_platform.h"
35 #include <sys/param.h>
36 #include <sys/endian.h>
37 #include <sys/systm.h>
38 #include <sys/conf.h>
39 #include <sys/bus.h>
40 #include <sys/kernel.h>
41 #include <sys/kthread.h>
42 #include <sys/sglist.h>
43 #include <sys/module.h>
44 #include <sys/lock.h>
45 #include <sys/resource.h>
46 #include <sys/rman.h>
47
48 #include <vm/vm.h>
49 #include <vm/vm_extern.h>
50 #include <vm/vm_kern.h>
51 #include <vm/pmap.h>
52
53 #include <machine/bus.h>
54
55 #ifdef FDT
56 #include <dev/fdt/fdt_common.h>
57 #include <dev/ofw/ofw_bus.h>
58 #include <dev/ofw/ofw_bus_subr.h>
59 #endif
60
61 #include <dev/xdma/xdma.h>
62 #include <dev/xdma/controller/pl330.h>
63
64 #include "xdma_if.h"
65
66 #define PL330_DEBUG
67 #undef PL330_DEBUG
68
69 #ifdef PL330_DEBUG
70 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
71 #else
72 #define dprintf(fmt, ...)
73 #endif
74
75 #define READ4(_sc, _reg) \
76 bus_read_4(_sc->res[0], _reg)
77 #define WRITE4(_sc, _reg, _val) \
78 bus_write_4(_sc->res[0], _reg, _val)
79
80 #define PL330_NCHANNELS 32
81 #define PL330_MAXLOAD 2048
82
83 struct pl330_channel {
84 struct pl330_softc *sc;
85 xdma_channel_t *xchan;
86 int used;
87 int index;
88 uint8_t *ibuf;
89 bus_addr_t ibuf_phys;
90 uint32_t enqueued;
91 uint32_t capacity;
92 };
93
94 struct pl330_fdt_data {
95 uint32_t periph_id;
96 };
97
98 struct pl330_softc {
99 device_t dev;
100 struct resource *res[PL330_NCHANNELS + 1];
101 void *ih[PL330_NCHANNELS];
102 struct pl330_channel channels[PL330_NCHANNELS];
103 };
104
105 static struct resource_spec pl330_spec[] = {
106 { SYS_RES_MEMORY, 0, RF_ACTIVE },
107 { SYS_RES_IRQ, 0, RF_ACTIVE },
108 { SYS_RES_IRQ, 1, RF_ACTIVE | RF_OPTIONAL },
109 { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL },
110 { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL },
111 { SYS_RES_IRQ, 4, RF_ACTIVE | RF_OPTIONAL },
112 { SYS_RES_IRQ, 5, RF_ACTIVE | RF_OPTIONAL },
113 { SYS_RES_IRQ, 6, RF_ACTIVE | RF_OPTIONAL },
114 { SYS_RES_IRQ, 7, RF_ACTIVE | RF_OPTIONAL },
115 { SYS_RES_IRQ, 8, RF_ACTIVE | RF_OPTIONAL },
116 { SYS_RES_IRQ, 9, RF_ACTIVE | RF_OPTIONAL },
117 { SYS_RES_IRQ, 10, RF_ACTIVE | RF_OPTIONAL },
118 { SYS_RES_IRQ, 11, RF_ACTIVE | RF_OPTIONAL },
119 { SYS_RES_IRQ, 12, RF_ACTIVE | RF_OPTIONAL },
120 { SYS_RES_IRQ, 13, RF_ACTIVE | RF_OPTIONAL },
121 { SYS_RES_IRQ, 14, RF_ACTIVE | RF_OPTIONAL },
122 { SYS_RES_IRQ, 15, RF_ACTIVE | RF_OPTIONAL },
123 { SYS_RES_IRQ, 16, RF_ACTIVE | RF_OPTIONAL },
124 { SYS_RES_IRQ, 17, RF_ACTIVE | RF_OPTIONAL },
125 { SYS_RES_IRQ, 18, RF_ACTIVE | RF_OPTIONAL },
126 { SYS_RES_IRQ, 19, RF_ACTIVE | RF_OPTIONAL },
127 { SYS_RES_IRQ, 20, RF_ACTIVE | RF_OPTIONAL },
128 { SYS_RES_IRQ, 21, RF_ACTIVE | RF_OPTIONAL },
129 { SYS_RES_IRQ, 22, RF_ACTIVE | RF_OPTIONAL },
130 { SYS_RES_IRQ, 23, RF_ACTIVE | RF_OPTIONAL },
131 { SYS_RES_IRQ, 24, RF_ACTIVE | RF_OPTIONAL },
132 { SYS_RES_IRQ, 25, RF_ACTIVE | RF_OPTIONAL },
133 { SYS_RES_IRQ, 26, RF_ACTIVE | RF_OPTIONAL },
134 { SYS_RES_IRQ, 27, RF_ACTIVE | RF_OPTIONAL },
135 { SYS_RES_IRQ, 28, RF_ACTIVE | RF_OPTIONAL },
136 { SYS_RES_IRQ, 29, RF_ACTIVE | RF_OPTIONAL },
137 { SYS_RES_IRQ, 30, RF_ACTIVE | RF_OPTIONAL },
138 { SYS_RES_IRQ, 31, RF_ACTIVE | RF_OPTIONAL },
139 { -1, 0 }
140 };
141
142 #define HWTYPE_NONE 0
143 #define HWTYPE_STD 1
144
145 static struct ofw_compat_data compat_data[] = {
146 { "arm,pl330", HWTYPE_STD },
147 { NULL, HWTYPE_NONE },
148 };
149
150 static void
pl330_intr(void * arg)151 pl330_intr(void *arg)
152 {
153 xdma_transfer_status_t status;
154 struct xdma_transfer_status st;
155 struct pl330_channel *chan;
156 struct xdma_channel *xchan;
157 struct pl330_softc *sc;
158 uint32_t pending;
159 int i;
160 int c;
161
162 sc = arg;
163
164 pending = READ4(sc, INTMIS);
165
166 dprintf("%s: 0x%x, LC0 %x, SAR %x DAR %x\n",
167 __func__, pending, READ4(sc, LC0(0)),
168 READ4(sc, SAR(0)), READ4(sc, DAR(0)));
169
170 WRITE4(sc, INTCLR, pending);
171
172 for (c = 0; c < PL330_NCHANNELS; c++) {
173 if ((pending & (1 << c)) == 0) {
174 continue;
175 }
176 chan = &sc->channels[c];
177 xchan = chan->xchan;
178 st.error = 0;
179 st.transferred = 0;
180 for (i = 0; i < chan->enqueued; i++) {
181 xchan_seg_done(xchan, &st);
182 }
183
184 /* Accept new requests. */
185 chan->capacity = PL330_MAXLOAD;
186
187 /* Finish operation */
188 status.error = 0;
189 status.transferred = 0;
190 xdma_callback(chan->xchan, &status);
191 }
192 }
193
194 static uint32_t
emit_mov(uint8_t * buf,uint32_t reg,uint32_t val)195 emit_mov(uint8_t *buf, uint32_t reg, uint32_t val)
196 {
197
198 buf[0] = DMAMOV;
199 buf[1] = reg;
200 buf[2] = val;
201 buf[3] = val >> 8;
202 buf[4] = val >> 16;
203 buf[5] = val >> 24;
204
205 return (6);
206 }
207
208 static uint32_t
emit_lp(uint8_t * buf,uint8_t idx,uint32_t iter)209 emit_lp(uint8_t *buf, uint8_t idx, uint32_t iter)
210 {
211
212 if (idx > 1)
213 return (0); /* We have two loops only. */
214
215 buf[0] = DMALP;
216 buf[0] |= (idx << 1);
217 buf[1] = (iter - 1) & 0xff;
218
219 return (2);
220 }
221
222 static uint32_t
emit_lpend(uint8_t * buf,uint8_t idx,uint8_t burst,uint8_t jump_addr_relative)223 emit_lpend(uint8_t *buf, uint8_t idx,
224 uint8_t burst, uint8_t jump_addr_relative)
225 {
226
227 buf[0] = DMALPEND;
228 buf[0] |= DMALPEND_NF;
229 buf[0] |= (idx << 2);
230 if (burst)
231 buf[0] |= (1 << 1) | (1 << 0);
232 else
233 buf[0] |= (0 << 1) | (1 << 0);
234 buf[1] = jump_addr_relative;
235
236 return (2);
237 }
238
239 static uint32_t
emit_ld(uint8_t * buf,uint8_t burst)240 emit_ld(uint8_t *buf, uint8_t burst)
241 {
242
243 buf[0] = DMALD;
244 if (burst)
245 buf[0] |= (1 << 1) | (1 << 0);
246 else
247 buf[0] |= (0 << 1) | (1 << 0);
248
249 return (1);
250 }
251
252 static uint32_t
emit_st(uint8_t * buf,uint8_t burst)253 emit_st(uint8_t *buf, uint8_t burst)
254 {
255
256 buf[0] = DMAST;
257 if (burst)
258 buf[0] |= (1 << 1) | (1 << 0);
259 else
260 buf[0] |= (0 << 1) | (1 << 0);
261
262 return (1);
263 }
264
265 static uint32_t
emit_end(uint8_t * buf)266 emit_end(uint8_t *buf)
267 {
268
269 buf[0] = DMAEND;
270
271 return (1);
272 }
273
274 static uint32_t
emit_sev(uint8_t * buf,uint32_t ev)275 emit_sev(uint8_t *buf, uint32_t ev)
276 {
277
278 buf[0] = DMASEV;
279 buf[1] = (ev << 3);
280
281 return (2);
282 }
283
284 static uint32_t
emit_wfp(uint8_t * buf,uint32_t p_id)285 emit_wfp(uint8_t *buf, uint32_t p_id)
286 {
287
288 buf[0] = DMAWFP;
289 buf[0] |= (1 << 0);
290 buf[1] = (p_id << 3);
291
292 return (2);
293 }
294
295 static uint32_t
emit_go(uint8_t * buf,uint32_t chan_id,uint32_t addr,uint8_t non_secure)296 emit_go(uint8_t *buf, uint32_t chan_id,
297 uint32_t addr, uint8_t non_secure)
298 {
299
300 buf[0] = DMAGO;
301 buf[0] |= (non_secure << 1);
302
303 buf[1] = chan_id;
304 buf[2] = addr;
305 buf[3] = addr >> 8;
306 buf[4] = addr >> 16;
307 buf[5] = addr >> 24;
308
309 return (6);
310 }
311
312 static int
pl330_probe(device_t dev)313 pl330_probe(device_t dev)
314 {
315 int hwtype;
316
317 if (!ofw_bus_status_okay(dev))
318 return (ENXIO);
319
320 hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
321 if (hwtype == HWTYPE_NONE)
322 return (ENXIO);
323
324 device_set_desc(dev, "ARM PrimeCell DMA Controller (PL330)");
325
326 return (BUS_PROBE_DEFAULT);
327 }
328
329 static int
pl330_attach(device_t dev)330 pl330_attach(device_t dev)
331 {
332 struct pl330_softc *sc;
333 phandle_t xref, node;
334 int err;
335 int i;
336
337 sc = device_get_softc(dev);
338 sc->dev = dev;
339
340 if (bus_alloc_resources(dev, pl330_spec, sc->res)) {
341 device_printf(dev, "could not allocate resources for device\n");
342 return (ENXIO);
343 }
344
345 /* Setup interrupt handler */
346 for (i = 0; i < PL330_NCHANNELS; i++) {
347 if (sc->res[i + 1] == NULL)
348 break;
349 err = bus_setup_intr(dev, sc->res[i + 1], INTR_TYPE_MISC | INTR_MPSAFE,
350 NULL, pl330_intr, sc, sc->ih[i]);
351 if (err) {
352 device_printf(dev, "Unable to alloc interrupt resource.\n");
353 return (ENXIO);
354 }
355 }
356
357 node = ofw_bus_get_node(dev);
358 xref = OF_xref_from_node(node);
359 OF_device_register_xref(xref, dev);
360
361 return (0);
362 }
363
364 static int
pl330_detach(device_t dev)365 pl330_detach(device_t dev)
366 {
367 return (0);
368 }
369
370 static int
pl330_channel_alloc(device_t dev,struct xdma_channel * xchan)371 pl330_channel_alloc(device_t dev, struct xdma_channel *xchan)
372 {
373 struct pl330_channel *chan;
374 struct pl330_softc *sc;
375 int i;
376
377 sc = device_get_softc(dev);
378
379 for (i = 0; i < PL330_NCHANNELS; i++) {
380 chan = &sc->channels[i];
381 if (chan->used == 0) {
382 chan->xchan = xchan;
383 xchan->chan = (void *)chan;
384 xchan->caps |= XCHAN_CAP_BUSDMA;
385 chan->index = i;
386 chan->sc = sc;
387 chan->used = 1;
388
389 chan->ibuf = (void *)kmem_alloc_contig(PAGE_SIZE * 8,
390 M_ZERO, 0, ~0, PAGE_SIZE, 0,
391 VM_MEMATTR_UNCACHEABLE);
392 chan->ibuf_phys = vtophys(chan->ibuf);
393
394 return (0);
395 }
396 }
397
398 return (-1);
399 }
400
401 static int
pl330_channel_free(device_t dev,struct xdma_channel * xchan)402 pl330_channel_free(device_t dev, struct xdma_channel *xchan)
403 {
404 struct pl330_channel *chan;
405
406 chan = (struct pl330_channel *)xchan->chan;
407 chan->used = 0;
408
409 return (0);
410 }
411
412 static int
pl330_channel_capacity(device_t dev,xdma_channel_t * xchan,uint32_t * capacity)413 pl330_channel_capacity(device_t dev, xdma_channel_t *xchan,
414 uint32_t *capacity)
415 {
416 struct pl330_channel *chan;
417
418 chan = (struct pl330_channel *)xchan->chan;
419
420 *capacity = chan->capacity;
421
422 return (0);
423 }
424
425 static int
pl330_ccr_port_width(struct xdma_sglist * sg,uint32_t * addr)426 pl330_ccr_port_width(struct xdma_sglist *sg, uint32_t *addr)
427 {
428 uint32_t reg;
429
430 reg = 0;
431
432 switch (sg->src_width) {
433 case 1:
434 reg |= CCR_SRC_BURST_SIZE_1;
435 break;
436 case 2:
437 reg |= CCR_SRC_BURST_SIZE_2;
438 break;
439 case 4:
440 reg |= CCR_SRC_BURST_SIZE_4;
441 break;
442 default:
443 return (-1);
444 }
445
446 switch (sg->dst_width) {
447 case 1:
448 reg |= CCR_DST_BURST_SIZE_1;
449 break;
450 case 2:
451 reg |= CCR_DST_BURST_SIZE_2;
452 break;
453 case 4:
454 reg |= CCR_DST_BURST_SIZE_4;
455 break;
456 default:
457 return (-1);
458 }
459
460 *addr |= reg;
461
462 return (0);
463 }
464
465 static int
pl330_channel_submit_sg(device_t dev,struct xdma_channel * xchan,struct xdma_sglist * sg,uint32_t sg_n)466 pl330_channel_submit_sg(device_t dev, struct xdma_channel *xchan,
467 struct xdma_sglist *sg, uint32_t sg_n)
468 {
469 struct pl330_fdt_data *data;
470 xdma_controller_t *xdma;
471 struct pl330_channel *chan;
472 struct pl330_softc *sc;
473 uint32_t src_addr_lo;
474 uint32_t dst_addr_lo;
475 uint32_t len;
476 uint32_t reg;
477 uint32_t offs;
478 uint32_t cnt;
479 uint8_t *ibuf;
480 uint8_t dbuf[6];
481 uint8_t offs0, offs1;
482 int err;
483 int i;
484
485 sc = device_get_softc(dev);
486
487 xdma = xchan->xdma;
488 data = (struct pl330_fdt_data *)xdma->data;
489
490 chan = (struct pl330_channel *)xchan->chan;
491 ibuf = chan->ibuf;
492
493 dprintf("%s: chan->index %d\n", __func__, chan->index);
494
495 offs = 0;
496
497 for (i = 0; i < sg_n; i++) {
498 if (sg[i].direction == XDMA_DEV_TO_MEM)
499 reg = CCR_DST_INC;
500 else {
501 reg = CCR_SRC_INC;
502 reg |= (CCR_DST_PROT_PRIV);
503 }
504
505 err = pl330_ccr_port_width(&sg[i], ®);
506 if (err != 0)
507 return (err);
508
509 offs += emit_mov(&chan->ibuf[offs], R_CCR, reg);
510
511 src_addr_lo = (uint32_t)sg[i].src_addr;
512 dst_addr_lo = (uint32_t)sg[i].dst_addr;
513 len = (uint32_t)sg[i].len;
514
515 dprintf("%s: src %x dst %x len %d periph_id %d\n", __func__,
516 src_addr_lo, dst_addr_lo, len, data->periph_id);
517
518 offs += emit_mov(&ibuf[offs], R_SAR, src_addr_lo);
519 offs += emit_mov(&ibuf[offs], R_DAR, dst_addr_lo);
520
521 if (sg[i].src_width != sg[i].dst_width)
522 return (-1); /* Not supported. */
523
524 cnt = (len / sg[i].src_width);
525 if (cnt > 128) {
526 offs += emit_lp(&ibuf[offs], 0, cnt / 128);
527 offs0 = offs;
528 offs += emit_lp(&ibuf[offs], 1, 128);
529 offs1 = offs;
530 } else {
531 offs += emit_lp(&ibuf[offs], 0, cnt);
532 offs0 = offs;
533 }
534 offs += emit_wfp(&ibuf[offs], data->periph_id);
535 offs += emit_ld(&ibuf[offs], 1);
536 offs += emit_st(&ibuf[offs], 1);
537
538 if (cnt > 128)
539 offs += emit_lpend(&ibuf[offs], 1, 1, (offs - offs1));
540
541 offs += emit_lpend(&ibuf[offs], 0, 1, (offs - offs0));
542 }
543
544 offs += emit_sev(&ibuf[offs], chan->index);
545 offs += emit_end(&ibuf[offs]);
546
547 emit_go(dbuf, chan->index, chan->ibuf_phys, 0);
548
549 reg = (dbuf[1] << 24) | (dbuf[0] << 16);
550 WRITE4(sc, DBGINST0, reg);
551 reg = (dbuf[5] << 24) | (dbuf[4] << 16) | (dbuf[3] << 8) | dbuf[2];
552 WRITE4(sc, DBGINST1, reg);
553
554 WRITE4(sc, INTCLR, 0xffffffff);
555 WRITE4(sc, INTEN, (1 << chan->index));
556
557 chan->enqueued = sg_n;
558 chan->capacity = 0;
559
560 /* Start operation */
561 WRITE4(sc, DBGCMD, 0);
562
563 return (0);
564 }
565
566 static int
pl330_channel_prep_sg(device_t dev,struct xdma_channel * xchan)567 pl330_channel_prep_sg(device_t dev, struct xdma_channel *xchan)
568 {
569 struct pl330_channel *chan;
570
571 dprintf("%s(%d)\n", __func__, device_get_unit(dev));
572
573 chan = (struct pl330_channel *)xchan->chan;
574 chan->capacity = PL330_MAXLOAD;
575
576 return (0);
577 }
578
579 static int
pl330_channel_control(device_t dev,xdma_channel_t * xchan,int cmd)580 pl330_channel_control(device_t dev, xdma_channel_t *xchan, int cmd)
581 {
582 switch (cmd) {
583 case XDMA_CMD_BEGIN:
584 case XDMA_CMD_TERMINATE:
585 case XDMA_CMD_PAUSE:
586 /* TODO: implement me */
587 return (-1);
588 }
589
590 return (0);
591 }
592
593 #ifdef FDT
594 static int
pl330_ofw_md_data(device_t dev,pcell_t * cells,int ncells,void ** ptr)595 pl330_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr)
596 {
597 struct pl330_fdt_data *data;
598
599 if (ncells != 1)
600 return (-1);
601
602 data = malloc(sizeof(struct pl330_fdt_data),
603 M_DEVBUF, (M_WAITOK | M_ZERO));
604 data->periph_id = cells[0];
605
606 *ptr = data;
607
608 return (0);
609 }
610 #endif
611
612 static device_method_t pl330_methods[] = {
613 /* Device interface */
614 DEVMETHOD(device_probe, pl330_probe),
615 DEVMETHOD(device_attach, pl330_attach),
616 DEVMETHOD(device_detach, pl330_detach),
617
618 /* xDMA Interface */
619 DEVMETHOD(xdma_channel_alloc, pl330_channel_alloc),
620 DEVMETHOD(xdma_channel_free, pl330_channel_free),
621 DEVMETHOD(xdma_channel_control, pl330_channel_control),
622
623 /* xDMA SG Interface */
624 DEVMETHOD(xdma_channel_capacity, pl330_channel_capacity),
625 DEVMETHOD(xdma_channel_prep_sg, pl330_channel_prep_sg),
626 DEVMETHOD(xdma_channel_submit_sg, pl330_channel_submit_sg),
627
628 #ifdef FDT
629 DEVMETHOD(xdma_ofw_md_data, pl330_ofw_md_data),
630 #endif
631
632 DEVMETHOD_END
633 };
634
635 static driver_t pl330_driver = {
636 "pl330",
637 pl330_methods,
638 sizeof(struct pl330_softc),
639 };
640
641 EARLY_DRIVER_MODULE(pl330, simplebus, pl330_driver, 0, 0,
642 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE);
643