1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019 Ruslan Bukin <br@bsdpad.com>
5 *
6 * This software was developed by SRI International and the University of
7 * Cambridge Computer Laboratory (Department of Computer Science and
8 * Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the
9 * DARPA SSITH research programme.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 /* Xilinx AXI DMA controller driver. */
34
35 #include <sys/cdefs.h>
36 #include "opt_platform.h"
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/conf.h>
40 #include <sys/bus.h>
41 #include <sys/kernel.h>
42 #include <sys/module.h>
43 #include <sys/rman.h>
44
45 #include <machine/bus.h>
46
47 #include <vm/vm.h>
48 #include <vm/vm_extern.h>
49 #include <vm/vm_page.h>
50
51 #ifdef FDT
52 #include <dev/fdt/fdt_common.h>
53 #include <dev/ofw/ofw_bus.h>
54 #include <dev/ofw/ofw_bus_subr.h>
55 #endif
56
57 #include <dev/xdma/xdma.h>
58 #include <dev/xilinx/axidma.h>
59
60 #include "xdma_if.h"
61
62 #define READ4(_sc, _reg) \
63 bus_space_read_4(_sc->bst, _sc->bsh, _reg)
64 #define WRITE4(_sc, _reg, _val) \
65 bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val)
66 #define READ8(_sc, _reg) \
67 bus_space_read_8(_sc->bst, _sc->bsh, _reg)
68 #define WRITE8(_sc, _reg, _val) \
69 bus_space_write_8(_sc->bst, _sc->bsh, _reg, _val)
70
71 #define AXIDMA_DEBUG
72 #undef AXIDMA_DEBUG
73
74 #ifdef AXIDMA_DEBUG
75 #define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
76 #else
77 #define dprintf(fmt, ...)
78 #endif
79
80 extern struct bus_space memmap_bus;
81
82 struct axidma_channel {
83 struct axidma_softc *sc;
84 xdma_channel_t *xchan;
85 bool used;
86 int idx_head;
87 int idx_tail;
88
89 struct axidma_desc **descs;
90 vm_paddr_t *descs_phys;
91 uint32_t descs_num;
92
93 vm_size_t mem_size;
94 vm_offset_t mem_paddr;
95 vm_offset_t mem_vaddr;
96
97 uint32_t descs_used_count;
98 };
99
100 struct axidma_softc {
101 device_t dev;
102 struct resource *res[3];
103 bus_space_tag_t bst;
104 bus_space_handle_t bsh;
105 void *ih[2];
106 struct axidma_desc desc;
107 struct axidma_channel channels[AXIDMA_NCHANNELS];
108 };
109
110 static struct resource_spec axidma_spec[] = {
111 { SYS_RES_MEMORY, 0, RF_ACTIVE },
112 { SYS_RES_IRQ, 0, RF_ACTIVE },
113 { SYS_RES_IRQ, 1, RF_ACTIVE },
114 { -1, 0 }
115 };
116
117 #define HWTYPE_NONE 0
118 #define HWTYPE_STD 1
119
120 static struct ofw_compat_data compat_data[] = {
121 { "xlnx,eth-dma", HWTYPE_STD },
122 { NULL, HWTYPE_NONE },
123 };
124
125 static int axidma_probe(device_t dev);
126 static int axidma_attach(device_t dev);
127 static int axidma_detach(device_t dev);
128
129 static inline uint32_t
axidma_next_desc(struct axidma_channel * chan,uint32_t curidx)130 axidma_next_desc(struct axidma_channel *chan, uint32_t curidx)
131 {
132
133 return ((curidx + 1) % chan->descs_num);
134 }
135
136 static void
axidma_intr(struct axidma_softc * sc,struct axidma_channel * chan)137 axidma_intr(struct axidma_softc *sc,
138 struct axidma_channel *chan)
139 {
140 xdma_transfer_status_t status;
141 xdma_transfer_status_t st;
142 struct axidma_fdt_data *data;
143 xdma_controller_t *xdma;
144 struct axidma_desc *desc;
145 struct xdma_channel *xchan;
146 uint32_t tot_copied;
147 int pending;
148 int errors;
149
150 xchan = chan->xchan;
151 xdma = xchan->xdma;
152 data = xdma->data;
153
154 pending = READ4(sc, AXI_DMASR(data->id));
155 WRITE4(sc, AXI_DMASR(data->id), pending);
156
157 errors = (pending & (DMASR_DMAINTERR | DMASR_DMASLVERR
158 | DMASR_DMADECOREERR | DMASR_SGINTERR
159 | DMASR_SGSLVERR | DMASR_SGDECERR));
160
161 dprintf("%s: AXI_DMASR %x\n", __func__,
162 READ4(sc, AXI_DMASR(data->id)));
163 dprintf("%s: AXI_CURDESC %x\n", __func__,
164 READ4(sc, AXI_CURDESC(data->id)));
165 dprintf("%s: AXI_TAILDESC %x\n", __func__,
166 READ4(sc, AXI_TAILDESC(data->id)));
167
168 tot_copied = 0;
169
170 while (chan->idx_tail != chan->idx_head) {
171 desc = chan->descs[chan->idx_tail];
172 if ((desc->status & BD_STATUS_CMPLT) == 0)
173 break;
174
175 st.error = errors;
176 st.transferred = desc->status & BD_CONTROL_LEN_M;
177 tot_copied += st.transferred;
178 xchan_seg_done(xchan, &st);
179
180 chan->idx_tail = axidma_next_desc(chan, chan->idx_tail);
181 atomic_subtract_int(&chan->descs_used_count, 1);
182 }
183
184 /* Finish operation */
185 status.error = errors;
186 status.transferred = tot_copied;
187 xdma_callback(chan->xchan, &status);
188 }
189
190 static void
axidma_intr_rx(void * arg)191 axidma_intr_rx(void *arg)
192 {
193 struct axidma_softc *sc;
194 struct axidma_channel *chan;
195
196 dprintf("%s\n", __func__);
197
198 sc = arg;
199 chan = &sc->channels[AXIDMA_RX_CHAN];
200
201 axidma_intr(sc, chan);
202 }
203
204 static void
axidma_intr_tx(void * arg)205 axidma_intr_tx(void *arg)
206 {
207 struct axidma_softc *sc;
208 struct axidma_channel *chan;
209
210 dprintf("%s\n", __func__);
211
212 sc = arg;
213 chan = &sc->channels[AXIDMA_TX_CHAN];
214
215 axidma_intr(sc, chan);
216 }
217
218 static int
axidma_reset(struct axidma_softc * sc,int chan_id)219 axidma_reset(struct axidma_softc *sc, int chan_id)
220 {
221 int timeout;
222
223 WRITE4(sc, AXI_DMACR(chan_id), DMACR_RESET);
224
225 timeout = 100;
226 do {
227 if ((READ4(sc, AXI_DMACR(chan_id)) & DMACR_RESET) == 0)
228 break;
229 } while (timeout--);
230
231 dprintf("timeout %d\n", timeout);
232
233 if (timeout == 0)
234 return (-1);
235
236 dprintf("%s: read control after reset: %x\n",
237 __func__, READ4(sc, AXI_DMACR(chan_id)));
238
239 return (0);
240 }
241
242 static int
axidma_probe(device_t dev)243 axidma_probe(device_t dev)
244 {
245 int hwtype;
246
247 if (!ofw_bus_status_okay(dev))
248 return (ENXIO);
249
250 hwtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data;
251 if (hwtype == HWTYPE_NONE)
252 return (ENXIO);
253
254 device_set_desc(dev, "Xilinx AXI DMA");
255
256 return (BUS_PROBE_DEFAULT);
257 }
258
259 static int
axidma_attach(device_t dev)260 axidma_attach(device_t dev)
261 {
262 struct axidma_softc *sc;
263 phandle_t xref, node;
264 int err;
265
266 sc = device_get_softc(dev);
267 sc->dev = dev;
268
269 if (bus_alloc_resources(dev, axidma_spec, sc->res)) {
270 device_printf(dev, "could not allocate resources.\n");
271 return (ENXIO);
272 }
273
274 /* CSR memory interface */
275 sc->bst = rman_get_bustag(sc->res[0]);
276 sc->bsh = rman_get_bushandle(sc->res[0]);
277
278 /* Setup interrupt handler */
279 err = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
280 NULL, axidma_intr_tx, sc, &sc->ih[0]);
281 if (err) {
282 device_printf(dev, "Unable to alloc interrupt resource.\n");
283 return (ENXIO);
284 }
285
286 /* Setup interrupt handler */
287 err = bus_setup_intr(dev, sc->res[2], INTR_TYPE_MISC | INTR_MPSAFE,
288 NULL, axidma_intr_rx, sc, &sc->ih[1]);
289 if (err) {
290 device_printf(dev, "Unable to alloc interrupt resource.\n");
291 return (ENXIO);
292 }
293
294 node = ofw_bus_get_node(dev);
295 xref = OF_xref_from_node(node);
296 OF_device_register_xref(xref, dev);
297
298 return (0);
299 }
300
301 static int
axidma_detach(device_t dev)302 axidma_detach(device_t dev)
303 {
304 struct axidma_softc *sc;
305
306 sc = device_get_softc(dev);
307
308 bus_teardown_intr(dev, sc->res[1], sc->ih[0]);
309 bus_teardown_intr(dev, sc->res[2], sc->ih[1]);
310 bus_release_resources(dev, axidma_spec, sc->res);
311
312 return (0);
313 }
314
315 static int
axidma_desc_free(struct axidma_softc * sc,struct axidma_channel * chan)316 axidma_desc_free(struct axidma_softc *sc, struct axidma_channel *chan)
317 {
318 struct xdma_channel *xchan;
319
320 xchan = chan->xchan;
321
322 free(chan->descs, M_DEVBUF);
323 free(chan->descs_phys, M_DEVBUF);
324
325 pmap_kremove_device(chan->mem_vaddr, chan->mem_size);
326 kva_free(chan->mem_vaddr, chan->mem_size);
327 vmem_free(xchan->vmem, chan->mem_paddr, chan->mem_size);
328
329 return (0);
330 }
331
332 static int
axidma_desc_alloc(struct axidma_softc * sc,struct xdma_channel * xchan,uint32_t desc_size)333 axidma_desc_alloc(struct axidma_softc *sc, struct xdma_channel *xchan,
334 uint32_t desc_size)
335 {
336 struct axidma_channel *chan;
337 int nsegments;
338 int i;
339
340 chan = (struct axidma_channel *)xchan->chan;
341 nsegments = chan->descs_num;
342
343 chan->descs = malloc(nsegments * sizeof(struct axidma_desc *),
344 M_DEVBUF, M_NOWAIT | M_ZERO);
345 if (chan->descs == NULL) {
346 device_printf(sc->dev,
347 "%s: Can't allocate memory.\n", __func__);
348 return (-1);
349 }
350
351 chan->descs_phys = malloc(nsegments * sizeof(bus_dma_segment_t),
352 M_DEVBUF, M_NOWAIT | M_ZERO);
353 chan->mem_size = desc_size * nsegments;
354 if (vmem_alloc(xchan->vmem, chan->mem_size, M_FIRSTFIT | M_NOWAIT,
355 &chan->mem_paddr)) {
356 device_printf(sc->dev, "Failed to allocate memory.\n");
357 return (-1);
358 }
359 chan->mem_vaddr = kva_alloc(chan->mem_size);
360 pmap_kenter_device(chan->mem_vaddr, chan->mem_size, chan->mem_paddr);
361
362 device_printf(sc->dev, "Allocated chunk %lx %lu\n",
363 chan->mem_paddr, chan->mem_size);
364
365 for (i = 0; i < nsegments; i++) {
366 chan->descs[i] = (struct axidma_desc *)
367 ((uint64_t)chan->mem_vaddr + desc_size * i);
368 chan->descs_phys[i] = chan->mem_paddr + desc_size * i;
369 }
370
371 return (0);
372 }
373
374 static int
axidma_channel_alloc(device_t dev,struct xdma_channel * xchan)375 axidma_channel_alloc(device_t dev, struct xdma_channel *xchan)
376 {
377 xdma_controller_t *xdma;
378 struct axidma_fdt_data *data;
379 struct axidma_channel *chan;
380 struct axidma_softc *sc;
381
382 sc = device_get_softc(dev);
383
384 if (xchan->caps & XCHAN_CAP_BUSDMA) {
385 device_printf(sc->dev,
386 "Error: busdma operation is not implemented.");
387 return (-1);
388 }
389
390 xdma = xchan->xdma;
391 data = xdma->data;
392
393 chan = &sc->channels[data->id];
394 if (chan->used == false) {
395 if (axidma_reset(sc, data->id) != 0)
396 return (-1);
397 chan->xchan = xchan;
398 xchan->caps |= XCHAN_CAP_BOUNCE;
399 xchan->chan = (void *)chan;
400 chan->sc = sc;
401 chan->used = true;
402 chan->idx_head = 0;
403 chan->idx_tail = 0;
404 chan->descs_used_count = 0;
405 chan->descs_num = AXIDMA_DESCS_NUM;
406
407 return (0);
408 }
409
410 return (-1);
411 }
412
413 static int
axidma_channel_free(device_t dev,struct xdma_channel * xchan)414 axidma_channel_free(device_t dev, struct xdma_channel *xchan)
415 {
416 struct axidma_channel *chan;
417 struct axidma_softc *sc;
418
419 sc = device_get_softc(dev);
420
421 chan = (struct axidma_channel *)xchan->chan;
422
423 axidma_desc_free(sc, chan);
424
425 chan->used = false;
426
427 return (0);
428 }
429
430 static int
axidma_channel_capacity(device_t dev,xdma_channel_t * xchan,uint32_t * capacity)431 axidma_channel_capacity(device_t dev, xdma_channel_t *xchan,
432 uint32_t *capacity)
433 {
434 struct axidma_channel *chan;
435 uint32_t c;
436
437 chan = (struct axidma_channel *)xchan->chan;
438
439 /* At least one descriptor must be left empty. */
440 c = (chan->descs_num - chan->descs_used_count - 1);
441
442 *capacity = c;
443
444 return (0);
445 }
446
447 static int
axidma_channel_submit_sg(device_t dev,struct xdma_channel * xchan,struct xdma_sglist * sg,uint32_t sg_n)448 axidma_channel_submit_sg(device_t dev, struct xdma_channel *xchan,
449 struct xdma_sglist *sg, uint32_t sg_n)
450 {
451 xdma_controller_t *xdma;
452 struct axidma_fdt_data *data;
453 struct axidma_channel *chan;
454 struct axidma_desc *desc;
455 struct axidma_softc *sc;
456 uint32_t src_addr;
457 uint32_t dst_addr;
458 uint32_t addr;
459 uint32_t len;
460 uint32_t tmp;
461 int i;
462
463 dprintf("%s: sg_n %d\n", __func__, sg_n);
464
465 sc = device_get_softc(dev);
466
467 chan = (struct axidma_channel *)xchan->chan;
468 xdma = xchan->xdma;
469 data = xdma->data;
470
471 if (sg_n == 0)
472 return (0);
473
474 tmp = 0;
475
476 for (i = 0; i < sg_n; i++) {
477 src_addr = (uint32_t)sg[i].src_addr;
478 dst_addr = (uint32_t)sg[i].dst_addr;
479 len = (uint32_t)sg[i].len;
480
481 dprintf("%s(%d): src %x dst %x len %d\n", __func__,
482 data->id, src_addr, dst_addr, len);
483
484 desc = chan->descs[chan->idx_head];
485 if (sg[i].direction == XDMA_MEM_TO_DEV)
486 desc->phys = src_addr;
487 else
488 desc->phys = dst_addr;
489 desc->status = 0;
490 desc->control = len;
491 if (sg[i].first == 1)
492 desc->control |= BD_CONTROL_TXSOF;
493 if (sg[i].last == 1)
494 desc->control |= BD_CONTROL_TXEOF;
495
496 tmp = chan->idx_head;
497
498 atomic_add_int(&chan->descs_used_count, 1);
499 chan->idx_head = axidma_next_desc(chan, chan->idx_head);
500 }
501
502 dprintf("%s(%d): _curdesc %x\n", __func__, data->id,
503 READ8(sc, AXI_CURDESC(data->id)));
504 dprintf("%s(%d): _curdesc %x\n", __func__, data->id,
505 READ8(sc, AXI_CURDESC(data->id)));
506 dprintf("%s(%d): status %x\n", __func__, data->id,
507 READ4(sc, AXI_DMASR(data->id)));
508
509 addr = chan->descs_phys[tmp];
510 WRITE8(sc, AXI_TAILDESC(data->id), addr);
511
512 return (0);
513 }
514
515 static int
axidma_channel_prep_sg(device_t dev,struct xdma_channel * xchan)516 axidma_channel_prep_sg(device_t dev, struct xdma_channel *xchan)
517 {
518 xdma_controller_t *xdma;
519 struct axidma_fdt_data *data;
520 struct axidma_channel *chan;
521 struct axidma_desc *desc;
522 struct axidma_softc *sc;
523 uint32_t addr;
524 uint32_t reg;
525 int ret;
526 int i;
527
528 sc = device_get_softc(dev);
529
530 chan = (struct axidma_channel *)xchan->chan;
531 xdma = xchan->xdma;
532 data = xdma->data;
533
534 dprintf("%s(%d)\n", __func__, data->id);
535
536 ret = axidma_desc_alloc(sc, xchan, sizeof(struct axidma_desc));
537 if (ret != 0) {
538 device_printf(sc->dev,
539 "%s: Can't allocate descriptors.\n", __func__);
540 return (-1);
541 }
542
543 for (i = 0; i < chan->descs_num; i++) {
544 desc = chan->descs[i];
545 bzero(desc, sizeof(struct axidma_desc));
546
547 if (i == (chan->descs_num - 1))
548 desc->next = chan->descs_phys[0];
549 else
550 desc->next = chan->descs_phys[i + 1];
551 desc->status = 0;
552 desc->control = 0;
553
554 dprintf("%s(%d): desc %d vaddr %lx next paddr %x\n", __func__,
555 data->id, i, (uint64_t)desc, le32toh(desc->next));
556 }
557
558 addr = chan->descs_phys[0];
559 WRITE8(sc, AXI_CURDESC(data->id), addr);
560
561 reg = READ4(sc, AXI_DMACR(data->id));
562 reg |= DMACR_IOC_IRQEN | DMACR_DLY_IRQEN | DMACR_ERR_IRQEN;
563 WRITE4(sc, AXI_DMACR(data->id), reg);
564 reg |= DMACR_RS;
565 WRITE4(sc, AXI_DMACR(data->id), reg);
566
567 return (0);
568 }
569
570 static int
axidma_channel_control(device_t dev,xdma_channel_t * xchan,int cmd)571 axidma_channel_control(device_t dev, xdma_channel_t *xchan, int cmd)
572 {
573
574 switch (cmd) {
575 case XDMA_CMD_BEGIN:
576 case XDMA_CMD_TERMINATE:
577 case XDMA_CMD_PAUSE:
578 /* TODO: implement me */
579 return (-1);
580 }
581
582 return (0);
583 }
584
585 #ifdef FDT
586 static int
axidma_ofw_md_data(device_t dev,pcell_t * cells,int ncells,void ** ptr)587 axidma_ofw_md_data(device_t dev, pcell_t *cells, int ncells, void **ptr)
588 {
589 struct axidma_fdt_data *data;
590
591 if (ncells != 1)
592 return (-1);
593
594 data = malloc(sizeof(struct axidma_fdt_data),
595 M_DEVBUF, (M_WAITOK | M_ZERO));
596 data->id = cells[0];
597
598 *ptr = data;
599
600 return (0);
601 }
602 #endif
603
604 static device_method_t axidma_methods[] = {
605 /* Device interface */
606 DEVMETHOD(device_probe, axidma_probe),
607 DEVMETHOD(device_attach, axidma_attach),
608 DEVMETHOD(device_detach, axidma_detach),
609
610 /* xDMA Interface */
611 DEVMETHOD(xdma_channel_alloc, axidma_channel_alloc),
612 DEVMETHOD(xdma_channel_free, axidma_channel_free),
613 DEVMETHOD(xdma_channel_control, axidma_channel_control),
614
615 /* xDMA SG Interface */
616 DEVMETHOD(xdma_channel_capacity, axidma_channel_capacity),
617 DEVMETHOD(xdma_channel_prep_sg, axidma_channel_prep_sg),
618 DEVMETHOD(xdma_channel_submit_sg, axidma_channel_submit_sg),
619
620 #ifdef FDT
621 DEVMETHOD(xdma_ofw_md_data, axidma_ofw_md_data),
622 #endif
623
624 DEVMETHOD_END
625 };
626
627 static driver_t axidma_driver = {
628 "axidma",
629 axidma_methods,
630 sizeof(struct axidma_softc),
631 };
632
633 EARLY_DRIVER_MODULE(axidma, simplebus, axidma_driver, 0, 0,
634 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_LATE);
635