1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright © 2023 Dmitry Salychev
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/mbuf.h>
32 #include <sys/malloc.h>
33 #include <sys/lock.h>
34 #include <sys/mutex.h>
35
36 #include <machine/bus.h>
37
38 #include "dpaa2_types.h"
39 #include "dpaa2_buf.h"
40 #include "dpaa2_bp.h"
41 #include "dpaa2_channel.h"
42 #include "dpaa2_swp.h"
43 #include "dpaa2_swp_if.h"
44 #include "dpaa2_ni.h"
45 #include "dpaa2_frame.h"
46
47 MALLOC_DEFINE(M_DPAA2_RXB, "dpaa2_rxb",
48 "DPAA2 DMA-mapped buffer (Rx)");
49 MALLOC_DEFINE(M_DPAA2_RXB_EXT, "dpaa2_rxb_ext",
50 "DPAA2 DMA-mapped buffer (Rx extension)");
51
52 /**
53 * @brief Allocate Rx buffers visible to QBMan and release them to the
54 * buffer pool.
55 */
56 int
dpaa2_buf_seed_pool(device_t dev,device_t bpdev,void * arg,uint32_t count,int size)57 dpaa2_buf_seed_pool(device_t dev, device_t bpdev, void *arg, uint32_t count,
58 int size)
59 {
60 struct dpaa2_ni_softc *sc = device_get_softc(dev);
61 struct dpaa2_bp_softc *bpsc = device_get_softc(bpdev);
62 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg;
63 struct dpaa2_buf *buf;
64 struct dpaa2_bufext_rx *bext;
65 const int alloc = DPAA2_ATOMIC_READ(&sc->buf_num);
66 const uint16_t bpid = bpsc->attr.bpid;
67 bus_dma_tag_t dmat;
68 bus_addr_t paddr[DPAA2_SWP_BUFS_PER_CMD];
69 int error, bufn = 0;
70
71 #ifdef _notyet_
72 /* Limit amount of buffers released to the pool */
73 count = (alloc + count > DPAA2_NI_BUFS_MAX)
74 ? DPAA2_NI_BUFS_MAX - alloc : count;
75 #endif
76
77 /* Release "count" buffers to the pool */
78 for (int i = alloc; i < alloc + count; i++) {
79 /* Enough buffers were allocated for a single command */
80 if (bufn == DPAA2_SWP_BUFS_PER_CMD) {
81 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr,
82 bufn);
83 if (error) {
84 device_printf(sc->dev, "%s: failed to release "
85 "buffers to the pool (1)\n", __func__);
86 return (error);
87 }
88 DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
89 bufn = 0;
90 }
91
92 buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_RXB, M_NOWAIT);
93 if (buf == NULL) {
94 device_printf(dev, "%s: Rx buf malloc() failed\n",
95 __func__);
96 return (ENOMEM);
97 }
98 bext = malloc(sizeof(struct dpaa2_bufext_rx), M_DPAA2_RXB_EXT,
99 M_NOWAIT);
100 if (bext == NULL) {
101 device_printf(dev, "%s: Rx bext malloc() failed\n",
102 __func__);
103 return (ENOMEM);
104 }
105 bext->ch = ch;
106 mtx_init(&bext->dma_mtx, "dpaa2_buf_dma_mtx", NULL,
107 MTX_DEF | MTX_NEW);
108 error = bus_dma_tag_create(
109 bus_get_dma_tag(dev), /* parent */
110 sc->buf_align, 0, /* alignment, boundary */
111 BUS_SPACE_MAXADDR, /* low restricted addr */
112 BUS_SPACE_MAXADDR, /* high restricted addr */
113 NULL, NULL, /* filter, filterarg */
114 RX_SEG_MAXSZ, /* maxsize */
115 RX_SEG_N, /* nsegments */
116 RX_SEG_SZ, /* maxsegsize */
117 0, /* flags */
118 NULL, /* lockfunc */
119 NULL, /* lockarg */
120 &dmat);
121 if (error) {
122 device_printf(dev, "%s: failed to create Rx buf dmat\n",
123 __func__);
124 return (error);
125 }
126 DPAA2_BUF_INIT_TAGOPT(buf, dmat, bext);
127
128 mtx_assert(&bext->dma_mtx, MA_NOTOWNED);
129 mtx_lock(&bext->dma_mtx);
130 error = dpaa2_buf_seed_rxb(dev, buf, size);
131 mtx_unlock(&bext->dma_mtx);
132 if (error != 0) {
133 device_printf(dev, "%s: dpaa2_buf_seed_rxb() failed: "
134 "error=%d/n", __func__, error);
135 break;
136 }
137 paddr[bufn] = buf->paddr;
138 bufn++;
139 }
140
141 /* Release reminder of the buffers to the pool */
142 if (bufn > 0) {
143 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr, bufn);
144 if (error) {
145 device_printf(sc->dev, "%s: failed to release "
146 "buffers to the pool (2)\n", __func__);
147 return (error);
148 }
149 DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
150 }
151
152 return (0);
153 }
154
155 /**
156 * @brief Prepare Rx buffer to be released to the buffer pool.
157 */
158 int
dpaa2_buf_seed_rxb(device_t dev,struct dpaa2_buf * buf,int size)159 dpaa2_buf_seed_rxb(device_t dev, struct dpaa2_buf *buf, int size)
160 {
161 struct dpaa2_ni_softc *sc = device_get_softc(dev);
162 struct dpaa2_swa *swa;
163 bool map_created = false;
164 bool mbuf_alloc = false;
165 int error;
166
167 #if defined(INVARIANTS)
168 DPAA2_BUF_ASSERT_RXPREP(buf);
169 struct dpaa2_bufext_rx *bext = (struct dpaa2_bufext_rx *)buf->opt;
170 mtx_assert(&bext->dma_mtx, MA_OWNED);
171 #endif /* INVARIANTS */
172
173 if (__predict_false(buf->dmap == NULL)) {
174 error = bus_dmamap_create(buf->dmat, 0, &buf->dmap);
175 if (error != 0) {
176 device_printf(dev, "%s: failed to create DMA map: "
177 "error=%d\n", __func__, error);
178 goto fail_map_create;
179 }
180 map_created = true;
181 }
182
183 if (__predict_true(buf->m == NULL)) {
184 buf->m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size);
185 if (__predict_false(buf->m == NULL)) {
186 device_printf(dev, "%s: m_getjcl(%d) failed\n",
187 __func__, size);
188 error = ENOMEM;
189 goto fail_mbuf_alloc;
190 }
191 buf->m->m_len = buf->m->m_ext.ext_size;
192 buf->m->m_pkthdr.len = buf->m->m_ext.ext_size;
193 mbuf_alloc = true;
194 }
195
196 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, buf->m, &buf->seg,
197 &buf->nseg, BUS_DMA_NOWAIT);
198 KASSERT(buf->nseg == 1, ("%s: one segment expected: nseg=%d", __func__,
199 buf->nseg));
200 KASSERT(error == 0, ("%s: bus_dmamap_load_mbuf_sg() failed: error=%d",
201 __func__, error));
202 if (__predict_false(error != 0 || buf->nseg != 1)) {
203 device_printf(sc->dev, "%s: bus_dmamap_load_mbuf_sg() failed: "
204 "error=%d, nsegs=%d\n", __func__, error, buf->nseg);
205 goto fail_mbuf_map;
206 }
207 buf->paddr = buf->seg.ds_addr;
208 buf->vaddr = buf->m->m_data;
209
210 /* Populate frame annotation for future use */
211 swa = (struct dpaa2_swa *)buf->vaddr;
212 swa->magic = DPAA2_MAGIC;
213 swa->buf = buf;
214
215 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREREAD);
216
217 DPAA2_BUF_ASSERT_RXREADY(buf);
218
219 return (0);
220
221 fail_mbuf_map:
222 if (mbuf_alloc) {
223 m_freem(buf->m);
224 buf->m = NULL;
225 }
226 fail_mbuf_alloc:
227 if (map_created) {
228 (void)bus_dmamap_destroy(buf->dmat, buf->dmap);
229 }
230 fail_map_create:
231 return (error);
232 }
233
234 /**
235 * @brief Prepare Tx buffer to be added to the Tx ring.
236 */
237 int
dpaa2_buf_seed_txb(device_t dev,struct dpaa2_buf * buf)238 dpaa2_buf_seed_txb(device_t dev, struct dpaa2_buf *buf)
239 {
240 struct dpaa2_buf *sgt = buf->sgt;
241 bool map_created = false;
242 int error;
243
244 DPAA2_BUF_ASSERT_TXPREP(buf);
245
246 if (buf->dmap == NULL) {
247 error = bus_dmamap_create(buf->dmat, 0, &buf->dmap);
248 if (error != 0) {
249 device_printf(dev, "%s: bus_dmamap_create() failed: "
250 "error=%d\n", __func__, error);
251 goto fail_map_create;
252 }
253 map_created = true;
254 }
255
256 if (sgt->vaddr == NULL) {
257 error = bus_dmamem_alloc(sgt->dmat, (void **)&sgt->vaddr,
258 BUS_DMA_ZERO | BUS_DMA_COHERENT, &sgt->dmap);
259 if (error != 0) {
260 device_printf(dev, "%s: bus_dmamem_alloc() failed: "
261 "error=%d\n", __func__, error);
262 goto fail_mem_alloc;
263 }
264 }
265
266 DPAA2_BUF_ASSERT_TXREADY(buf);
267
268 return (0);
269
270 fail_mem_alloc:
271 if (map_created) {
272 (void)bus_dmamap_destroy(buf->dmat, buf->dmap);
273 }
274 fail_map_create:
275 return (error);
276 }
277