1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright © 2023 Dmitry Salychev
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/mbuf.h>
32 #include <sys/malloc.h>
33 #include <sys/lock.h>
34 #include <sys/mutex.h>
35
36 #include <machine/bus.h>
37
38 #include "dpaa2_types.h"
39 #include "dpaa2_buf.h"
40 #include "dpaa2_bp.h"
41 #include "dpaa2_channel.h"
42 #include "dpaa2_swp.h"
43 #include "dpaa2_swp_if.h"
44 #include "dpaa2_ni.h"
45
46 MALLOC_DEFINE(M_DPAA2_RXB, "dpaa2_rxb", "DPAA2 DMA-mapped buffer (Rx)");
47
48 /**
49 * @brief Allocate Rx buffers visible to QBMan and release them to the
50 * buffer pool.
51 */
52 int
dpaa2_buf_seed_pool(device_t dev,device_t bpdev,void * arg,uint32_t count,int size,struct mtx * dma_mtx)53 dpaa2_buf_seed_pool(device_t dev, device_t bpdev, void *arg, uint32_t count,
54 int size, struct mtx *dma_mtx)
55 {
56 struct dpaa2_ni_softc *sc = device_get_softc(dev);
57 struct dpaa2_bp_softc *bpsc = device_get_softc(bpdev);
58 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg;
59 struct dpaa2_buf *buf;
60 const int alloc = DPAA2_ATOMIC_READ(&sc->buf_num);
61 const uint16_t bpid = bpsc->attr.bpid;
62 bus_addr_t paddr[DPAA2_SWP_BUFS_PER_CMD];
63 int error, bufn = 0;
64
65 #if defined(INVARIANTS)
66 KASSERT(ch->rx_dmat != NULL, ("%s: no DMA tag?", __func__));
67 if (dma_mtx != NULL) {
68 mtx_assert(dma_mtx, MA_OWNED);
69 }
70 #endif /* INVARIANTS */
71
72 #ifdef _notyet_
73 /* Limit amount of buffers released to the pool */
74 count = (alloc + count > DPAA2_NI_BUFS_MAX)
75 ? DPAA2_NI_BUFS_MAX - alloc : count;
76 #endif
77
78 /* Release "count" buffers to the pool */
79 for (int i = alloc; i < alloc + count; i++) {
80 /* Enough buffers were allocated for a single command */
81 if (bufn == DPAA2_SWP_BUFS_PER_CMD) {
82 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr,
83 bufn);
84 if (error) {
85 device_printf(sc->dev, "%s: failed to release "
86 "buffers to the pool (1)\n", __func__);
87 return (error);
88 }
89 DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
90 bufn = 0;
91 }
92
93 buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_RXB, M_NOWAIT);
94 if (buf == NULL) {
95 device_printf(dev, "%s: malloc() failed\n", __func__);
96 return (ENOMEM);
97 }
98 DPAA2_BUF_INIT_TAGOPT(buf, ch->rx_dmat, ch);
99
100 error = dpaa2_buf_seed_rxb(dev, buf, size, dma_mtx);
101 if (error != 0) {
102 device_printf(dev, "%s: dpaa2_buf_seed_rxb() failed: "
103 "error=%d/n", __func__, error);
104 break;
105 }
106 paddr[bufn] = buf->paddr;
107 bufn++;
108 }
109
110 /* Release reminder of the buffers to the pool */
111 if (bufn > 0) {
112 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr, bufn);
113 if (error) {
114 device_printf(sc->dev, "%s: failed to release "
115 "buffers to the pool (2)\n", __func__);
116 return (error);
117 }
118 DPAA2_ATOMIC_ADD(&sc->buf_num, bufn);
119 }
120
121 return (0);
122 }
123
124 /**
125 * @brief Prepare Rx buffer to be released to the buffer pool.
126 */
127 int
dpaa2_buf_seed_rxb(device_t dev,struct dpaa2_buf * buf,int size,struct mtx * dma_mtx)128 dpaa2_buf_seed_rxb(device_t dev, struct dpaa2_buf *buf, int size,
129 struct mtx *dma_mtx)
130 {
131 struct dpaa2_ni_softc *sc = device_get_softc(dev);
132 struct dpaa2_fa *fa;
133 bool map_created = false;
134 bool mbuf_alloc = false;
135 int error;
136
137 #if defined(INVARIANTS)
138 DPAA2_BUF_ASSERT_RXPREP(buf);
139 if (dma_mtx != NULL) {
140 mtx_assert(dma_mtx, MA_OWNED);
141 }
142 #endif /* INVARIANTS */
143
144 if (__predict_false(buf->dmap == NULL)) {
145 error = bus_dmamap_create(buf->dmat, 0, &buf->dmap);
146 if (error != 0) {
147 device_printf(dev, "%s: failed to create DMA map: "
148 "error=%d\n", __func__, error);
149 goto fail_map_create;
150 }
151 map_created = true;
152 }
153
154 if (__predict_true(buf->m == NULL)) {
155 buf->m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size);
156 if (__predict_false(buf->m == NULL)) {
157 device_printf(dev, "%s: m_getjcl() failed\n", __func__);
158 error = ENOMEM;
159 goto fail_mbuf_alloc;
160 }
161 buf->m->m_len = buf->m->m_ext.ext_size;
162 buf->m->m_pkthdr.len = buf->m->m_ext.ext_size;
163 mbuf_alloc = true;
164 }
165
166 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, buf->m, &buf->seg,
167 &buf->nseg, BUS_DMA_NOWAIT);
168 KASSERT(buf->nseg == 1, ("%s: one segment expected: nseg=%d", __func__,
169 buf->nseg));
170 KASSERT(error == 0, ("%s: bus_dmamap_load_mbuf_sg() failed: error=%d",
171 __func__, error));
172 if (__predict_false(error != 0 || buf->nseg != 1)) {
173 device_printf(sc->dev, "%s: bus_dmamap_load_mbuf_sg() failed: "
174 "error=%d, nsegs=%d\n", __func__, error, buf->nseg);
175 goto fail_mbuf_map;
176 }
177 buf->paddr = buf->seg.ds_addr;
178 buf->vaddr = buf->m->m_data;
179
180 /* Populate frame annotation for future use */
181 fa = (struct dpaa2_fa *)buf->vaddr;
182 fa->magic = DPAA2_MAGIC;
183 fa->buf = buf;
184
185 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREREAD);
186
187 DPAA2_BUF_ASSERT_RXREADY(buf);
188
189 return (0);
190
191 fail_mbuf_map:
192 if (mbuf_alloc) {
193 m_freem(buf->m);
194 buf->m = NULL;
195 }
196 fail_mbuf_alloc:
197 if (map_created) {
198 (void)bus_dmamap_destroy(buf->dmat, buf->dmap);
199 }
200 fail_map_create:
201 return (error);
202 }
203
204 /**
205 * @brief Prepare Tx buffer to be added to the Tx ring.
206 */
207 int
dpaa2_buf_seed_txb(device_t dev,struct dpaa2_buf * buf)208 dpaa2_buf_seed_txb(device_t dev, struct dpaa2_buf *buf)
209 {
210 struct dpaa2_buf *sgt = buf->sgt;
211 bool map_created = false;
212 int error;
213
214 DPAA2_BUF_ASSERT_TXPREP(buf);
215
216 if (buf->dmap == NULL) {
217 error = bus_dmamap_create(buf->dmat, 0, &buf->dmap);
218 if (error != 0) {
219 device_printf(dev, "%s: bus_dmamap_create() failed: "
220 "error=%d\n", __func__, error);
221 goto fail_map_create;
222 }
223 map_created = true;
224 }
225
226 if (sgt->vaddr == NULL) {
227 error = bus_dmamem_alloc(sgt->dmat, (void **)&sgt->vaddr,
228 BUS_DMA_ZERO | BUS_DMA_COHERENT, &sgt->dmap);
229 if (error != 0) {
230 device_printf(dev, "%s: bus_dmamem_alloc() failed: "
231 "error=%d\n", __func__, error);
232 goto fail_mem_alloc;
233 }
234 }
235
236 DPAA2_BUF_ASSERT_TXREADY(buf);
237
238 return (0);
239
240 fail_mem_alloc:
241 if (map_created) {
242 (void)bus_dmamap_destroy(buf->dmat, buf->dmap);
243 }
244 fail_map_create:
245 return (error);
246 }
247