1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright © 2023 Dmitry Salychev 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/types.h> 29 #include <sys/param.h> 30 #include <sys/systm.h> 31 #include <sys/mbuf.h> 32 #include <sys/malloc.h> 33 #include <sys/lock.h> 34 #include <sys/mutex.h> 35 36 #include <machine/bus.h> 37 38 #include "dpaa2_types.h" 39 #include "dpaa2_buf.h" 40 #include "dpaa2_bp.h" 41 #include "dpaa2_channel.h" 42 #include "dpaa2_swp.h" 43 #include "dpaa2_swp_if.h" 44 #include "dpaa2_ni.h" 45 #include "dpaa2_frame.h" 46 47 MALLOC_DEFINE(M_DPAA2_RXB, "dpaa2_rxb", 48 "DPAA2 DMA-mapped buffer (Rx)"); 49 MALLOC_DEFINE(M_DPAA2_RXB_EXT, "dpaa2_rxb_ext", 50 "DPAA2 DMA-mapped buffer (Rx extension)"); 51 52 /** 53 * @brief Allocate Rx buffers visible to QBMan and release them to the 54 * buffer pool. 55 */ 56 int 57 dpaa2_buf_seed_pool(device_t dev, device_t bpdev, void *arg, uint32_t count, 58 int size) 59 { 60 struct dpaa2_ni_softc *sc = device_get_softc(dev); 61 struct dpaa2_bp_softc *bpsc = device_get_softc(bpdev); 62 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg; 63 struct dpaa2_buf *buf; 64 struct dpaa2_bufext_rx *bext; 65 const int alloc = DPAA2_ATOMIC_READ(&sc->buf_num); 66 const uint16_t bpid = bpsc->attr.bpid; 67 bus_dma_tag_t dmat; 68 bus_addr_t paddr[DPAA2_SWP_BUFS_PER_CMD]; 69 int error, bufn = 0; 70 71 #ifdef _notyet_ 72 /* Limit amount of buffers released to the pool */ 73 count = (alloc + count > DPAA2_NI_BUFS_MAX) 74 ? DPAA2_NI_BUFS_MAX - alloc : count; 75 #endif 76 77 /* Release "count" buffers to the pool */ 78 for (int i = alloc; i < alloc + count; i++) { 79 /* Enough buffers were allocated for a single command */ 80 if (bufn == DPAA2_SWP_BUFS_PER_CMD) { 81 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr, 82 bufn); 83 if (error) { 84 device_printf(sc->dev, "%s: failed to release " 85 "buffers to the pool (1)\n", __func__); 86 return (error); 87 } 88 DPAA2_ATOMIC_ADD(&sc->buf_num, bufn); 89 bufn = 0; 90 } 91 92 buf = malloc(sizeof(struct dpaa2_buf), M_DPAA2_RXB, M_NOWAIT); 93 if (buf == NULL) { 94 device_printf(dev, "%s: Rx buf malloc() failed\n", 95 __func__); 96 return (ENOMEM); 97 } 98 bext = malloc(sizeof(struct dpaa2_bufext_rx), M_DPAA2_RXB_EXT, 99 M_NOWAIT); 100 if (bext == NULL) { 101 device_printf(dev, "%s: Rx bext malloc() failed\n", 102 __func__); 103 return (ENOMEM); 104 } 105 bext->ch = ch; 106 mtx_init(&bext->dma_mtx, "dpaa2_buf_dma_mtx", NULL, 107 MTX_DEF | MTX_NEW); 108 error = bus_dma_tag_create( 109 bus_get_dma_tag(dev), /* parent */ 110 sc->buf_align, 0, /* alignment, boundary */ 111 BUS_SPACE_MAXADDR, /* low restricted addr */ 112 BUS_SPACE_MAXADDR, /* high restricted addr */ 113 NULL, NULL, /* filter, filterarg */ 114 RX_SEG_MAXSZ, /* maxsize */ 115 RX_SEG_N, /* nsegments */ 116 RX_SEG_SZ, /* maxsegsize */ 117 0, /* flags */ 118 NULL, /* lockfunc */ 119 NULL, /* lockarg */ 120 &dmat); 121 if (error) { 122 device_printf(dev, "%s: failed to create Rx buf dmat\n", 123 __func__); 124 return (error); 125 } 126 DPAA2_BUF_INIT_TAGOPT(buf, dmat, bext); 127 128 mtx_assert(&bext->dma_mtx, MA_NOTOWNED); 129 mtx_lock(&bext->dma_mtx); 130 error = dpaa2_buf_seed_rxb(dev, buf, size); 131 mtx_unlock(&bext->dma_mtx); 132 if (error != 0) { 133 device_printf(dev, "%s: dpaa2_buf_seed_rxb() failed: " 134 "error=%d/n", __func__, error); 135 break; 136 } 137 paddr[bufn] = buf->paddr; 138 bufn++; 139 } 140 141 /* Release reminder of the buffers to the pool */ 142 if (bufn > 0) { 143 error = DPAA2_SWP_RELEASE_BUFS(ch->io_dev, bpid, paddr, bufn); 144 if (error) { 145 device_printf(sc->dev, "%s: failed to release " 146 "buffers to the pool (2)\n", __func__); 147 return (error); 148 } 149 DPAA2_ATOMIC_ADD(&sc->buf_num, bufn); 150 } 151 152 return (0); 153 } 154 155 /** 156 * @brief Prepare Rx buffer to be released to the buffer pool. 157 */ 158 int 159 dpaa2_buf_seed_rxb(device_t dev, struct dpaa2_buf *buf, int size) 160 { 161 struct dpaa2_ni_softc *sc = device_get_softc(dev); 162 struct dpaa2_swa *swa; 163 bool map_created = false; 164 bool mbuf_alloc = false; 165 int error; 166 167 #if defined(INVARIANTS) 168 DPAA2_BUF_ASSERT_RXPREP(buf); 169 struct dpaa2_bufext_rx *bext = (struct dpaa2_bufext_rx *)buf->opt; 170 mtx_assert(&bext->dma_mtx, MA_OWNED); 171 #endif /* INVARIANTS */ 172 173 if (__predict_false(buf->dmap == NULL)) { 174 error = bus_dmamap_create(buf->dmat, 0, &buf->dmap); 175 if (error != 0) { 176 device_printf(dev, "%s: failed to create DMA map: " 177 "error=%d\n", __func__, error); 178 goto fail_map_create; 179 } 180 map_created = true; 181 } 182 183 if (__predict_true(buf->m == NULL)) { 184 buf->m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, size); 185 if (__predict_false(buf->m == NULL)) { 186 device_printf(dev, "%s: m_getjcl(%d) failed\n", 187 __func__, size); 188 error = ENOMEM; 189 goto fail_mbuf_alloc; 190 } 191 buf->m->m_len = buf->m->m_ext.ext_size; 192 buf->m->m_pkthdr.len = buf->m->m_ext.ext_size; 193 mbuf_alloc = true; 194 } 195 196 error = bus_dmamap_load_mbuf_sg(buf->dmat, buf->dmap, buf->m, &buf->seg, 197 &buf->nseg, BUS_DMA_NOWAIT); 198 KASSERT(buf->nseg == 1, ("%s: one segment expected: nseg=%d", __func__, 199 buf->nseg)); 200 KASSERT(error == 0, ("%s: bus_dmamap_load_mbuf_sg() failed: error=%d", 201 __func__, error)); 202 if (__predict_false(error != 0 || buf->nseg != 1)) { 203 device_printf(sc->dev, "%s: bus_dmamap_load_mbuf_sg() failed: " 204 "error=%d, nsegs=%d\n", __func__, error, buf->nseg); 205 goto fail_mbuf_map; 206 } 207 buf->paddr = buf->seg.ds_addr; 208 buf->vaddr = buf->m->m_data; 209 210 /* Populate frame annotation for future use */ 211 swa = (struct dpaa2_swa *)buf->vaddr; 212 swa->magic = DPAA2_MAGIC; 213 swa->buf = buf; 214 215 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_PREREAD); 216 217 DPAA2_BUF_ASSERT_RXREADY(buf); 218 219 return (0); 220 221 fail_mbuf_map: 222 if (mbuf_alloc) { 223 m_freem(buf->m); 224 buf->m = NULL; 225 } 226 fail_mbuf_alloc: 227 if (map_created) { 228 (void)bus_dmamap_destroy(buf->dmat, buf->dmap); 229 } 230 fail_map_create: 231 return (error); 232 } 233 234 /** 235 * @brief Prepare Tx buffer to be added to the Tx ring. 236 */ 237 int 238 dpaa2_buf_seed_txb(device_t dev, struct dpaa2_buf *buf) 239 { 240 struct dpaa2_buf *sgt = buf->sgt; 241 bool map_created = false; 242 int error; 243 244 DPAA2_BUF_ASSERT_TXPREP(buf); 245 246 if (buf->dmap == NULL) { 247 error = bus_dmamap_create(buf->dmat, 0, &buf->dmap); 248 if (error != 0) { 249 device_printf(dev, "%s: bus_dmamap_create() failed: " 250 "error=%d\n", __func__, error); 251 goto fail_map_create; 252 } 253 map_created = true; 254 } 255 256 if (sgt->vaddr == NULL) { 257 error = bus_dmamem_alloc(sgt->dmat, (void **)&sgt->vaddr, 258 BUS_DMA_ZERO | BUS_DMA_COHERENT, &sgt->dmap); 259 if (error != 0) { 260 device_printf(dev, "%s: bus_dmamem_alloc() failed: " 261 "error=%d\n", __func__, error); 262 goto fail_mem_alloc; 263 } 264 } 265 266 DPAA2_BUF_ASSERT_TXREADY(buf); 267 268 return (0); 269 270 fail_mem_alloc: 271 if (map_created) { 272 (void)bus_dmamap_destroy(buf->dmat, buf->dmap); 273 } 274 fail_map_create: 275 return (error); 276 } 277