1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2023 Chelsio Communications, Inc. 5 * Written by: John Baldwin <jhb@FreeBSD.org> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "common/common.h" 30 31 /* 32 * Support routines to manage TPT entries used for both RDMA and NVMe 33 * offloads. This includes allocating STAG indices and managing the 34 * PBL pool. 35 */ 36 37 #define T4_ULPTX_MIN_IO 32 38 #define T4_MAX_INLINE_SIZE 96 39 #define T4_ULPTX_MAX_DMA 1024 40 41 /* PBL and STAG Memory Managers. */ 42 43 #define MIN_PBL_SHIFT 5 /* 32B == min PBL size (4 entries) */ 44 45 uint32_t 46 t4_pblpool_alloc(struct adapter *sc, int size) 47 { 48 vmem_addr_t addr; 49 50 if (vmem_xalloc(sc->pbl_arena, roundup(size, (1 << MIN_PBL_SHIFT)), 51 4, 0, 0, VMEM_ADDR_MIN, VMEM_ADDR_MAX, M_FIRSTFIT | M_NOWAIT, 52 &addr) != 0) 53 return (0); 54 #ifdef VERBOSE_TRACES 55 CTR(KTR_CXGBE, "%s: addr 0x%lx size %d", __func__, addr, size); 56 #endif 57 return (addr); 58 } 59 60 void 61 t4_pblpool_free(struct adapter *sc, uint32_t addr, int size) 62 { 63 #ifdef VERBOSE_TRACES 64 CTR(KTR_CXGBE, "%s: addr 0x%x size %d", __func__, addr, size); 65 #endif 66 vmem_xfree(sc->pbl_arena, addr, roundup(size, (1 << MIN_PBL_SHIFT))); 67 } 68 69 uint32_t 70 t4_stag_alloc(struct adapter *sc, int size) 71 { 72 vmem_addr_t stag_idx; 73 74 if (vmem_alloc(sc->stag_arena, size, M_FIRSTFIT | M_NOWAIT, 75 &stag_idx) != 0) 76 return (T4_STAG_UNSET); 77 #ifdef VERBOSE_TRACES 78 CTR(KTR_CXGBE, "%s: idx 0x%lx size %d", __func__, stag_idx, size); 79 #endif 80 return (stag_idx); 81 } 82 83 void 84 t4_stag_free(struct adapter *sc, uint32_t stag_idx, int size) 85 { 86 #ifdef VERBOSE_TRACES 87 CTR(KTR_CXGBE, "%s: idx 0x%x size %d", __func__, stag_idx, size); 88 #endif 89 vmem_free(sc->stag_arena, stag_idx, size); 90 } 91 92 void 93 t4_init_tpt(struct adapter *sc) 94 { 95 if (sc->vres.pbl.size != 0) 96 sc->pbl_arena = vmem_create("PBL_MEM_POOL", sc->vres.pbl.start, 97 sc->vres.pbl.size, 1, 0, M_FIRSTFIT | M_WAITOK); 98 if (sc->vres.stag.size != 0) 99 sc->stag_arena = vmem_create("STAG", 1, 100 sc->vres.stag.size >> 5, 1, 0, M_FIRSTFIT | M_WAITOK); 101 } 102 103 void 104 t4_free_tpt(struct adapter *sc) 105 { 106 if (sc->pbl_arena != NULL) 107 vmem_destroy(sc->pbl_arena); 108 if (sc->stag_arena != NULL) 109 vmem_destroy(sc->stag_arena); 110 } 111 112 /* 113 * TPT support routines. TPT entries are stored in the STAG adapter 114 * memory region and are written to via ULP_TX_MEM_WRITE commands in 115 * FW_ULPTX_WR work requests. 116 */ 117 118 void 119 t4_write_mem_dma_wr(struct adapter *sc, void *wr, int wr_len, int tid, 120 uint32_t addr, uint32_t len, vm_paddr_t data, uint64_t cookie) 121 { 122 struct ulp_mem_io *ulpmc; 123 struct ulptx_sgl *sgl; 124 125 MPASS(wr_len == T4_WRITE_MEM_DMA_LEN); 126 127 addr &= 0x7FFFFFF; 128 129 memset(wr, 0, wr_len); 130 ulpmc = wr; 131 INIT_ULPTX_WR(ulpmc, wr_len, 0, tid); 132 if (cookie != 0) { 133 ulpmc->wr.wr_hi |= htobe32(F_FW_WR_COMPL); 134 ulpmc->wr.wr_lo = cookie; 135 } 136 ulpmc->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 137 V_T5_ULP_MEMIO_ORDER(1) | 138 V_T5_ULP_MEMIO_FID(sc->sge.ofld_rxq[0].iq.abs_id)); 139 if (chip_id(sc) >= CHELSIO_T7) 140 ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN(len >> 5)); 141 else 142 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(len >> 5)); 143 ulpmc->len16 = htobe32((tid << 8) | 144 DIV_ROUND_UP(wr_len - sizeof(ulpmc->wr), 16)); 145 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(addr)); 146 147 sgl = (struct ulptx_sgl *)(ulpmc + 1); 148 sgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) | V_ULPTX_NSGE(1)); 149 sgl->len0 = htobe32(len); 150 sgl->addr0 = htobe64(data); 151 } 152 153 void 154 t4_write_mem_inline_wr(struct adapter *sc, void *wr, int wr_len, int tid, 155 uint32_t addr, uint32_t len, void *data, uint64_t cookie) 156 { 157 struct ulp_mem_io *ulpmc; 158 struct ulptx_idata *ulpsc; 159 160 MPASS(len > 0 && len <= T4_MAX_INLINE_SIZE); 161 MPASS(wr_len == T4_WRITE_MEM_INLINE_LEN(len)); 162 163 addr &= 0x7FFFFFF; 164 165 memset(wr, 0, wr_len); 166 ulpmc = wr; 167 INIT_ULPTX_WR(ulpmc, wr_len, 0, tid); 168 169 if (cookie != 0) { 170 ulpmc->wr.wr_hi |= htobe32(F_FW_WR_COMPL); 171 ulpmc->wr.wr_lo = cookie; 172 } 173 174 ulpmc->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) | 175 F_T5_ULP_MEMIO_IMM); 176 177 if (chip_id(sc) >= CHELSIO_T7) 178 ulpmc->dlen = htobe32(V_T7_ULP_MEMIO_DATA_LEN( 179 DIV_ROUND_UP(len, T4_ULPTX_MIN_IO))); 180 else 181 ulpmc->dlen = htobe32(V_ULP_MEMIO_DATA_LEN( 182 DIV_ROUND_UP(len, T4_ULPTX_MIN_IO))); 183 ulpmc->len16 = htobe32((tid << 8) | 184 DIV_ROUND_UP(wr_len - sizeof(ulpmc->wr), 16)); 185 ulpmc->lock_addr = htobe32(V_ULP_MEMIO_ADDR(addr)); 186 187 ulpsc = (struct ulptx_idata *)(ulpmc + 1); 188 ulpsc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM)); 189 ulpsc->len = htobe32(roundup(len, T4_ULPTX_MIN_IO)); 190 191 if (data != NULL) 192 memcpy(ulpsc + 1, data, len); 193 } 194