1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2023 Google LLC 5 * 6 * Redistribution and use in source and binary forms, with or without modification, 7 * are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, this 10 * list of conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * 3. Neither the name of the copyright holder nor the names of its contributors 17 * may be used to endorse or promote products derived from this software without 18 * specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #include <sys/malloc.h> 32 33 #include "gve.h" 34 #include "gve_adminq.h" 35 36 static MALLOC_DEFINE(M_GVE_QPL, "gve qpl", "gve qpl allocations"); 37 38 static uint32_t 39 gve_num_tx_qpls(struct gve_priv *priv) 40 { 41 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 42 return (0); 43 44 return (priv->tx_cfg.max_queues); 45 } 46 47 static uint32_t 48 gve_num_rx_qpls(struct gve_priv *priv) 49 { 50 if (priv->queue_format != GVE_GQI_QPL_FORMAT) 51 return (0); 52 53 return (priv->rx_cfg.max_queues); 54 } 55 56 static void 57 gve_free_qpl(struct gve_priv *priv, uint32_t id) 58 { 59 struct gve_queue_page_list *qpl = &priv->qpls[id]; 60 int i; 61 62 for (i = 0; i < qpl->num_dmas; i++) { 63 gve_dmamap_destroy(&qpl->dmas[i]); 64 } 65 66 if (qpl->kva) { 67 pmap_qremove(qpl->kva, qpl->num_pages); 68 kva_free(qpl->kva, PAGE_SIZE * qpl->num_pages); 69 } 70 71 for (i = 0; i < qpl->num_pages; i++) { 72 /* 73 * Free the page only if this is the last ref. 74 * Tx pages are known to have no other refs at 75 * this point, but Rx pages might still be in 76 * use by the networking stack, see gve_mextadd_free. 77 */ 78 if (vm_page_unwire_noq(qpl->pages[i])) { 79 if (!qpl->kva) { 80 pmap_qremove((vm_offset_t)qpl->dmas[i].cpu_addr, 1); 81 kva_free((vm_offset_t)qpl->dmas[i].cpu_addr, PAGE_SIZE); 82 } 83 vm_page_free(qpl->pages[i]); 84 } 85 86 priv->num_registered_pages--; 87 } 88 89 if (qpl->pages != NULL) 90 free(qpl->pages, M_GVE_QPL); 91 92 if (qpl->dmas != NULL) 93 free(qpl->dmas, M_GVE_QPL); 94 } 95 96 static int 97 gve_alloc_qpl(struct gve_priv *priv, uint32_t id, int npages, bool single_kva) 98 { 99 struct gve_queue_page_list *qpl = &priv->qpls[id]; 100 int err; 101 int i; 102 103 if (npages + priv->num_registered_pages > priv->max_registered_pages) { 104 device_printf(priv->dev, "Reached max number of registered pages %ju > %ju\n", 105 (uintmax_t)npages + priv->num_registered_pages, 106 (uintmax_t)priv->max_registered_pages); 107 return (EINVAL); 108 } 109 110 qpl->id = id; 111 qpl->num_pages = 0; 112 qpl->num_dmas = 0; 113 114 qpl->dmas = malloc(npages * sizeof(*qpl->dmas), M_GVE_QPL, 115 M_WAITOK | M_ZERO); 116 117 qpl->pages = malloc(npages * sizeof(*qpl->pages), M_GVE_QPL, 118 M_WAITOK | M_ZERO); 119 120 qpl->kva = 0; 121 if (single_kva) { 122 qpl->kva = kva_alloc(PAGE_SIZE * npages); 123 if (!qpl->kva) { 124 device_printf(priv->dev, "Failed to create the single kva for QPL %d\n", id); 125 err = ENOMEM; 126 goto abort; 127 } 128 } 129 130 for (i = 0; i < npages; i++) { 131 qpl->pages[i] = vm_page_alloc_noobj(VM_ALLOC_WIRED | 132 VM_ALLOC_WAITOK | 133 VM_ALLOC_ZERO); 134 135 if (!single_kva) { 136 qpl->dmas[i].cpu_addr = (void *)kva_alloc(PAGE_SIZE); 137 if (!qpl->dmas[i].cpu_addr) { 138 device_printf(priv->dev, "Failed to create kva for page %d in QPL %d", i, id); 139 err = ENOMEM; 140 goto abort; 141 } 142 pmap_qenter((vm_offset_t)qpl->dmas[i].cpu_addr, &(qpl->pages[i]), 1); 143 } else 144 qpl->dmas[i].cpu_addr = (void *)(qpl->kva + (PAGE_SIZE * i)); 145 146 147 qpl->num_pages++; 148 } 149 150 if (single_kva) 151 pmap_qenter(qpl->kva, qpl->pages, npages); 152 153 for (i = 0; i < npages; i++) { 154 err = gve_dmamap_create(priv, /*size=*/PAGE_SIZE, /*align=*/PAGE_SIZE, 155 &qpl->dmas[i]); 156 if (err != 0) { 157 device_printf(priv->dev, "Failed to dma-map page %d in QPL %d\n", i, id); 158 goto abort; 159 } 160 161 qpl->num_dmas++; 162 priv->num_registered_pages++; 163 } 164 165 return (0); 166 167 abort: 168 gve_free_qpl(priv, id); 169 return (err); 170 } 171 172 void 173 gve_free_qpls(struct gve_priv *priv) 174 { 175 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 176 int i; 177 178 if (num_qpls == 0) 179 return; 180 181 if (priv->qpls != NULL) { 182 for (i = 0; i < num_qpls; i++) 183 gve_free_qpl(priv, i); 184 free(priv->qpls, M_GVE_QPL); 185 } 186 } 187 188 int gve_alloc_qpls(struct gve_priv *priv) 189 { 190 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 191 int err; 192 int i; 193 194 if (num_qpls == 0) 195 return (0); 196 197 priv->qpls = malloc(num_qpls * sizeof(*priv->qpls), M_GVE_QPL, 198 M_WAITOK | M_ZERO); 199 200 for (i = 0; i < gve_num_tx_qpls(priv); i++) { 201 err = gve_alloc_qpl(priv, i, priv->tx_desc_cnt / GVE_QPL_DIVISOR, 202 /*single_kva=*/true); 203 if (err != 0) 204 goto abort; 205 } 206 207 for (; i < num_qpls; i++) { 208 err = gve_alloc_qpl(priv, i, priv->rx_desc_cnt, /*single_kva=*/false); 209 if (err != 0) 210 goto abort; 211 } 212 213 return (0); 214 215 abort: 216 gve_free_qpls(priv); 217 return (err); 218 } 219 220 static int 221 gve_unregister_n_qpls(struct gve_priv *priv, int n) 222 { 223 int err; 224 int i; 225 226 for (i = 0; i < n; i++) { 227 err = gve_adminq_unregister_page_list(priv, priv->qpls[i].id); 228 if (err != 0) { 229 device_printf(priv->dev, 230 "Failed to unregister qpl %d, err: %d\n", 231 priv->qpls[i].id, err); 232 } 233 } 234 235 if (err != 0) 236 return (err); 237 238 return (0); 239 } 240 241 int 242 gve_register_qpls(struct gve_priv *priv) 243 { 244 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 245 int err; 246 int i; 247 248 if (gve_get_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK)) 249 return (0); 250 251 for (i = 0; i < num_qpls; i++) { 252 err = gve_adminq_register_page_list(priv, &priv->qpls[i]); 253 if (err != 0) { 254 device_printf(priv->dev, 255 "Failed to register qpl %d, err: %d\n", 256 priv->qpls[i].id, err); 257 goto abort; 258 } 259 } 260 261 gve_set_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK); 262 return (0); 263 264 abort: 265 gve_unregister_n_qpls(priv, i); 266 return (err); 267 } 268 269 int 270 gve_unregister_qpls(struct gve_priv *priv) 271 { 272 int num_qpls = gve_num_tx_qpls(priv) + gve_num_rx_qpls(priv); 273 int err; 274 275 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK)) 276 return (0); 277 278 err = gve_unregister_n_qpls(priv, num_qpls); 279 if (err != 0) 280 return (err); 281 282 gve_clear_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK); 283 return (0); 284 } 285