1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2023 Google LLC 5 * 6 * Redistribution and use in source and binary forms, with or without modification, 7 * are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, this 10 * list of conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * 3. Neither the name of the copyright holder nor the names of its contributors 17 * may be used to endorse or promote products derived from this software without 18 * specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #include "gve.h" 32 33 uint32_t 34 gve_reg_bar_read_4(struct gve_priv *priv, bus_size_t offset) 35 { 36 return (be32toh(bus_read_4(priv->reg_bar, offset))); 37 } 38 39 void 40 gve_reg_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val) 41 { 42 bus_write_4(priv->reg_bar, offset, htobe32(val)); 43 } 44 45 void 46 gve_db_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val) 47 { 48 bus_write_4(priv->db_bar, offset, htobe32(val)); 49 } 50 51 void 52 gve_alloc_counters(counter_u64_t *stat, int num_stats) 53 { 54 int i; 55 56 for (i = 0; i < num_stats; i++) 57 stat[i] = counter_u64_alloc(M_WAITOK); 58 } 59 60 void 61 gve_free_counters(counter_u64_t *stat, int num_stats) 62 { 63 int i; 64 65 for (i = 0; i < num_stats; i++) 66 counter_u64_free(stat[i]); 67 } 68 69 /* Currently assumes a single segment. */ 70 static void 71 gve_dmamap_load_callback(void *arg, bus_dma_segment_t *segs, int nseg, 72 int error) 73 { 74 if (error == 0) 75 *(bus_addr_t *) arg = segs[0].ds_addr; 76 } 77 78 int 79 gve_dma_alloc_coherent(struct gve_priv *priv, int size, int align, 80 struct gve_dma_handle *dma) 81 { 82 int err; 83 device_t dev = priv->dev; 84 85 err = bus_dma_tag_create( 86 bus_get_dma_tag(dev), /* parent */ 87 align, 0, /* alignment, bounds */ 88 BUS_SPACE_MAXADDR, /* lowaddr */ 89 BUS_SPACE_MAXADDR, /* highaddr */ 90 NULL, NULL, /* filter, filterarg */ 91 size, /* maxsize */ 92 1, /* nsegments */ 93 size, /* maxsegsize */ 94 BUS_DMA_ALLOCNOW, /* flags */ 95 NULL, /* lockfunc */ 96 NULL, /* lockarg */ 97 &dma->tag); 98 if (err != 0) { 99 device_printf(dev, "%s: bus_dma_tag_create failed: %d\n", 100 __func__, err); 101 goto clear_tag; 102 } 103 104 err = bus_dmamem_alloc(dma->tag, (void **) &dma->cpu_addr, 105 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, 106 &dma->map); 107 if (err != 0) { 108 device_printf(dev, "%s: bus_dmamem_alloc(%ju) failed: %d\n", 109 __func__, (uintmax_t)size, err); 110 goto destroy_tag; 111 } 112 113 /* An address set by the callback will never be -1 */ 114 dma->bus_addr = (bus_addr_t)-1; 115 err = bus_dmamap_load(dma->tag, dma->map, dma->cpu_addr, size, 116 gve_dmamap_load_callback, &dma->bus_addr, BUS_DMA_NOWAIT); 117 if (err != 0 || dma->bus_addr == (bus_addr_t)-1) { 118 device_printf(dev, "%s: bus_dmamap_load failed: %d\n", __func__, err); 119 goto free_mem; 120 } 121 122 return (0); 123 124 free_mem: 125 bus_dmamem_free(dma->tag, dma->cpu_addr, dma->map); 126 destroy_tag: 127 bus_dma_tag_destroy(dma->tag); 128 clear_tag: 129 dma->tag = NULL; 130 131 return (err); 132 } 133 134 void 135 gve_dma_free_coherent(struct gve_dma_handle *dma) 136 { 137 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 138 bus_dmamap_unload(dma->tag, dma->map); 139 bus_dmamem_free(dma->tag, dma->cpu_addr, dma->map); 140 bus_dma_tag_destroy(dma->tag); 141 } 142 143 int 144 gve_dmamap_create(struct gve_priv *priv, int size, int align, 145 struct gve_dma_handle *dma) 146 { 147 int err; 148 device_t dev = priv->dev; 149 150 err = bus_dma_tag_create( 151 bus_get_dma_tag(dev), /* parent */ 152 align, 0, /* alignment, bounds */ 153 BUS_SPACE_MAXADDR, /* lowaddr */ 154 BUS_SPACE_MAXADDR, /* highaddr */ 155 NULL, NULL, /* filter, filterarg */ 156 size, /* maxsize */ 157 1, /* nsegments */ 158 size, /* maxsegsize */ 159 BUS_DMA_ALLOCNOW, /* flags */ 160 NULL, /* lockfunc */ 161 NULL, /* lockarg */ 162 &dma->tag); 163 if (err != 0) { 164 device_printf(dev, "%s: bus_dma_tag_create failed: %d\n", 165 __func__, err); 166 goto clear_tag; 167 } 168 169 err = bus_dmamap_create(dma->tag, BUS_DMA_COHERENT, &dma->map); 170 if (err != 0) { 171 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 172 __func__, err); 173 goto destroy_tag; 174 } 175 176 /* An address set by the callback will never be -1 */ 177 dma->bus_addr = (bus_addr_t)-1; 178 err = bus_dmamap_load(dma->tag, dma->map, dma->cpu_addr, size, 179 gve_dmamap_load_callback, &dma->bus_addr, BUS_DMA_WAITOK); 180 if (err != 0 || dma->bus_addr == (bus_addr_t)-1) { 181 device_printf(dev, "%s: bus_dmamap_load failed: %d\n", 182 __func__, err); 183 goto destroy_map; 184 } 185 186 return (0); 187 188 destroy_map: 189 bus_dmamap_destroy(dma->tag, dma->map); 190 destroy_tag: 191 bus_dma_tag_destroy(dma->tag); 192 clear_tag: 193 dma->tag = NULL; 194 195 return (err); 196 } 197 198 void 199 gve_dmamap_destroy(struct gve_dma_handle *dma) 200 { 201 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 202 bus_dmamap_unload(dma->tag, dma->map); 203 bus_dmamap_destroy(dma->tag, dma->map); 204 bus_dma_tag_destroy(dma->tag); 205 } 206 207 static int 208 gve_mgmnt_intr(void *arg) 209 { 210 struct gve_priv *priv = arg; 211 212 taskqueue_enqueue(priv->service_tq, &priv->service_task); 213 return (FILTER_HANDLED); 214 } 215 216 void 217 gve_free_irqs(struct gve_priv *priv) 218 { 219 struct gve_irq *irq; 220 int num_irqs; 221 int rid; 222 int rc; 223 int i; 224 225 if (priv->irq_tbl == NULL) { 226 device_printf(priv->dev, "No irq table, nothing to free\n"); 227 return; 228 } 229 230 num_irqs = priv->tx_cfg.num_queues + priv->rx_cfg.num_queues + 1; 231 232 for (i = 0; i < num_irqs; i++) { 233 irq = &priv->irq_tbl[i]; 234 if (irq->res == NULL) 235 continue; 236 237 rid = rman_get_rid(irq->res); 238 239 rc = bus_teardown_intr(priv->dev, irq->res, irq->cookie); 240 if (rc != 0) 241 device_printf(priv->dev, "Failed to teardown irq num %d\n", 242 rid); 243 244 rc = bus_release_resource(priv->dev, SYS_RES_IRQ, 245 rid, irq->res); 246 if (rc != 0) 247 device_printf(priv->dev, "Failed to release irq num %d\n", 248 rid); 249 250 irq->res = NULL; 251 irq->cookie = NULL; 252 } 253 254 free(priv->irq_tbl, M_GVE); 255 priv->irq_tbl = NULL; 256 257 /* Safe to call even if msix was never alloced */ 258 pci_release_msi(priv->dev); 259 } 260 261 int 262 gve_alloc_irqs(struct gve_priv *priv) 263 { 264 int num_tx = priv->tx_cfg.num_queues; 265 int num_rx = priv->rx_cfg.num_queues; 266 int req_nvecs = num_tx + num_rx + 1; 267 int got_nvecs = req_nvecs; 268 struct gve_irq *irq; 269 int i, j, m; 270 int rid; 271 int err; 272 273 struct gve_ring_com *com; 274 struct gve_rx_ring *rx; 275 struct gve_tx_ring *tx; 276 277 if (pci_alloc_msix(priv->dev, &got_nvecs) != 0) { 278 device_printf(priv->dev, "Failed to acquire any msix vectors\n"); 279 err = ENXIO; 280 goto abort; 281 } else if (got_nvecs != req_nvecs) { 282 device_printf(priv->dev, "Tried to acquire %d msix vectors, got only %d\n", 283 req_nvecs, got_nvecs); 284 err = ENOSPC; 285 goto abort; 286 } 287 288 if (bootverbose) 289 device_printf(priv->dev, "Enabled MSIX with %d vectors\n", got_nvecs); 290 291 priv->irq_tbl = malloc(sizeof(struct gve_irq) * req_nvecs, M_GVE, 292 M_WAITOK | M_ZERO); 293 294 for (i = 0; i < num_tx; i++) { 295 irq = &priv->irq_tbl[i]; 296 tx = &priv->tx[i]; 297 com = &tx->com; 298 rid = i + 1; 299 300 irq->res = bus_alloc_resource_any(priv->dev, SYS_RES_IRQ, 301 &rid, RF_ACTIVE); 302 if (irq->res == NULL) { 303 device_printf(priv->dev, "Failed to alloc irq %d for Tx queue %d\n", 304 rid, i); 305 err = ENOMEM; 306 goto abort; 307 } 308 309 err = bus_setup_intr(priv->dev, irq->res, INTR_TYPE_NET | INTR_MPSAFE, 310 gve_tx_intr, NULL, &priv->tx[i], &irq->cookie); 311 if (err != 0) { 312 device_printf(priv->dev, "Failed to setup irq %d for Tx queue %d, " 313 "err: %d\n", rid, i, err); 314 goto abort; 315 } 316 317 bus_describe_intr(priv->dev, irq->res, irq->cookie, "tx%d", i); 318 com->ntfy_id = i; 319 } 320 321 for (j = 0; j < num_rx; j++) { 322 irq = &priv->irq_tbl[i + j]; 323 rx = &priv->rx[j]; 324 com = &rx->com; 325 rid = i + j + 1; 326 327 irq->res = bus_alloc_resource_any(priv->dev, SYS_RES_IRQ, 328 &rid, RF_ACTIVE); 329 if (irq->res == NULL) { 330 device_printf(priv->dev, 331 "Failed to alloc irq %d for Rx queue %d", rid, j); 332 err = ENOMEM; 333 goto abort; 334 } 335 336 err = bus_setup_intr(priv->dev, irq->res, INTR_TYPE_NET | INTR_MPSAFE, 337 gve_rx_intr, NULL, &priv->rx[j], &irq->cookie); 338 if (err != 0) { 339 device_printf(priv->dev, "Failed to setup irq %d for Rx queue %d, " 340 "err: %d\n", rid, j, err); 341 goto abort; 342 } 343 344 bus_describe_intr(priv->dev, irq->res, irq->cookie, "rx%d", j); 345 com->ntfy_id = i + j; 346 } 347 348 m = i + j; 349 rid = m + 1; 350 irq = &priv->irq_tbl[m]; 351 352 irq->res = bus_alloc_resource_any(priv->dev, SYS_RES_IRQ, 353 &rid, RF_ACTIVE); 354 if (irq->res == NULL) { 355 device_printf(priv->dev, "Failed to allocate irq %d for mgmnt queue\n", rid); 356 err = ENOMEM; 357 goto abort; 358 } 359 360 err = bus_setup_intr(priv->dev, irq->res, INTR_TYPE_NET | INTR_MPSAFE, 361 gve_mgmnt_intr, NULL, priv, &irq->cookie); 362 if (err != 0) { 363 device_printf(priv->dev, "Failed to setup irq %d for mgmnt queue, err: %d\n", 364 rid, err); 365 goto abort; 366 } 367 368 bus_describe_intr(priv->dev, irq->res, irq->cookie, "mgmnt"); 369 370 return (0); 371 372 abort: 373 gve_free_irqs(priv); 374 return (err); 375 } 376 377 void 378 gve_unmask_all_queue_irqs(struct gve_priv *priv) 379 { 380 struct gve_tx_ring *tx; 381 struct gve_rx_ring *rx; 382 int idx; 383 384 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 385 tx = &priv->tx[idx]; 386 gve_db_bar_write_4(priv, tx->com.irq_db_offset, 0); 387 } 388 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 389 rx = &priv->rx[idx]; 390 gve_db_bar_write_4(priv, rx->com.irq_db_offset, 0); 391 } 392 } 393 394 void 395 gve_mask_all_queue_irqs(struct gve_priv *priv) 396 { 397 for (int idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 398 struct gve_tx_ring *tx = &priv->tx[idx]; 399 gve_db_bar_write_4(priv, tx->com.irq_db_offset, GVE_IRQ_MASK); 400 } 401 for (int idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 402 struct gve_rx_ring *rx = &priv->rx[idx]; 403 gve_db_bar_write_4(priv, rx->com.irq_db_offset, GVE_IRQ_MASK); 404 } 405 } 406