1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2023-2024 Google LLC 5 * 6 * Redistribution and use in source and binary forms, with or without modification, 7 * are permitted provided that the following conditions are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright notice, this 10 * list of conditions and the following disclaimer. 11 * 12 * 2. Redistributions in binary form must reproduce the above copyright notice, 13 * this list of conditions and the following disclaimer in the documentation 14 * and/or other materials provided with the distribution. 15 * 16 * 3. Neither the name of the copyright holder nor the names of its contributors 17 * may be used to endorse or promote products derived from this software without 18 * specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 #include "gve.h" 32 #include "gve_dqo.h" 33 34 uint32_t 35 gve_reg_bar_read_4(struct gve_priv *priv, bus_size_t offset) 36 { 37 return (be32toh(bus_read_4(priv->reg_bar, offset))); 38 } 39 40 void 41 gve_reg_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val) 42 { 43 bus_write_4(priv->reg_bar, offset, htobe32(val)); 44 } 45 46 void 47 gve_db_bar_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val) 48 { 49 bus_write_4(priv->db_bar, offset, htobe32(val)); 50 } 51 52 void 53 gve_db_bar_dqo_write_4(struct gve_priv *priv, bus_size_t offset, uint32_t val) 54 { 55 bus_write_4(priv->db_bar, offset, val); 56 } 57 58 void 59 gve_alloc_counters(counter_u64_t *stat, int num_stats) 60 { 61 int i; 62 63 for (i = 0; i < num_stats; i++) 64 stat[i] = counter_u64_alloc(M_WAITOK); 65 } 66 67 void 68 gve_free_counters(counter_u64_t *stat, int num_stats) 69 { 70 int i; 71 72 for (i = 0; i < num_stats; i++) { 73 if (stat[i] != NULL) { 74 counter_u64_free(stat[i]); 75 stat[i] = NULL; 76 } 77 } 78 } 79 80 /* Currently assumes a single segment. */ 81 static void 82 gve_dmamap_load_callback(void *arg, bus_dma_segment_t *segs, int nseg, 83 int error) 84 { 85 if (error == 0) 86 *(bus_addr_t *) arg = segs[0].ds_addr; 87 } 88 89 int 90 gve_dma_alloc_coherent(struct gve_priv *priv, int size, int align, 91 struct gve_dma_handle *dma) 92 { 93 int err; 94 device_t dev = priv->dev; 95 96 err = bus_dma_tag_create( 97 bus_get_dma_tag(dev), /* parent */ 98 align, 0, /* alignment, bounds */ 99 BUS_SPACE_MAXADDR, /* lowaddr */ 100 BUS_SPACE_MAXADDR, /* highaddr */ 101 NULL, NULL, /* filter, filterarg */ 102 size, /* maxsize */ 103 1, /* nsegments */ 104 size, /* maxsegsize */ 105 BUS_DMA_ALLOCNOW, /* flags */ 106 NULL, /* lockfunc */ 107 NULL, /* lockarg */ 108 &dma->tag); 109 if (err != 0) { 110 device_printf(dev, "%s: bus_dma_tag_create failed: %d\n", 111 __func__, err); 112 goto clear_tag; 113 } 114 115 err = bus_dmamem_alloc(dma->tag, (void **) &dma->cpu_addr, 116 BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, 117 &dma->map); 118 if (err != 0) { 119 device_printf(dev, "%s: bus_dmamem_alloc(%ju) failed: %d\n", 120 __func__, (uintmax_t)size, err); 121 goto destroy_tag; 122 } 123 124 /* An address set by the callback will never be -1 */ 125 dma->bus_addr = (bus_addr_t)-1; 126 err = bus_dmamap_load(dma->tag, dma->map, dma->cpu_addr, size, 127 gve_dmamap_load_callback, &dma->bus_addr, BUS_DMA_NOWAIT); 128 if (err != 0 || dma->bus_addr == (bus_addr_t)-1) { 129 device_printf(dev, "%s: bus_dmamap_load failed: %d\n", __func__, err); 130 goto free_mem; 131 } 132 133 return (0); 134 135 free_mem: 136 bus_dmamem_free(dma->tag, dma->cpu_addr, dma->map); 137 destroy_tag: 138 bus_dma_tag_destroy(dma->tag); 139 clear_tag: 140 dma->tag = NULL; 141 142 return (err); 143 } 144 145 void 146 gve_dma_free_coherent(struct gve_dma_handle *dma) 147 { 148 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 149 bus_dmamap_unload(dma->tag, dma->map); 150 bus_dmamem_free(dma->tag, dma->cpu_addr, dma->map); 151 bus_dma_tag_destroy(dma->tag); 152 } 153 154 int 155 gve_dmamap_create(struct gve_priv *priv, int size, int align, 156 struct gve_dma_handle *dma) 157 { 158 int err; 159 device_t dev = priv->dev; 160 161 err = bus_dma_tag_create( 162 bus_get_dma_tag(dev), /* parent */ 163 align, 0, /* alignment, bounds */ 164 BUS_SPACE_MAXADDR, /* lowaddr */ 165 BUS_SPACE_MAXADDR, /* highaddr */ 166 NULL, NULL, /* filter, filterarg */ 167 size, /* maxsize */ 168 1, /* nsegments */ 169 size, /* maxsegsize */ 170 BUS_DMA_ALLOCNOW, /* flags */ 171 NULL, /* lockfunc */ 172 NULL, /* lockarg */ 173 &dma->tag); 174 if (err != 0) { 175 device_printf(dev, "%s: bus_dma_tag_create failed: %d\n", 176 __func__, err); 177 goto clear_tag; 178 } 179 180 err = bus_dmamap_create(dma->tag, BUS_DMA_COHERENT, &dma->map); 181 if (err != 0) { 182 device_printf(dev, "%s: bus_dmamap_create failed: %d\n", 183 __func__, err); 184 goto destroy_tag; 185 } 186 187 /* An address set by the callback will never be -1 */ 188 dma->bus_addr = (bus_addr_t)-1; 189 err = bus_dmamap_load(dma->tag, dma->map, dma->cpu_addr, size, 190 gve_dmamap_load_callback, &dma->bus_addr, BUS_DMA_WAITOK); 191 if (err != 0 || dma->bus_addr == (bus_addr_t)-1) { 192 device_printf(dev, "%s: bus_dmamap_load failed: %d\n", 193 __func__, err); 194 goto destroy_map; 195 } 196 197 return (0); 198 199 destroy_map: 200 bus_dmamap_destroy(dma->tag, dma->map); 201 destroy_tag: 202 bus_dma_tag_destroy(dma->tag); 203 clear_tag: 204 dma->tag = NULL; 205 206 return (err); 207 } 208 209 void 210 gve_dmamap_destroy(struct gve_dma_handle *dma) 211 { 212 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 213 bus_dmamap_unload(dma->tag, dma->map); 214 bus_dmamap_destroy(dma->tag, dma->map); 215 bus_dma_tag_destroy(dma->tag); 216 } 217 218 static int 219 gve_mgmnt_intr(void *arg) 220 { 221 struct gve_priv *priv = arg; 222 223 taskqueue_enqueue(priv->service_tq, &priv->service_task); 224 return (FILTER_HANDLED); 225 } 226 227 void 228 gve_free_irqs(struct gve_priv *priv) 229 { 230 struct gve_irq *irq; 231 int num_irqs; 232 int rid; 233 int rc; 234 int i; 235 236 if (priv->irq_tbl == NULL) { 237 device_printf(priv->dev, "No irq table, nothing to free\n"); 238 return; 239 } 240 241 num_irqs = priv->tx_cfg.max_queues + priv->rx_cfg.max_queues + 1; 242 243 for (i = 0; i < num_irqs; i++) { 244 irq = &priv->irq_tbl[i]; 245 if (irq->res == NULL) 246 continue; 247 248 rid = rman_get_rid(irq->res); 249 250 rc = bus_teardown_intr(priv->dev, irq->res, irq->cookie); 251 if (rc != 0) 252 device_printf(priv->dev, "Failed to teardown irq num %d\n", 253 rid); 254 255 rc = bus_release_resource(priv->dev, SYS_RES_IRQ, 256 rid, irq->res); 257 if (rc != 0) 258 device_printf(priv->dev, "Failed to release irq num %d\n", 259 rid); 260 261 irq->res = NULL; 262 irq->cookie = NULL; 263 } 264 265 free(priv->irq_tbl, M_GVE); 266 priv->irq_tbl = NULL; 267 268 /* Safe to call even if msix was never alloced */ 269 pci_release_msi(priv->dev); 270 } 271 272 int 273 gve_alloc_irqs(struct gve_priv *priv) 274 { 275 int num_tx = priv->tx_cfg.max_queues; 276 int num_rx = priv->rx_cfg.max_queues; 277 int req_nvecs = num_tx + num_rx + 1; 278 int got_nvecs = req_nvecs; 279 struct gve_irq *irq; 280 int i, j, m; 281 int rid; 282 int err; 283 284 struct gve_ring_com *com; 285 struct gve_rx_ring *rx; 286 struct gve_tx_ring *tx; 287 288 if (pci_alloc_msix(priv->dev, &got_nvecs) != 0) { 289 device_printf(priv->dev, "Failed to acquire any msix vectors\n"); 290 err = ENXIO; 291 goto abort; 292 } else if (got_nvecs != req_nvecs) { 293 device_printf(priv->dev, "Tried to acquire %d msix vectors, got only %d\n", 294 req_nvecs, got_nvecs); 295 err = ENOSPC; 296 goto abort; 297 } 298 299 if (bootverbose) 300 device_printf(priv->dev, "Enabled MSIX with %d vectors\n", got_nvecs); 301 302 priv->irq_tbl = malloc(sizeof(struct gve_irq) * req_nvecs, M_GVE, 303 M_WAITOK | M_ZERO); 304 305 for (i = 0; i < num_tx; i++) { 306 irq = &priv->irq_tbl[i]; 307 tx = &priv->tx[i]; 308 com = &tx->com; 309 rid = i + 1; 310 311 irq->res = bus_alloc_resource_any(priv->dev, SYS_RES_IRQ, 312 &rid, RF_ACTIVE); 313 if (irq->res == NULL) { 314 device_printf(priv->dev, "Failed to alloc irq %d for Tx queue %d\n", 315 rid, i); 316 err = ENOMEM; 317 goto abort; 318 } 319 320 err = bus_setup_intr(priv->dev, irq->res, INTR_TYPE_NET | INTR_MPSAFE, 321 gve_is_gqi(priv) ? gve_tx_intr : gve_tx_intr_dqo, NULL, 322 &priv->tx[i], &irq->cookie); 323 if (err != 0) { 324 device_printf(priv->dev, "Failed to setup irq %d for Tx queue %d, " 325 "err: %d\n", rid, i, err); 326 goto abort; 327 } 328 329 bus_describe_intr(priv->dev, irq->res, irq->cookie, "tx%d", i); 330 com->ntfy_id = i; 331 } 332 333 for (j = 0; j < num_rx; j++) { 334 irq = &priv->irq_tbl[i + j]; 335 rx = &priv->rx[j]; 336 com = &rx->com; 337 rid = i + j + 1; 338 339 irq->res = bus_alloc_resource_any(priv->dev, SYS_RES_IRQ, 340 &rid, RF_ACTIVE); 341 if (irq->res == NULL) { 342 device_printf(priv->dev, 343 "Failed to alloc irq %d for Rx queue %d", rid, j); 344 err = ENOMEM; 345 goto abort; 346 } 347 348 err = bus_setup_intr(priv->dev, irq->res, INTR_TYPE_NET | INTR_MPSAFE, 349 gve_is_gqi(priv) ? gve_rx_intr : gve_rx_intr_dqo, NULL, 350 &priv->rx[j], &irq->cookie); 351 if (err != 0) { 352 device_printf(priv->dev, "Failed to setup irq %d for Rx queue %d, " 353 "err: %d\n", rid, j, err); 354 goto abort; 355 } 356 357 bus_describe_intr(priv->dev, irq->res, irq->cookie, "rx%d", j); 358 com->ntfy_id = i + j; 359 } 360 361 m = i + j; 362 rid = m + 1; 363 irq = &priv->irq_tbl[m]; 364 365 irq->res = bus_alloc_resource_any(priv->dev, SYS_RES_IRQ, 366 &rid, RF_ACTIVE); 367 if (irq->res == NULL) { 368 device_printf(priv->dev, "Failed to allocate irq %d for mgmnt queue\n", rid); 369 err = ENOMEM; 370 goto abort; 371 } 372 373 err = bus_setup_intr(priv->dev, irq->res, INTR_TYPE_NET | INTR_MPSAFE, 374 gve_mgmnt_intr, NULL, priv, &irq->cookie); 375 if (err != 0) { 376 device_printf(priv->dev, "Failed to setup irq %d for mgmnt queue, err: %d\n", 377 rid, err); 378 goto abort; 379 } 380 381 bus_describe_intr(priv->dev, irq->res, irq->cookie, "mgmnt"); 382 383 return (0); 384 385 abort: 386 gve_free_irqs(priv); 387 return (err); 388 } 389 390 /* 391 * Builds register value to write to DQO IRQ doorbell to enable with specified 392 * ITR interval. 393 */ 394 static uint32_t 395 gve_setup_itr_interval_dqo(uint32_t interval_us) 396 { 397 uint32_t result = GVE_ITR_ENABLE_BIT_DQO; 398 399 /* Interval has 2us granularity. */ 400 interval_us >>= 1; 401 402 interval_us &= GVE_ITR_INTERVAL_DQO_MASK; 403 result |= (interval_us << GVE_ITR_INTERVAL_DQO_SHIFT); 404 405 return (result); 406 } 407 408 void 409 gve_unmask_all_queue_irqs(struct gve_priv *priv) 410 { 411 struct gve_tx_ring *tx; 412 struct gve_rx_ring *rx; 413 int idx; 414 415 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 416 tx = &priv->tx[idx]; 417 if (gve_is_gqi(priv)) 418 gve_db_bar_write_4(priv, tx->com.irq_db_offset, 0); 419 else 420 gve_db_bar_dqo_write_4(priv, tx->com.irq_db_offset, 421 gve_setup_itr_interval_dqo(GVE_TX_IRQ_RATELIMIT_US_DQO)); 422 } 423 424 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 425 rx = &priv->rx[idx]; 426 if (gve_is_gqi(priv)) 427 gve_db_bar_write_4(priv, rx->com.irq_db_offset, 0); 428 else 429 gve_db_bar_dqo_write_4(priv, rx->com.irq_db_offset, 430 gve_setup_itr_interval_dqo(GVE_RX_IRQ_RATELIMIT_US_DQO)); 431 } 432 } 433 434 void 435 gve_mask_all_queue_irqs(struct gve_priv *priv) 436 { 437 for (int idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 438 struct gve_tx_ring *tx = &priv->tx[idx]; 439 gve_db_bar_write_4(priv, tx->com.irq_db_offset, GVE_IRQ_MASK); 440 } 441 for (int idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 442 struct gve_rx_ring *rx = &priv->rx[idx]; 443 gve_db_bar_write_4(priv, rx->com.irq_db_offset, GVE_IRQ_MASK); 444 } 445 } 446 447 /* 448 * In some cases, such as tracking timeout events, we must mark a timestamp as 449 * invalid when we do not want to consider its value. Such timestamps must be 450 * checked for validity before reading them. 451 */ 452 void 453 gve_invalidate_timestamp(int64_t *timestamp_sec) 454 { 455 atomic_store_64(timestamp_sec, GVE_TIMESTAMP_INVALID); 456 } 457 458 /* 459 * Returns 0 if the timestamp is invalid, otherwise returns the elapsed seconds 460 * since the timestamp was set. 461 */ 462 int64_t 463 gve_seconds_since(int64_t *timestamp_sec) 464 { 465 struct bintime curr_time; 466 int64_t enqueued_time; 467 468 getbintime(&curr_time); 469 enqueued_time = atomic_load_64(timestamp_sec); 470 if (enqueued_time == GVE_TIMESTAMP_INVALID) 471 return (0); 472 return ((int64_t)(curr_time.sec - enqueued_time)); 473 } 474 475 void 476 gve_set_timestamp(int64_t *timestamp_sec) 477 { 478 struct bintime curr_time; 479 480 getbintime(&curr_time); 481 atomic_store_64(timestamp_sec, curr_time.sec); 482 } 483 484 bool 485 gve_timestamp_valid(int64_t *timestamp_sec) 486 { 487 return (atomic_load_64(timestamp_sec) != GVE_TIMESTAMP_INVALID); 488 } 489