1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include "opt_rss.h" 40 41 #include <sys/param.h> 42 #include <sys/bus.h> 43 #include <sys/kernel.h> 44 #include <sys/malloc.h> 45 #include <sys/queue.h> 46 #include <sys/rman.h> 47 #include <sys/syslog.h> 48 #include <sys/taskqueue.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 53 #include <dev/pci/pcireg.h> 54 #include <dev/pci/pcivar.h> 55 56 #ifdef RSS 57 #include <net/rss_config.h> 58 #endif 59 60 #include "common/efx.h" 61 62 #include "sfxge.h" 63 64 static int 65 sfxge_intr_line_filter(void *arg) 66 { 67 struct sfxge_evq *evq; 68 struct sfxge_softc *sc; 69 efx_nic_t *enp; 70 struct sfxge_intr *intr; 71 boolean_t fatal; 72 uint32_t qmask; 73 74 evq = (struct sfxge_evq *)arg; 75 sc = evq->sc; 76 enp = sc->enp; 77 intr = &sc->intr; 78 79 KASSERT(intr != NULL, ("intr == NULL")); 80 KASSERT(intr->type == EFX_INTR_LINE, 81 ("intr->type != EFX_INTR_LINE")); 82 83 if (intr->state != SFXGE_INTR_STARTED) 84 return (FILTER_STRAY); 85 86 (void)efx_intr_status_line(enp, &fatal, &qmask); 87 88 if (fatal) { 89 (void) efx_intr_disable(enp); 90 (void) efx_intr_fatal(enp); 91 return (FILTER_HANDLED); 92 } 93 94 if (qmask != 0) { 95 intr->zero_count = 0; 96 return (FILTER_SCHEDULE_THREAD); 97 } 98 99 /* SF bug 15783: If the function is not asserting its IRQ and 100 * we read the queue mask on the cycle before a flag is added 101 * to the mask, this inhibits the function from asserting the 102 * IRQ even though we don't see the flag set. To work around 103 * this, we must re-prime all event queues and report the IRQ 104 * as handled when we see a mask of zero. To allow for shared 105 * IRQs, we don't repeat this if we see a mask of zero twice 106 * or more in a row. 107 */ 108 if (intr->zero_count++ == 0) { 109 if (evq->init_state == SFXGE_EVQ_STARTED) { 110 if (efx_ev_qpending(evq->common, evq->read_ptr)) 111 return (FILTER_SCHEDULE_THREAD); 112 efx_ev_qprime(evq->common, evq->read_ptr); 113 return (FILTER_HANDLED); 114 } 115 } 116 117 return (FILTER_STRAY); 118 } 119 120 static void 121 sfxge_intr_line(void *arg) 122 { 123 struct sfxge_evq *evq = arg; 124 125 (void)sfxge_ev_qpoll(evq); 126 } 127 128 static void 129 sfxge_intr_message(void *arg) 130 { 131 struct sfxge_evq *evq; 132 struct sfxge_softc *sc; 133 efx_nic_t *enp; 134 struct sfxge_intr *intr; 135 unsigned int index; 136 boolean_t fatal; 137 138 evq = (struct sfxge_evq *)arg; 139 sc = evq->sc; 140 enp = sc->enp; 141 intr = &sc->intr; 142 index = evq->index; 143 144 KASSERT(intr != NULL, ("intr == NULL")); 145 KASSERT(intr->type == EFX_INTR_MESSAGE, 146 ("intr->type != EFX_INTR_MESSAGE")); 147 148 if (__predict_false(intr->state != SFXGE_INTR_STARTED)) 149 return; 150 151 (void)efx_intr_status_message(enp, index, &fatal); 152 153 if (fatal) { 154 (void)efx_intr_disable(enp); 155 (void)efx_intr_fatal(enp); 156 return; 157 } 158 159 (void)sfxge_ev_qpoll(evq); 160 } 161 162 static int 163 sfxge_intr_bus_enable(struct sfxge_softc *sc) 164 { 165 struct sfxge_intr *intr; 166 struct sfxge_intr_hdl *table; 167 driver_filter_t *filter; 168 driver_intr_t *handler; 169 int index; 170 int err; 171 172 intr = &sc->intr; 173 table = intr->table; 174 175 switch (intr->type) { 176 case EFX_INTR_MESSAGE: 177 filter = NULL; /* not shared */ 178 handler = sfxge_intr_message; 179 break; 180 181 case EFX_INTR_LINE: 182 filter = sfxge_intr_line_filter; 183 handler = sfxge_intr_line; 184 break; 185 186 default: 187 KASSERT(0, ("Invalid interrupt type")); 188 return (EINVAL); 189 } 190 191 /* Try to add the handlers */ 192 for (index = 0; index < intr->n_alloc; index++) { 193 if ((err = bus_setup_intr(sc->dev, table[index].eih_res, 194 INTR_MPSAFE|INTR_TYPE_NET, filter, handler, 195 sc->evq[index], &table[index].eih_tag)) != 0) { 196 goto fail; 197 } 198 if (intr->n_alloc > 1) 199 bus_describe_intr(sc->dev, table[index].eih_res, 200 table[index].eih_tag, "%d", index); 201 #ifdef RSS 202 bus_bind_intr(sc->dev, table[index].eih_res, 203 rss_getcpu(index)); 204 #else 205 bus_bind_intr(sc->dev, table[index].eih_res, index); 206 #endif 207 208 } 209 210 return (0); 211 212 fail: 213 /* Remove remaining handlers */ 214 while (--index >= 0) 215 bus_teardown_intr(sc->dev, table[index].eih_res, 216 table[index].eih_tag); 217 218 return (err); 219 } 220 221 static void 222 sfxge_intr_bus_disable(struct sfxge_softc *sc) 223 { 224 struct sfxge_intr *intr; 225 struct sfxge_intr_hdl *table; 226 int i; 227 228 intr = &sc->intr; 229 table = intr->table; 230 231 /* Remove all handlers */ 232 for (i = 0; i < intr->n_alloc; i++) 233 bus_teardown_intr(sc->dev, table[i].eih_res, 234 table[i].eih_tag); 235 } 236 237 static int 238 sfxge_intr_alloc(struct sfxge_softc *sc, int count) 239 { 240 device_t dev; 241 struct sfxge_intr_hdl *table; 242 struct sfxge_intr *intr; 243 struct resource *res; 244 int rid; 245 int error; 246 int i; 247 248 dev = sc->dev; 249 intr = &sc->intr; 250 error = 0; 251 252 table = malloc(count * sizeof(struct sfxge_intr_hdl), 253 M_SFXGE, M_WAITOK); 254 intr->table = table; 255 256 for (i = 0; i < count; i++) { 257 rid = i + 1; 258 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 259 RF_SHAREABLE | RF_ACTIVE); 260 if (res == NULL) { 261 device_printf(dev, "Couldn't allocate interrupts for " 262 "message %d\n", rid); 263 error = ENOMEM; 264 break; 265 } 266 table[i].eih_rid = rid; 267 table[i].eih_res = res; 268 } 269 270 if (error != 0) { 271 count = i - 1; 272 for (i = 0; i < count; i++) 273 bus_release_resource(dev, SYS_RES_IRQ, 274 table[i].eih_rid, table[i].eih_res); 275 } 276 277 return (error); 278 } 279 280 static void 281 sfxge_intr_teardown_msix(struct sfxge_softc *sc) 282 { 283 device_t dev; 284 struct resource *resp; 285 int rid; 286 287 dev = sc->dev; 288 resp = sc->intr.msix_res; 289 290 rid = rman_get_rid(resp); 291 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 292 } 293 294 static int 295 sfxge_intr_setup_msix(struct sfxge_softc *sc) 296 { 297 struct sfxge_intr *intr; 298 struct resource *resp; 299 device_t dev; 300 int count; 301 int rid; 302 303 dev = sc->dev; 304 intr = &sc->intr; 305 306 /* Check if MSI-X is available. */ 307 count = pci_msix_count(dev); 308 if (count == 0) 309 return (EINVAL); 310 311 /* Do not try to allocate more than already estimated EVQ maximum */ 312 KASSERT(sc->evq_max > 0, ("evq_max is zero")); 313 count = MIN(count, sc->evq_max); 314 315 rid = PCIR_BAR(4); 316 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 317 if (resp == NULL) 318 return (ENOMEM); 319 320 if (pci_alloc_msix(dev, &count) != 0) { 321 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 322 return (ENOMEM); 323 } 324 325 /* Allocate interrupt handlers. */ 326 if (sfxge_intr_alloc(sc, count) != 0) { 327 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 328 pci_release_msi(dev); 329 return (ENOMEM); 330 } 331 332 intr->type = EFX_INTR_MESSAGE; 333 intr->n_alloc = count; 334 intr->msix_res = resp; 335 336 return (0); 337 } 338 339 static int 340 sfxge_intr_setup_msi(struct sfxge_softc *sc) 341 { 342 struct sfxge_intr_hdl *table; 343 struct sfxge_intr *intr; 344 device_t dev; 345 int count; 346 int error; 347 348 dev = sc->dev; 349 intr = &sc->intr; 350 table = intr->table; 351 352 /* 353 * Check if MSI is available. All messages must be written to 354 * the same address and on x86 this means the IRQs have the 355 * same CPU affinity. So we only ever allocate 1. 356 */ 357 count = pci_msi_count(dev) ? 1 : 0; 358 if (count == 0) 359 return (EINVAL); 360 361 if ((error = pci_alloc_msi(dev, &count)) != 0) 362 return (ENOMEM); 363 364 /* Allocate interrupt handler. */ 365 if (sfxge_intr_alloc(sc, count) != 0) { 366 pci_release_msi(dev); 367 return (ENOMEM); 368 } 369 370 intr->type = EFX_INTR_MESSAGE; 371 intr->n_alloc = count; 372 373 return (0); 374 } 375 376 static int 377 sfxge_intr_setup_fixed(struct sfxge_softc *sc) 378 { 379 struct sfxge_intr_hdl *table; 380 struct sfxge_intr *intr; 381 struct resource *res; 382 device_t dev; 383 int rid; 384 385 dev = sc->dev; 386 intr = &sc->intr; 387 388 rid = 0; 389 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 390 RF_SHAREABLE | RF_ACTIVE); 391 if (res == NULL) 392 return (ENOMEM); 393 394 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK); 395 table[0].eih_rid = rid; 396 table[0].eih_res = res; 397 398 intr->type = EFX_INTR_LINE; 399 intr->n_alloc = 1; 400 intr->table = table; 401 402 return (0); 403 } 404 405 static const char *const __sfxge_err[] = { 406 "", 407 "SRAM out-of-bounds", 408 "Buffer ID out-of-bounds", 409 "Internal memory parity", 410 "Receive buffer ownership", 411 "Transmit buffer ownership", 412 "Receive descriptor ownership", 413 "Transmit descriptor ownership", 414 "Event queue ownership", 415 "Event queue FIFO overflow", 416 "Illegal address", 417 "SRAM parity" 418 }; 419 420 void 421 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0, 422 uint32_t dword1) 423 { 424 struct sfxge_softc *sc = (struct sfxge_softc *)arg; 425 device_t dev = sc->dev; 426 427 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)", 428 device_get_name(dev), device_get_unit(dev), 429 __sfxge_err[code], dword1, dword0); 430 } 431 432 void 433 sfxge_intr_stop(struct sfxge_softc *sc) 434 { 435 struct sfxge_intr *intr; 436 437 intr = &sc->intr; 438 439 KASSERT(intr->state == SFXGE_INTR_STARTED, 440 ("Interrupts not started")); 441 442 intr->state = SFXGE_INTR_INITIALIZED; 443 444 /* Disable interrupts at the NIC */ 445 efx_intr_disable(sc->enp); 446 447 /* Disable interrupts at the bus */ 448 sfxge_intr_bus_disable(sc); 449 450 /* Tear down common code interrupt bits. */ 451 efx_intr_fini(sc->enp); 452 } 453 454 int 455 sfxge_intr_start(struct sfxge_softc *sc) 456 { 457 struct sfxge_intr *intr; 458 efsys_mem_t *esmp; 459 int rc; 460 461 intr = &sc->intr; 462 esmp = &intr->status; 463 464 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 465 ("Interrupts not initialized")); 466 467 /* Zero the memory. */ 468 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE); 469 470 /* Initialize common code interrupt bits. */ 471 (void)efx_intr_init(sc->enp, intr->type, esmp); 472 473 /* Enable interrupts at the bus */ 474 if ((rc = sfxge_intr_bus_enable(sc)) != 0) 475 goto fail; 476 477 intr->state = SFXGE_INTR_STARTED; 478 479 /* Enable interrupts at the NIC */ 480 efx_intr_enable(sc->enp); 481 482 return (0); 483 484 fail: 485 /* Tear down common code interrupt bits. */ 486 efx_intr_fini(sc->enp); 487 488 intr->state = SFXGE_INTR_INITIALIZED; 489 490 return (rc); 491 } 492 493 void 494 sfxge_intr_fini(struct sfxge_softc *sc) 495 { 496 struct sfxge_intr_hdl *table; 497 struct sfxge_intr *intr; 498 efsys_mem_t *esmp; 499 device_t dev; 500 int i; 501 502 dev = sc->dev; 503 intr = &sc->intr; 504 esmp = &intr->status; 505 table = intr->table; 506 507 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 508 ("intr->state != SFXGE_INTR_INITIALIZED")); 509 510 /* Free DMA memory. */ 511 sfxge_dma_free(esmp); 512 513 /* Free interrupt handles. */ 514 for (i = 0; i < intr->n_alloc; i++) 515 bus_release_resource(dev, SYS_RES_IRQ, 516 table[i].eih_rid, table[i].eih_res); 517 518 if (table[0].eih_rid != 0) 519 pci_release_msi(dev); 520 521 if (intr->msix_res != NULL) 522 sfxge_intr_teardown_msix(sc); 523 524 /* Free the handle table */ 525 free(table, M_SFXGE); 526 intr->table = NULL; 527 intr->n_alloc = 0; 528 529 /* Clear the interrupt type */ 530 intr->type = EFX_INTR_INVALID; 531 532 intr->state = SFXGE_INTR_UNINITIALIZED; 533 } 534 535 int 536 sfxge_intr_init(struct sfxge_softc *sc) 537 { 538 device_t dev; 539 struct sfxge_intr *intr; 540 efsys_mem_t *esmp; 541 int rc; 542 543 dev = sc->dev; 544 intr = &sc->intr; 545 esmp = &intr->status; 546 547 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED, 548 ("Interrupts already initialized")); 549 550 /* Try to setup MSI-X or MSI interrupts if available. */ 551 if ((rc = sfxge_intr_setup_msix(sc)) == 0) 552 device_printf(dev, "Using MSI-X interrupts\n"); 553 else if ((rc = sfxge_intr_setup_msi(sc)) == 0) 554 device_printf(dev, "Using MSI interrupts\n"); 555 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) { 556 device_printf(dev, "Using fixed interrupts\n"); 557 } else { 558 device_printf(dev, "Couldn't setup interrupts\n"); 559 return (ENOMEM); 560 } 561 562 /* Set up DMA for interrupts. */ 563 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0) 564 return (ENOMEM); 565 566 intr->state = SFXGE_INTR_INITIALIZED; 567 568 return (0); 569 } 570