1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include "opt_rss.h" 40 41 #include <sys/param.h> 42 #include <sys/bus.h> 43 #include <sys/kernel.h> 44 #include <sys/malloc.h> 45 #include <sys/queue.h> 46 #include <sys/rman.h> 47 #include <sys/syslog.h> 48 #include <sys/taskqueue.h> 49 50 #include <machine/bus.h> 51 #include <machine/resource.h> 52 53 #include <dev/pci/pcireg.h> 54 #include <dev/pci/pcivar.h> 55 56 #ifdef RSS 57 #include <net/rss_config.h> 58 #endif 59 60 #include "common/efx.h" 61 62 #include "sfxge.h" 63 64 static int 65 sfxge_intr_line_filter(void *arg) 66 { 67 struct sfxge_evq *evq; 68 struct sfxge_softc *sc; 69 efx_nic_t *enp; 70 struct sfxge_intr *intr; 71 boolean_t fatal; 72 uint32_t qmask; 73 74 evq = (struct sfxge_evq *)arg; 75 sc = evq->sc; 76 enp = sc->enp; 77 intr = &sc->intr; 78 79 KASSERT(intr != NULL, ("intr == NULL")); 80 KASSERT(intr->type == EFX_INTR_LINE, 81 ("intr->type != EFX_INTR_LINE")); 82 83 if (intr->state != SFXGE_INTR_STARTED) 84 return (FILTER_STRAY); 85 86 (void)efx_intr_status_line(enp, &fatal, &qmask); 87 88 if (fatal) { 89 (void) efx_intr_disable(enp); 90 (void) efx_intr_fatal(enp); 91 return (FILTER_HANDLED); 92 } 93 94 if (qmask != 0) { 95 intr->zero_count = 0; 96 return (FILTER_SCHEDULE_THREAD); 97 } 98 99 /* SF bug 15783: If the function is not asserting its IRQ and 100 * we read the queue mask on the cycle before a flag is added 101 * to the mask, this inhibits the function from asserting the 102 * IRQ even though we don't see the flag set. To work around 103 * this, we must re-prime all event queues and report the IRQ 104 * as handled when we see a mask of zero. To allow for shared 105 * IRQs, we don't repeat this if we see a mask of zero twice 106 * or more in a row. 107 */ 108 if (intr->zero_count++ == 0) { 109 if (evq->init_state == SFXGE_EVQ_STARTED) { 110 if (efx_ev_qpending(evq->common, evq->read_ptr)) 111 return (FILTER_SCHEDULE_THREAD); 112 efx_ev_qprime(evq->common, evq->read_ptr); 113 return (FILTER_HANDLED); 114 } 115 } 116 117 return (FILTER_STRAY); 118 } 119 120 static void 121 sfxge_intr_line(void *arg) 122 { 123 struct sfxge_evq *evq = arg; 124 125 (void)sfxge_ev_qpoll(evq); 126 } 127 128 static void 129 sfxge_intr_message(void *arg) 130 { 131 struct sfxge_evq *evq; 132 struct sfxge_softc *sc; 133 efx_nic_t *enp; 134 struct sfxge_intr *intr; 135 unsigned int index; 136 boolean_t fatal; 137 138 evq = (struct sfxge_evq *)arg; 139 sc = evq->sc; 140 enp = sc->enp; 141 intr = &sc->intr; 142 index = evq->index; 143 144 KASSERT(intr != NULL, ("intr == NULL")); 145 KASSERT(intr->type == EFX_INTR_MESSAGE, 146 ("intr->type != EFX_INTR_MESSAGE")); 147 148 if (__predict_false(intr->state != SFXGE_INTR_STARTED)) 149 return; 150 151 (void)efx_intr_status_message(enp, index, &fatal); 152 153 if (fatal) { 154 (void)efx_intr_disable(enp); 155 (void)efx_intr_fatal(enp); 156 return; 157 } 158 159 (void)sfxge_ev_qpoll(evq); 160 } 161 162 static int 163 sfxge_intr_bus_enable(struct sfxge_softc *sc) 164 { 165 struct sfxge_intr *intr; 166 struct sfxge_intr_hdl *table; 167 driver_filter_t *filter; 168 driver_intr_t *handler; 169 int index; 170 int err; 171 172 intr = &sc->intr; 173 table = intr->table; 174 175 switch (intr->type) { 176 case EFX_INTR_MESSAGE: 177 filter = NULL; /* not shared */ 178 handler = sfxge_intr_message; 179 break; 180 181 case EFX_INTR_LINE: 182 filter = sfxge_intr_line_filter; 183 handler = sfxge_intr_line; 184 break; 185 186 default: 187 KASSERT(0, ("Invalid interrupt type")); 188 return (EINVAL); 189 } 190 191 /* Try to add the handlers */ 192 for (index = 0; index < intr->n_alloc; index++) { 193 if ((err = bus_setup_intr(sc->dev, table[index].eih_res, 194 INTR_MPSAFE|INTR_TYPE_NET, filter, handler, 195 sc->evq[index], &table[index].eih_tag)) != 0) { 196 goto fail; 197 } 198 #ifdef SFXGE_HAVE_DESCRIBE_INTR 199 if (intr->n_alloc > 1) 200 bus_describe_intr(sc->dev, table[index].eih_res, 201 table[index].eih_tag, "%d", index); 202 #endif 203 #ifdef RSS 204 bus_bind_intr(sc->dev, table[index].eih_res, 205 rss_getcpu(index)); 206 #else 207 bus_bind_intr(sc->dev, table[index].eih_res, index); 208 #endif 209 210 } 211 212 return (0); 213 214 fail: 215 /* Remove remaining handlers */ 216 while (--index >= 0) 217 bus_teardown_intr(sc->dev, table[index].eih_res, 218 table[index].eih_tag); 219 220 return (err); 221 } 222 223 static void 224 sfxge_intr_bus_disable(struct sfxge_softc *sc) 225 { 226 struct sfxge_intr *intr; 227 struct sfxge_intr_hdl *table; 228 int i; 229 230 intr = &sc->intr; 231 table = intr->table; 232 233 /* Remove all handlers */ 234 for (i = 0; i < intr->n_alloc; i++) 235 bus_teardown_intr(sc->dev, table[i].eih_res, 236 table[i].eih_tag); 237 } 238 239 static int 240 sfxge_intr_alloc(struct sfxge_softc *sc, int count) 241 { 242 device_t dev; 243 struct sfxge_intr_hdl *table; 244 struct sfxge_intr *intr; 245 struct resource *res; 246 int rid; 247 int error; 248 int i; 249 250 dev = sc->dev; 251 intr = &sc->intr; 252 error = 0; 253 254 table = malloc(count * sizeof(struct sfxge_intr_hdl), 255 M_SFXGE, M_WAITOK); 256 intr->table = table; 257 258 for (i = 0; i < count; i++) { 259 rid = i + 1; 260 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 261 RF_SHAREABLE | RF_ACTIVE); 262 if (res == NULL) { 263 device_printf(dev, "Couldn't allocate interrupts for " 264 "message %d\n", rid); 265 error = ENOMEM; 266 break; 267 } 268 table[i].eih_rid = rid; 269 table[i].eih_res = res; 270 } 271 272 if (error != 0) { 273 count = i - 1; 274 for (i = 0; i < count; i++) 275 bus_release_resource(dev, SYS_RES_IRQ, 276 table[i].eih_rid, table[i].eih_res); 277 } 278 279 return (error); 280 } 281 282 static void 283 sfxge_intr_teardown_msix(struct sfxge_softc *sc) 284 { 285 device_t dev; 286 struct resource *resp; 287 int rid; 288 289 dev = sc->dev; 290 resp = sc->intr.msix_res; 291 292 rid = rman_get_rid(resp); 293 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 294 } 295 296 static int 297 sfxge_intr_setup_msix(struct sfxge_softc *sc) 298 { 299 struct sfxge_intr *intr; 300 struct resource *resp; 301 device_t dev; 302 int count; 303 int rid; 304 305 dev = sc->dev; 306 intr = &sc->intr; 307 308 /* Check if MSI-X is available. */ 309 count = pci_msix_count(dev); 310 if (count == 0) 311 return (EINVAL); 312 313 /* Do not try to allocate more than already estimated EVQ maximum */ 314 KASSERT(sc->evq_max > 0, ("evq_max is zero")); 315 count = MIN(count, sc->evq_max); 316 317 rid = PCIR_BAR(4); 318 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 319 if (resp == NULL) 320 return (ENOMEM); 321 322 if (pci_alloc_msix(dev, &count) != 0) { 323 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 324 return (ENOMEM); 325 } 326 327 /* Allocate interrupt handlers. */ 328 if (sfxge_intr_alloc(sc, count) != 0) { 329 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 330 pci_release_msi(dev); 331 return (ENOMEM); 332 } 333 334 intr->type = EFX_INTR_MESSAGE; 335 intr->n_alloc = count; 336 intr->msix_res = resp; 337 338 return (0); 339 } 340 341 static int 342 sfxge_intr_setup_msi(struct sfxge_softc *sc) 343 { 344 struct sfxge_intr_hdl *table; 345 struct sfxge_intr *intr; 346 device_t dev; 347 int count; 348 int error; 349 350 dev = sc->dev; 351 intr = &sc->intr; 352 table = intr->table; 353 354 /* 355 * Check if MSI is available. All messages must be written to 356 * the same address and on x86 this means the IRQs have the 357 * same CPU affinity. So we only ever allocate 1. 358 */ 359 count = pci_msi_count(dev) ? 1 : 0; 360 if (count == 0) 361 return (EINVAL); 362 363 if ((error = pci_alloc_msi(dev, &count)) != 0) 364 return (ENOMEM); 365 366 /* Allocate interrupt handler. */ 367 if (sfxge_intr_alloc(sc, count) != 0) { 368 pci_release_msi(dev); 369 return (ENOMEM); 370 } 371 372 intr->type = EFX_INTR_MESSAGE; 373 intr->n_alloc = count; 374 375 return (0); 376 } 377 378 static int 379 sfxge_intr_setup_fixed(struct sfxge_softc *sc) 380 { 381 struct sfxge_intr_hdl *table; 382 struct sfxge_intr *intr; 383 struct resource *res; 384 device_t dev; 385 int rid; 386 387 dev = sc->dev; 388 intr = &sc->intr; 389 390 rid = 0; 391 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 392 RF_SHAREABLE | RF_ACTIVE); 393 if (res == NULL) 394 return (ENOMEM); 395 396 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK); 397 table[0].eih_rid = rid; 398 table[0].eih_res = res; 399 400 intr->type = EFX_INTR_LINE; 401 intr->n_alloc = 1; 402 intr->table = table; 403 404 return (0); 405 } 406 407 static const char *const __sfxge_err[] = { 408 "", 409 "SRAM out-of-bounds", 410 "Buffer ID out-of-bounds", 411 "Internal memory parity", 412 "Receive buffer ownership", 413 "Transmit buffer ownership", 414 "Receive descriptor ownership", 415 "Transmit descriptor ownership", 416 "Event queue ownership", 417 "Event queue FIFO overflow", 418 "Illegal address", 419 "SRAM parity" 420 }; 421 422 void 423 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0, 424 uint32_t dword1) 425 { 426 struct sfxge_softc *sc = (struct sfxge_softc *)arg; 427 device_t dev = sc->dev; 428 429 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)", 430 device_get_name(dev), device_get_unit(dev), 431 __sfxge_err[code], dword1, dword0); 432 } 433 434 void 435 sfxge_intr_stop(struct sfxge_softc *sc) 436 { 437 struct sfxge_intr *intr; 438 439 intr = &sc->intr; 440 441 KASSERT(intr->state == SFXGE_INTR_STARTED, 442 ("Interrupts not started")); 443 444 intr->state = SFXGE_INTR_INITIALIZED; 445 446 /* Disable interrupts at the NIC */ 447 efx_intr_disable(sc->enp); 448 449 /* Disable interrupts at the bus */ 450 sfxge_intr_bus_disable(sc); 451 452 /* Tear down common code interrupt bits. */ 453 efx_intr_fini(sc->enp); 454 } 455 456 int 457 sfxge_intr_start(struct sfxge_softc *sc) 458 { 459 struct sfxge_intr *intr; 460 efsys_mem_t *esmp; 461 int rc; 462 463 intr = &sc->intr; 464 esmp = &intr->status; 465 466 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 467 ("Interrupts not initialized")); 468 469 /* Zero the memory. */ 470 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE); 471 472 /* Initialize common code interrupt bits. */ 473 (void)efx_intr_init(sc->enp, intr->type, esmp); 474 475 /* Enable interrupts at the bus */ 476 if ((rc = sfxge_intr_bus_enable(sc)) != 0) 477 goto fail; 478 479 intr->state = SFXGE_INTR_STARTED; 480 481 /* Enable interrupts at the NIC */ 482 efx_intr_enable(sc->enp); 483 484 return (0); 485 486 fail: 487 /* Tear down common code interrupt bits. */ 488 efx_intr_fini(sc->enp); 489 490 intr->state = SFXGE_INTR_INITIALIZED; 491 492 return (rc); 493 } 494 495 void 496 sfxge_intr_fini(struct sfxge_softc *sc) 497 { 498 struct sfxge_intr_hdl *table; 499 struct sfxge_intr *intr; 500 efsys_mem_t *esmp; 501 device_t dev; 502 int i; 503 504 dev = sc->dev; 505 intr = &sc->intr; 506 esmp = &intr->status; 507 table = intr->table; 508 509 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 510 ("intr->state != SFXGE_INTR_INITIALIZED")); 511 512 /* Free DMA memory. */ 513 sfxge_dma_free(esmp); 514 515 /* Free interrupt handles. */ 516 for (i = 0; i < intr->n_alloc; i++) 517 bus_release_resource(dev, SYS_RES_IRQ, 518 table[i].eih_rid, table[i].eih_res); 519 520 if (table[0].eih_rid != 0) 521 pci_release_msi(dev); 522 523 if (intr->msix_res != NULL) 524 sfxge_intr_teardown_msix(sc); 525 526 /* Free the handle table */ 527 free(table, M_SFXGE); 528 intr->table = NULL; 529 intr->n_alloc = 0; 530 531 /* Clear the interrupt type */ 532 intr->type = EFX_INTR_INVALID; 533 534 intr->state = SFXGE_INTR_UNINITIALIZED; 535 } 536 537 int 538 sfxge_intr_init(struct sfxge_softc *sc) 539 { 540 device_t dev; 541 struct sfxge_intr *intr; 542 efsys_mem_t *esmp; 543 int rc; 544 545 dev = sc->dev; 546 intr = &sc->intr; 547 esmp = &intr->status; 548 549 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED, 550 ("Interrupts already initialized")); 551 552 /* Try to setup MSI-X or MSI interrupts if available. */ 553 if ((rc = sfxge_intr_setup_msix(sc)) == 0) 554 device_printf(dev, "Using MSI-X interrupts\n"); 555 else if ((rc = sfxge_intr_setup_msi(sc)) == 0) 556 device_printf(dev, "Using MSI interrupts\n"); 557 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) { 558 device_printf(dev, "Using fixed interrupts\n"); 559 } else { 560 device_printf(dev, "Couldn't setup interrupts\n"); 561 return (ENOMEM); 562 } 563 564 /* Set up DMA for interrupts. */ 565 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0) 566 return (ENOMEM); 567 568 intr->state = SFXGE_INTR_INITIALIZED; 569 570 return (0); 571 } 572